From 89ff7ac0a31903bd9b84d5c7408726098727980b Mon Sep 17 00:00:00 2001 From: cinnaboot Date: Fri, 18 Oct 2019 00:25:40 -0400 Subject: [PATCH] modify db names to prepare for tv queries --- video_metadata/schema.sql | 83 +++++++++++++----------- video_metadata/views/file_listing.py | 2 +- video_metadata/views/tmdb.py | 94 ++++++++++++++++------------ 3 files changed, 99 insertions(+), 80 deletions(-) diff --git a/video_metadata/schema.sql b/video_metadata/schema.sql index da86498..e1ad6e4 100644 --- a/video_metadata/schema.sql +++ b/video_metadata/schema.sql @@ -1,24 +1,53 @@ -DROP TABLE IF EXISTS movie_query_mapping; -CREATE TABLE movie_query_mapping( - query_id int, - movie_id int, - foreign key(query_id) references movie_queries(query_id), - foreign key(movie_id) references movie_details(movie_id) +DROP TABLE IF EXISTS query_cache; +CREATE TABLE query_cache( + query_id integer primary key autoincrement, + query text, + year integer ); -DROP TABLE IF EXISTS movie_paths; -CREATE TABLE movie_paths( - movie_id int not null, - movie_path text unique not null, +DROP TABLE IF EXISTS query_details; +CREATE TABLE query_details( + media_type text, -- NOTE: tv, movie, or unkown + movie_id integer, -- NOTE: only one of movie_id, tv_id should be set + tv_id integer, + title text, + overview text, + release_date date, + original_language text, + poster_path text +); + +DROP TABLE IF EXISTS query_mapping; +CREATE TABLE query_mapping( + query_id integer, + movie_id integer, -- NOTE: only one of movie_id, tv_id should be set + tv_id integer, + foreign key(query_id) references query_cache(query_id), + foreign key(movie_id) references query_details(movie_id), + foreign key(tv_id) references query_details(tv_id) +); + +DROP TABLE IF EXISTS path_cache; +CREATE TABLE path_cache( + path text unique not null, + movie_id integer, + tv_id integer, add_date datetime, -- NOTE: date the path was mapped - foreign key(movie_id) references movie_details(movie_id) + foreign key(movie_id) references query_details(movie_id), + foreign key(tv_id) references query_details(tv_id) ); DROP TABLE IF EXISTS movie_genres; CREATE TABLE movie_genres( - genre_id integer primary key not null, - genre_name + id integer primary key not null, + name text +); + +DROP TABLE IF EXISTS tv_genres; +CREATE TABLE tv_genres( + id integer primary key not null, + name text ); DROP TABLE IF EXISTS movie_genre_mapping; @@ -26,24 +55,7 @@ CREATE TABLE movie_genre_mapping( genre_id integer, movie_id integer, foreign key(genre_id) references movie_genres(genre_id), - foreign key(movie_id) references movie_details(movie_id) -); - -DROP TABLE IF EXISTS movie_queries; -CREATE TABLE movie_queries( - query_id integer primary key autoincrement, - query text, - year integer -); - -DROP TABLE IF EXISTS movie_details; -CREATE TABLE movie_details( - movie_id primary key not null, - title text, - overview text, - release_date date, - original_language text, - poster_path text + foreign key(movie_id) references query_details(movie_id) ); DROP TABLE IF EXISTS actor_details; @@ -59,15 +71,10 @@ CREATE TABLE actor_movie_mapping( movie_id integer, actor_id integer, character_name text, - foreign key(movie_id) references movie_details(movie_id), + foreign key(movie_id) references query_details(movie_id), foreign key(actor_id) references actor_details(actor_id) ); --- TODO: tv shows +-- TODO: tv seasons, episodes -DROP TABLE IF EXISTS tv_queries; -CREATE TABLE tv_queries( - query text, - year integer -); diff --git a/video_metadata/views/file_listing.py b/video_metadata/views/file_listing.py index c69e18d..ad3112d 100644 --- a/video_metadata/views/file_listing.py +++ b/video_metadata/views/file_listing.py @@ -74,7 +74,7 @@ def getPathInfo(entry): pe.file_extension = ext # check if the path is already stored in db cur = db.get_db().cursor() - cur.execute('SELECT COUNT(*) FROM movie_paths WHERE movie_path = ?', + cur.execute('SELECT COUNT(*) FROM path_cache WHERE path = ?', (pe.rel_path, ) ) pe.in_db = cur.fetchone()[0] > 0 diff --git a/video_metadata/views/tmdb.py b/video_metadata/views/tmdb.py index 9970a70..6f183a1 100644 --- a/video_metadata/views/tmdb.py +++ b/video_metadata/views/tmdb.py @@ -37,7 +37,7 @@ bp = Blueprint('tmdb', __name__, url_prefix='/tmdb') #################### # data structures -class query_type(Enum): +class media_type(Enum): UNKNOWN = 0 MOVIE = 1 TV_SHOW = 2 @@ -49,7 +49,7 @@ class query_data: movie_year: int = 0 parsed_query_str: str = '' query_params: str = '' - query_type: query_type = query_type.UNKNOWN + media_type: media_type = media_type.UNKNOWN raw_query_str: str = '' rel_path: str = '' @@ -78,7 +78,6 @@ class view_result: # add flask 1.1 to requirements when alpine distro updates flask # NOTE: haven't found a better way to json encode dataclasses :( class result_encoder(json.JSONEncoder): - def default(self, obj): if isinstance(obj, view_result): return obj.__dict__ @@ -116,18 +115,18 @@ def list_movie_test(sorting = 'title'): cur = conn.cursor() view_results = [] query = f'''SELECT - md.*, mp.movie_path + md.*, pc.path FROM - movie_details md - JOIN movie_paths mp ON mp.movie_id=md.movie_id + query_details md + JOIN path_cache pc ON pc.movie_id=md.movie_id ORDER BY {orderby}''' - # TODO: should probably loop on movie_id, and add a list of movie_paths because + # TODO: should probably loop on movie_id, and add a list of paths because # there can be multiple paths associated with a movie_id for row in cur.execute(query): vr = buildViewResult(row) - vr.rel_path = row['movie_path'] + vr.rel_path = row['path'] view_results.append(vr) return render_template('tmdb/list_movie_test.html', @@ -196,17 +195,21 @@ def query_path_json(): path = request.args['rel_path'] cur = db.get_db().cursor() q = '''SELECT - md.*, mp.* + md.*, pc.* FROM - movie_paths mp - JOIN movie_details md ON md.movie_id = mp.movie_id + path_cache pc + JOIN query_details md ON md.movie_id = pc.movie_id WHERE - mp.movie_path = ? + pc.path = ? LIMIT 1''' cur.execute(q, (path, )) row = cur.fetchone() + if errors: + for e in errors: + current_app.logger.error(e) + res = Response(json.dumps(buildViewResult(row), cls=result_encoder)) res.headers.set('Content-Type', 'application/json') return res @@ -227,7 +230,7 @@ def create_mapping(): cur = conn.cursor() current_app.logger.debug(f'{util.getFunctionName()}, mapping path "{rel_path}" for id: {movie_id}') try: - cur.execute('INSERT INTO movie_paths(movie_id, movie_path, add_date) VALUES(?, ?, datetime())', + cur.execute('INSERT INTO path_cache(movie_id, path, add_date) VALUES(?, ?, datetime())', (movie_id, rel_path )) except DatabaseError as e: errors.append(str(e)) @@ -250,7 +253,7 @@ def doQueryAlgorithm(query:query_data) -> dict: errors, view_results = [], [] response = response_data() - # TODO: if query_obj.query_type is still 'UNKNOWN' from parseQueryStr(), make queries to + # TODO: if query_obj.media_type is still 'UNKNOWN' from parseQueryStr(), make queries to # both tmdb endpoints (movie and tv) if query.raw_query_str and parseQueryStr(query): if isQueryCached(query.parsed_query_str): @@ -271,7 +274,7 @@ def doQueryAlgorithm(query:query_data) -> dict: # see) https://developers.themoviedb.org/3/search/search-movies if 'results' in response.parsed_response: for result in response.parsed_response['results']: - view_results.append(parseTMDBResult(result, query.query_type)) + view_results.append(parseTMDBResult(result, query.media_type)) storeTMDBQueryResults(query, view_results) @@ -297,26 +300,28 @@ def parseQueryStr(query_obj): if i > 1878 and i <= current_year: query_obj.parsed_query_str = s[0:m.start()].strip() # NOTE: slice query string to year match query_obj.movie_year = i - query_obj.query_type = query_type.MOVIE - break + query_obj.media_type = media_type.MOVIE + return True - # TODO: parse tv show titles - if query_obj.query_type == query_type.UNKNOWN: - query_obj.errors.append('{}: unable to determine query type for {}'.format( - util.getFunctionName(), query_obj.raw_query_str) - ) - return False + # NOTE: if not a movie, check for a string that looks like a tv show + if query_obj.media_type == media_type.UNKNOWN: + m = re.compile(r's[0-9]{2}|season?[0-9]{2}', re.IGNORECASE).search(s) - return True + if m: + query_obj.parsed_query_str = s[0:m.start()].strip() + query_obj.media_type = media_type.TV_SHOW + return True + + return False def isQueryCached(query_str): conn = db.get_db() cur = conn.cursor() - cur.execute('SELECT COUNT(*) FROM movie_queries WHERE query=?', (query_str,)) + cur.execute('SELECT COUNT(*) FROM query_cache WHERE query=?', (query_str,)) count = cur.fetchone()[0] current_app.logger.debug( - f'{util.getFunctionName()}, movie_queries count: {count}, query: "{query_str}"' + f'{util.getFunctionName()}, query_cache count: {count}, query: "{query_str}"' ) return (count > 0) @@ -332,9 +337,9 @@ def makeLocalQuery(query_obj): fancy_query = '''SELECT md.* FROM - movie_queries mq - JOIN movie_query_mapping map ON mq.query_id = map.query_id - JOIN movie_details md ON md.movie_id = map.movie_id + query_cache mq + JOIN query_mapping map ON mq.query_id = map.query_id + JOIN query_details md ON md.movie_id = map.movie_id WHERE mq.query = ?''' @@ -362,14 +367,18 @@ def buildViewResult(db_row): # TODO: add info and debug logging for network requests # TODO: need to check response headers for rate limiting requests def makeTMDBQuery(query_obj, response_obj): - if query_obj.query_type == query_type.MOVIE: + if query_obj.media_type == media_type.MOVIE: search_part = TMDB_SEARCH_MOVIE - elif query_obj.query_type == query_type.TV_SHOW: + elif query_obj.media_type == media_type.TV_SHOW: search_part = TMDB_SEARCH_TV else: query_obj.errors.append('%s(), query type is not set' % util.getFunctionName()) return False + current_app.logger.info( + f'{util.getFunctionName()}, making remote query to TMDB API: ' + f'{query_obj.media_type}, "{query_obj.parsed_query_str}"' + ) request_headers = {"Accept": "text/html,application/xhtml+xml,application/xml"} query_obj.query_params = urllib.parse.urlencode( { @@ -412,20 +421,22 @@ def makeTMDBQuery(query_obj, response_obj): # NOTE: @param result is parsed JSON object from tmdb API -def parseTMDBResult(result, query_type): +def parseTMDBResult(result, media_type): parsed = view_result() parsed.id = result['id'] - parsed.title = result['title'] - parsed.release_date = result['release_date'] parsed.overview = result['overview'] parsed.genre_ids = result['genre_ids'] parsed.original_language = result['original_language'] parsed.poster_path = result['poster_path'] - if query_type == query_type.MOVIE: + if media_type == media_type.MOVIE: + parsed.title = result['title'] + parsed.release_date = result['release_date'] parsed.tmdb_link = TMDB_MOVIE_LINK_URL + str(result['id']) - elif query_type == query_type.TV_SHOW: + elif media_type == media_type.TV_SHOW: + parsed.title = result['name'] + parsed.release_date = result['first_air_date'] parsed.tmdb_link = TMDB_TV_LINK_URL + str(result['id']) return parsed @@ -441,7 +452,7 @@ def storeTMDBQueryResults(query_obj, result_list): try: cur.execute('BEGIN TRANSACTION') - cur.execute('INSERT INTO movie_queries(query, year) VALUES (?, ?)', + cur.execute('INSERT INTO query_cache(query, year) VALUES (?, ?)', (query_obj.parsed_query_str, query_obj.movie_year) ) query_id = cur.lastrowid @@ -451,7 +462,7 @@ def storeTMDBQueryResults(query_obj, result_list): for result in result_list: # NOTE: check stored details before trying to add to local db to avoid unique key # constraint error - cur.execute('SELECT COUNT(*) FROM movie_details WHERE movie_id=?', (result.id,)) + cur.execute('SELECT COUNT(*) FROM query_details WHERE movie_id=?', (result.id,)) count = cur.fetchone()[0] if count > 0: current_app.logger.debug(f'{util.getFunctionName()}, ' @@ -460,15 +471,16 @@ def storeTMDBQueryResults(query_obj, result_list): break try: - current_app.logger.debug(f'{util.getFunctionName()}, adding movie details for "{result.title}"') - cur.execute("INSERT INTO movie_details(" + current_app.logger.debug(f'{util.getFunctionName()}, adding query details for "{result.title}"') + # TODO: need to add tv_id to query_cache + cur.execute("INSERT INTO query_details(" + "movie_id, title, overview, release_date, original_language, poster_path)" + " VALUES (?,?,?,?,?,?)", (result.id, result.title, result.overview, result.release_date, result.original_language, result.poster_path) ) - cur.execute("INSERT INTO movie_query_mapping(query_id, movie_id)" + cur.execute("INSERT INTO query_mapping(query_id, movie_id)" + " VALUES (?,?)", (query_id, result.id) ) except DatabaseError as e: