From d0746cec5ade00b0d3bd246e047fb050c453f7b8 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sun, 18 Dec 2022 22:27:42 -0700 Subject: [PATCH] Fix server 2 always running async runner. Speedup plex get watched --- src/functions.py | 45 +++++++------ src/jellyfin.py | 14 +++-- src/main.py | 58 ++++++++++++----- src/plex.py | 77 +++++++++++------------ test/test_main.py | 156 +++++++++++++++++++++++----------------------- 5 files changed, 185 insertions(+), 165 deletions(-) diff --git a/src/functions.py b/src/functions.py index 51e4293..80ee608 100644 --- a/src/functions.py +++ b/src/functions.py @@ -178,28 +178,6 @@ def combine_watched_dicts(dicts: list): return combined_dict -def future_thread_executor(args: list, workers: int = -1): - futures_list = [] - results = [] - - if workers == -1: - workers = min(32, os.cpu_count() * 1.25) - - with ThreadPoolExecutor(max_workers=workers) as executor: - for arg in args: - # * arg unpacks the list into actual arguments - futures_list.append(executor.submit(*arg)) - - for future in futures_list: - try: - result = future.result() - results.append(result) - except Exception as e: - raise Exception(e) - - return results - - def cleanup_watched( watched_list_1, watched_list_2, user_mapping=None, library_mapping=None ): @@ -326,4 +304,25 @@ def is_episode_in_dict(episode, episode_watched_list_2_keys_dict): return True # If the loop completes without finding a match, return False - return False \ No newline at end of file + return False + +def future_thread_executor(args: list, workers: int = -1): + futures_list = [] + results = [] + + if workers == -1: + workers = min(32, os.cpu_count() * 2) + + with ThreadPoolExecutor(max_workers=workers) as executor: + for arg in args: + # * arg unpacks the list into actual arguments + futures_list.append(executor.submit(*arg)) + + for future in futures_list: + try: + result = future.result() + results.append(result) + except Exception as e: + raise Exception(e) + + return results diff --git a/src/jellyfin.py b/src/jellyfin.py index 364b12e..ace4958 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -239,7 +239,8 @@ class Jellyfin: ][episode_identifiers["season_name"]].append( episode_guids ) - + + logger(f"Jellyfin: Got watched for {user_name} in library {library_title}") return user_watched except Exception as e: logger( @@ -342,7 +343,7 @@ class Jellyfin: for user_name, user_id in users.items(): watched.append( - await self.get_users_watched( + self.get_users_watched( user_name, user_id, blacklist_library, @@ -353,6 +354,7 @@ class Jellyfin: ) ) + watched = await asyncio.gather(*watched, return_exceptions=True) for user_watched in watched: user_watched_temp = combine_watched_dicts(user_watched) for user, user_watched_temp in user_watched_temp.items(): @@ -534,12 +536,12 @@ class Jellyfin: else: logger( f"Jellyfin: Skipping episode {jellyfin_episode['Name']} as it is not in mark list for {user_name}", - 1, + 3, ) else: logger( f"Jellyfin: Skipping show {jellyfin_show['Name']} as it is not in mark list for {user_name}", - 1, + 3, ) if ( @@ -618,13 +620,13 @@ class Jellyfin: else: logger( f"Jellyfin: Library {library} or {library_other} not found in library list", - 2, + 1, ) continue else: logger( f"Jellyfin: Library {library} not found in library list", - 2, + 1, ) continue diff --git a/src/main.py b/src/main.py index b758d03..1f690ca 100644 --- a/src/main.py +++ b/src/main.py @@ -310,6 +310,32 @@ def generate_server_connections(): return servers +def get_server_watched(server_connection: list, users: dict, blacklist_library: list, whitelist_library: list, blacklist_library_type: list, whitelist_library_type: list, library_mapping: dict): + if server_connection[0] == "plex": + return server_connection[1].get_watched( + users, + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + library_mapping, + ) + elif server_connection[0] == "jellyfin": + return asyncio.run(server_connection[1].get_watched( + users, + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + library_mapping, + )) + + +def update_server_watched(server_connection: list, server_watched_filtered: dict, user_mapping: dict, library_mapping: dict, dryrun: bool): + if server_connection[0] == "plex": + server_connection[1].update_watched(server_watched_filtered, user_mapping, library_mapping, dryrun) + elif server_connection[0] == "jellyfin": + asyncio.run(server_connection[1].update_watched(server_watched_filtered, user_mapping, library_mapping, dryrun)) def main_loop(): logfile = os.getenv("LOGFILE", "log.log") @@ -379,7 +405,8 @@ def main_loop(): ) logger("Creating watched lists", 1) - server_1_watched = server_1_connection.get_watched( + server_1_watched = get_server_watched( + server_1, server_1_users, blacklist_library, whitelist_library, @@ -388,15 +415,14 @@ def main_loop(): library_mapping, ) logger("Finished creating watched list server 1", 1) - server_2_watched = asyncio.run( - server_2_connection.get_watched( - server_2_users, - blacklist_library, - whitelist_library, - blacklist_library_type, - whitelist_library_type, - library_mapping, - ) + server_2_watched = get_server_watched( + server_2, + server_2_users, + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + library_mapping, ) logger("Finished creating watched list server 2", 1) logger(f"Server 1 watched: {server_1_watched}", 3) @@ -421,13 +447,12 @@ def main_loop(): 1, ) - server_1_connection.update_watched( - server_2_watched_filtered, user_mapping, library_mapping, dryrun + update_server_watched( + server_1, server_2_watched_filtered, user_mapping, library_mapping, dryrun ) - asyncio.run( - server_2_connection.update_watched( - server_1_watched_filtered, user_mapping, library_mapping, dryrun - ) + + update_server_watched( + server_2, server_1_watched_filtered, user_mapping, library_mapping, dryrun ) @@ -455,6 +480,7 @@ def main(): logger(error, log_type=2) logger(traceback.format_exc(), 2) + logger(f"Retrying in {sleep_duration}", log_type=0) sleep(sleep_duration) diff --git a/src/plex.py b/src/plex.py index 30fb391..267f22f 100644 --- a/src/plex.py +++ b/src/plex.py @@ -54,14 +54,13 @@ def get_user_watched(user, user_plex, library): user_watched[user_name][library.title] = {} library_videos = user_plex.library.section(library.title) - for show in library_videos.search(unwatched=False): + shows = library_videos.search(unwatched=False) + for show in shows: show_guids = {} for show_guid in show.guids: - # Extract after :// from guid.id - show_guid_source = ( - re.search(r"(.*)://", show_guid.id).group(1).lower() - ) - show_guid_id = re.search(r"://(.*)", show_guid.id).group(1) + # Extract source and id from guid.id + m = re.match(r"(.*)://(.*)", show_guid.id) + show_guid_source, show_guid_id = m.group(1).lower(), m.group(2) show_guids[show_guid_source] = show_guid_id show_guids["title"] = show.title @@ -69,40 +68,33 @@ def get_user_watched(user, user_plex, library): [x.split("/")[-1] for x in show.locations] ) show_guids = frozenset(show_guids.items()) + + # Get all watched episodes for show + episode_guids = {} + for episode in show.watched(): + if episode.viewCount > 0: + episode_guids_temp = {} + for guid in episode.guids: + # Extract after :// from guid.id + m = re.match(r"(.*)://(.*)", guid.id) + guid_source, guid_id = m.group(1).lower(), m.group(2) + episode_guids_temp[guid_source] = guid_id - for season in show.seasons(): - episode_guids = [] - for episode in season.episodes(): - if episode.viewCount > 0: - episode_guids_temp = {} - for guid in episode.guids: - # Extract after :// from guid.id - guid_source = ( - re.search(r"(.*)://", guid.id).group(1).lower() - ) - guid_id = re.search(r"://(.*)", guid.id).group(1) - episode_guids_temp[guid_source] = guid_id - - episode_guids_temp["locations"] = tuple( - [x.split("/")[-1] for x in episode.locations] - ) - episode_guids.append(episode_guids_temp) - - if episode_guids: - # append show, season, episode - if show_guids not in user_watched[user_name][library.title]: - user_watched[user_name][library.title][show_guids] = {} - if ( - season.title - not in user_watched[user_name][library.title][show_guids] - ): - user_watched[user_name][library.title][show_guids][ - season.title - ] = {} - user_watched[user_name][library.title][show_guids][ - season.title - ] = episode_guids + episode_guids_temp["locations"] = tuple( + [x.split("/")[-1] for x in episode.locations] + ) + if episode.parentTitle not in episode_guids: + episode_guids[episode.parentTitle] = [] + episode_guids[episode.parentTitle].append(episode_guids_temp) + + if episode_guids: + # append show, season, episode + if show_guids not in user_watched[user_name][library.title]: + user_watched[user_name][library.title][show_guids] = {} + + user_watched[user_name][library.title][show_guids] = episode_guids + logger(f"Plex: Got watched for {user_name} in library {library.title}", 0) return user_watched except Exception as e: logger( @@ -223,12 +215,12 @@ def update_user_watched(user, user_plex, library, videos, dryrun): else: logger( f"Plex: Skipping episode {episode_search.title} as it is not in mark list for {user.title}", - 1, + 3, ) else: logger( f"Plex: Skipping show {show_search.title} as it is not in mark list for {user.title}", - 1, + 3, ) if not videos_movies_ids and not videos_shows_ids and not videos_episodes_ids: @@ -415,12 +407,13 @@ class Plex: else: logger( f"Plex: Library {library} or {library_other} not found in library list", - 2, + 1, ) continue else: logger( - f"Plex: Library {library} not found in library list", 2 + f"Plex: Library {library} not found in library list", + 1, ) continue diff --git a/test/test_main.py b/test/test_main.py index 8352f10..fbe990b 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,78 +1,78 @@ -import sys -import os - -# getting the name of the directory -# where the this file is present. -current = os.path.dirname(os.path.realpath(__file__)) - -# Getting the parent directory name -# where the current directory is present. -parent = os.path.dirname(current) - -# adding the parent directory to -# the sys.path. -sys.path.append(parent) - -from src.main import setup_black_white_lists - - -def test_setup_black_white_lists(): - # Simple - blacklist_library = "library1, library2" - whitelist_library = "library1, library2" - blacklist_library_type = "library_type1, library_type2" - whitelist_library_type = "library_type1, library_type2" - blacklist_users = "user1, user2" - whitelist_users = "user1, user2" - - ( - results_blacklist_library, - return_whitelist_library, - return_blacklist_library_type, - return_whitelist_library_type, - return_blacklist_users, - return_whitelist_users, - ) = setup_black_white_lists( - blacklist_library, - whitelist_library, - blacklist_library_type, - whitelist_library_type, - blacklist_users, - whitelist_users, - ) - - assert results_blacklist_library == ["library1", "library2"] - assert return_whitelist_library == ["library1", "library2"] - assert return_blacklist_library_type == ["library_type1", "library_type2"] - assert return_whitelist_library_type == ["library_type1", "library_type2"] - assert return_blacklist_users == ["user1", "user2"] - assert return_whitelist_users == ["user1", "user2"] - - # Library Mapping and user mapping - library_mapping = {"library1": "library3"} - user_mapping = {"user1": "user3"} - - ( - results_blacklist_library, - return_whitelist_library, - return_blacklist_library_type, - return_whitelist_library_type, - return_blacklist_users, - return_whitelist_users, - ) = setup_black_white_lists( - blacklist_library, - whitelist_library, - blacklist_library_type, - whitelist_library_type, - blacklist_users, - whitelist_users, - library_mapping, - user_mapping, - ) - - assert results_blacklist_library == ["library1", "library2", "library3"] - assert return_whitelist_library == ["library1", "library2", "library3"] - assert return_blacklist_library_type == ["library_type1", "library_type2"] - assert return_whitelist_library_type == ["library_type1", "library_type2"] - assert return_blacklist_users == ["user1", "user2", "user3"] - assert return_whitelist_users == ["user1", "user2", "user3"] +import sys +import os + +# getting the name of the directory +# where the this file is present. +current = os.path.dirname(os.path.realpath(__file__)) + +# Getting the parent directory name +# where the current directory is present. +parent = os.path.dirname(current) + +# adding the parent directory to +# the sys.path. +sys.path.append(parent) + +from src.main import setup_black_white_lists + + +def test_setup_black_white_lists(): + # Simple + blacklist_library = "library1, library2" + whitelist_library = "library1, library2" + blacklist_library_type = "library_type1, library_type2" + whitelist_library_type = "library_type1, library_type2" + blacklist_users = "user1, user2" + whitelist_users = "user1, user2" + + ( + results_blacklist_library, + return_whitelist_library, + return_blacklist_library_type, + return_whitelist_library_type, + return_blacklist_users, + return_whitelist_users, + ) = setup_black_white_lists( + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + blacklist_users, + whitelist_users, + ) + + assert results_blacklist_library == ["library1", "library2"] + assert return_whitelist_library == ["library1", "library2"] + assert return_blacklist_library_type == ["library_type1", "library_type2"] + assert return_whitelist_library_type == ["library_type1", "library_type2"] + assert return_blacklist_users == ["user1", "user2"] + assert return_whitelist_users == ["user1", "user2"] + + # Library Mapping and user mapping + library_mapping = {"library1": "library3"} + user_mapping = {"user1": "user3"} + + ( + results_blacklist_library, + return_whitelist_library, + return_blacklist_library_type, + return_whitelist_library_type, + return_blacklist_users, + return_whitelist_users, + ) = setup_black_white_lists( + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + blacklist_users, + whitelist_users, + library_mapping, + user_mapping, + ) + + assert results_blacklist_library == ["library1", "library2", "library3"] + assert return_whitelist_library == ["library1", "library2", "library3"] + assert return_blacklist_library_type == ["library_type1", "library_type2"] + assert return_whitelist_library_type == ["library_type1", "library_type2"] + assert return_blacklist_users == ["user1", "user2", "user3"] + assert return_whitelist_users == ["user1", "user2", "user3"]