From 50faf061afa91312815147abefcc136a69abc772 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 21 Nov 2022 18:21:29 -0700 Subject: [PATCH 01/11] Remove dockerfile defaults --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c3409ff..bee126d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,16 +6,16 @@ ENV DEBUG_LEVEL 'INFO' ENV SLEEP_DURATION '3600' ENV LOGFILE 'log.log' -ENV USER_MAPPING '{ "User Test": "User Test2" }' -ENV LIBRARY_MAPPING '{ "Shows Test": "TV Shows Test" }' +ENV USER_MAPPING '' +ENV LIBRARY_MAPPING '' -ENV PLEX_BASEURL 'http://localhost:32400' +ENV PLEX_BASEURL '' ENV PLEX_TOKEN '' ENV PLEX_USERNAME '' ENV PLEX_PASSWORD '' ENV PLEX_SERVERNAME '' -ENV JELLYFIN_BASEURL 'http://localhost:8096' +ENV JELLYFIN_BASEURL '' ENV JELLYFIN_TOKEN '' ENV BLACKLIST_LIBRARY '' From 251937431b0ea8f6211dbba6987be83d8078450f Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sun, 18 Dec 2022 01:50:45 -0700 Subject: [PATCH 02/11] Move cleanup_watched to functions and simplify --- src/functions.py | 144 ++++++++++++++++++++++++++++++++- src/main.py | 203 +---------------------------------------------- 2 files changed, 144 insertions(+), 203 deletions(-) diff --git a/src/functions.py b/src/functions.py index fdba5e9..51e4293 100644 --- a/src/functions.py +++ b/src/functions.py @@ -1,4 +1,4 @@ -import os +import os, copy from concurrent.futures import ThreadPoolExecutor from dotenv import load_dotenv @@ -100,6 +100,10 @@ def generate_library_guids_dict(user_list: dict): episode_output_dict = {} movies_output_dict = {} + # Handle the case where user_list is empty or does not contain the expected keys and values + if not user_list: + return show_output_dict, episode_output_dict, movies_output_dict + try: show_output_keys = user_list.keys() show_output_keys = [dict(x) for x in list(show_output_keys)] @@ -162,7 +166,14 @@ def combine_watched_dicts(dicts: list): if key not in combined_dict: combined_dict[key] = {} for subkey, subvalue in value.items(): - combined_dict[key][subkey] = subvalue + if subkey in combined_dict[key]: + # If the subkey already exists in the combined dictionary, + # check if the values are different and raise an exception if they are + if combined_dict[key][subkey] != subvalue: + raise ValueError(f"Conflicting values for subkey '{subkey}' under key '{key}'") + else: + # If the subkey does not exist in the combined dictionary, add it + combined_dict[key][subkey] = subvalue return combined_dict @@ -187,3 +198,132 @@ def future_thread_executor(args: list, workers: int = -1): raise Exception(e) return results + + +def cleanup_watched( + watched_list_1, watched_list_2, user_mapping=None, library_mapping=None +): + modified_watched_list_1 = copy.deepcopy(watched_list_1) + + # remove entries from watched_list_1 that are in watched_list_2 + for user_1 in watched_list_1: + user_other = None + if user_mapping: + user_other = search_mapping(user_mapping, user_1) + user_2 = get_other(watched_list_2, user_1, user_other) + if user_2 is None: + continue + + for library_1 in watched_list_1[user_1]: + library_other = None + if library_mapping: + library_other = search_mapping(library_mapping, library_1) + library_2 = get_other(watched_list_2[user_2], library_1, library_other) + if library_2 is None: + continue + + ( + _, + episode_watched_list_2_keys_dict, + movies_watched_list_2_keys_dict, + ) = generate_library_guids_dict(watched_list_2[user_2][library_2]) + + # Movies + if isinstance(watched_list_1[user_1][library_1], list): + for movie in watched_list_1[user_1][library_1]: + if is_movie_in_dict(movie, movies_watched_list_2_keys_dict): + logger(f"Removing {movie} from {library_1}", 3) + modified_watched_list_1[user_1][library_1].remove(movie) + + # TV Shows + elif isinstance(watched_list_1[user_1][library_1], dict): + for show_key_1 in watched_list_1[user_1][library_1].keys(): + show_key_dict = dict(show_key_1) + for season in watched_list_1[user_1][library_1][show_key_1]: + for episode in watched_list_1[user_1][library_1][show_key_1][season]: + if is_episode_in_dict(episode, episode_watched_list_2_keys_dict): + if episode in modified_watched_list_1[user_1][library_1][show_key_1][season]: + logger(f"Removing {episode} from {show_key_dict['title']}", 3) + modified_watched_list_1[user_1][library_1][show_key_1][season].remove(episode) + + # Remove empty seasons + if len(modified_watched_list_1[user_1][library_1][show_key_1][season]) == 0: + if season in modified_watched_list_1[user_1][library_1][show_key_1]: + logger(f"Removing {season} from {show_key_dict['title']} because it is empty", 3) + del modified_watched_list_1[user_1][library_1][show_key_1][season] + + # Remove empty shows + if len(modified_watched_list_1[user_1][library_1][show_key_1]) == 0: + if show_key_1 in modified_watched_list_1[user_1][library_1]: + logger(f"Removing {show_key_dict['title']} because it is empty", 3) + del modified_watched_list_1[user_1][library_1][show_key_1] + + for user_1 in watched_list_1: + for library_1 in watched_list_1[user_1]: + if library_1 in modified_watched_list_1[user_1]: + # If library is empty then remove it + if len(modified_watched_list_1[user_1][library_1]) == 0: + logger(f"Removing {library_1} from {user_1} because it is empty", 1) + del modified_watched_list_1[user_1][library_1] + + if user_1 in modified_watched_list_1: + # If user is empty delete user + if len(modified_watched_list_1[user_1]) == 0: + logger(f"Removing {user_1} from watched list 1 because it is empty", 1) + del modified_watched_list_1[user_1] + + return modified_watched_list_1 + + +def get_other(watched_list_2, object_1, object_2): + if object_1 in watched_list_2: + return object_1 + elif object_2 in watched_list_2: + return object_2 + else: + logger(f"{object_1} and {object_2} not found in watched list 2", 1) + return None + + +def is_movie_in_dict(movie, movies_watched_list_2_keys_dict): + # Iterate through the keys and values of the movie dictionary + for movie_key, movie_value in movie.items(): + # If the key is "locations", check if the "locations" key is present in the movies_watched_list_2_keys_dict dictionary + if movie_key == "locations": + if "locations" in movies_watched_list_2_keys_dict.keys(): + # Iterate through the locations in the movie dictionary + for location in movie_value: + # If the location is in the movies_watched_list_2_keys_dict dictionary, return True + if location in movies_watched_list_2_keys_dict["locations"]: + return True + # If the key is not "locations", check if the movie_key is present in the movies_watched_list_2_keys_dict dictionary + else: + if movie_key in movies_watched_list_2_keys_dict.keys(): + # If the movie_value is in the movies_watched_list_2_keys_dict dictionary, return True + if movie_value in movies_watched_list_2_keys_dict[movie_key]: + return True + + # If the loop completes without finding a match, return False + return False + + +def is_episode_in_dict(episode, episode_watched_list_2_keys_dict): + # Iterate through the keys and values of the episode dictionary + for episode_key, episode_value in episode.items(): + # If the key is "locations", check if the "locations" key is present in the episode_watched_list_2_keys_dict dictionary + if episode_key == "locations": + if "locations" in episode_watched_list_2_keys_dict.keys(): + # Iterate through the locations in the episode dictionary + for location in episode_value: + # If the location is in the episode_watched_list_2_keys_dict dictionary, return True + if location in episode_watched_list_2_keys_dict["locations"]: + return True + # If the key is not "locations", check if the episode_key is present in the episode_watched_list_2_keys_dict dictionary + else: + if episode_key in episode_watched_list_2_keys_dict.keys(): + # If the episode_value is in the episode_watched_list_2_keys_dict dictionary, return True + if episode_value in episode_watched_list_2_keys_dict[episode_key]: + return True + + # If the loop completes without finding a match, return False + return False \ No newline at end of file diff --git a/src/main.py b/src/main.py index ade6a3d..b758d03 100644 --- a/src/main.py +++ b/src/main.py @@ -1,4 +1,4 @@ -import copy, os, traceback, json, asyncio +import os, traceback, json, asyncio from dotenv import load_dotenv from time import sleep, perf_counter @@ -6,212 +6,13 @@ from src.functions import ( logger, str_to_bool, search_mapping, - generate_library_guids_dict, + cleanup_watched, ) from src.plex import Plex from src.jellyfin import Jellyfin load_dotenv(override=True) - -def cleanup_watched( - watched_list_1, watched_list_2, user_mapping=None, library_mapping=None -): - modified_watched_list_1 = copy.deepcopy(watched_list_1) - - # remove entries from plex_watched that are in jellyfin_watched - for user_1 in watched_list_1: - user_other = None - if user_mapping: - user_other = search_mapping(user_mapping, user_1) - if user_1 in modified_watched_list_1: - if user_1 in watched_list_2: - user_2 = user_1 - elif user_other in watched_list_2: - user_2 = user_other - else: - logger(f"User {user_1} and {user_other} not found in watched list 2", 1) - continue - - for library_1 in watched_list_1[user_1]: - library_other = None - if library_mapping: - library_other = search_mapping(library_mapping, library_1) - if library_1 in modified_watched_list_1[user_1]: - if library_1 in watched_list_2[user_2]: - library_2 = library_1 - elif library_other in watched_list_2[user_2]: - library_2 = library_other - else: - logger( - f"library {library_1} and {library_other} not found in watched list 2", - 1, - ) - continue - - ( - _, - episode_watched_list_2_keys_dict, - movies_watched_list_2_keys_dict, - ) = generate_library_guids_dict(watched_list_2[user_2][library_2]) - - # Movies - if isinstance(watched_list_1[user_1][library_1], list): - for movie in watched_list_1[user_1][library_1]: - movie_found = False - for movie_key, movie_value in movie.items(): - if movie_key == "locations": - if ( - "locations" - in movies_watched_list_2_keys_dict.keys() - ): - for location in movie_value: - if ( - location - in movies_watched_list_2_keys_dict[ - "locations" - ] - ): - movie_found = True - break - else: - if ( - movie_key - in movies_watched_list_2_keys_dict.keys() - ): - if ( - movie_value - in movies_watched_list_2_keys_dict[ - movie_key - ] - ): - movie_found = True - - if movie_found: - logger(f"Removing {movie} from {library_1}", 3) - modified_watched_list_1[user_1][library_1].remove( - movie - ) - break - - # TV Shows - elif isinstance(watched_list_1[user_1][library_1], dict): - # Generate full list of provider ids for episodes in watch_list_2 to easily compare if they exist in watch_list_1 - - for show_key_1 in watched_list_1[user_1][library_1].keys(): - show_key_dict = dict(show_key_1) - for season in watched_list_1[user_1][library_1][show_key_1]: - for episode in watched_list_1[user_1][library_1][ - show_key_1 - ][season]: - episode_found = False - for episode_key, episode_value in episode.items(): - # If episode_key and episode_value are in episode_watched_list_2_keys_dict exactly, then remove from watch_list_1 - if episode_key == "locations": - if ( - "locations" - in episode_watched_list_2_keys_dict.keys() - ): - for location in episode_value: - if ( - location - in episode_watched_list_2_keys_dict[ - "locations" - ] - ): - episode_found = True - break - - else: - if ( - episode_key - in episode_watched_list_2_keys_dict.keys() - ): - if ( - episode_value - in episode_watched_list_2_keys_dict[ - episode_key - ] - ): - episode_found = True - - if episode_found: - if ( - episode - in modified_watched_list_1[user_1][ - library_1 - ][show_key_1][season] - ): - logger( - f"Removing {episode} from {show_key_dict['title']}", - 3, - ) - modified_watched_list_1[user_1][ - library_1 - ][show_key_1][season].remove(episode) - break - - # Remove empty seasons - if ( - len( - modified_watched_list_1[user_1][library_1][ - show_key_1 - ][season] - ) - == 0 - ): - if ( - season - in modified_watched_list_1[user_1][library_1][ - show_key_1 - ] - ): - logger( - f"Removing {season} from {show_key_dict['title']} because it is empty", - 3, - ) - del modified_watched_list_1[user_1][library_1][ - show_key_1 - ][season] - - # If the show is empty, remove the show - if ( - len( - modified_watched_list_1[user_1][library_1][ - show_key_1 - ] - ) - == 0 - ): - if ( - show_key_1 - in modified_watched_list_1[user_1][library_1] - ): - logger( - f"Removing {show_key_dict['title']} from {library_1} because it is empty", - 1, - ) - del modified_watched_list_1[user_1][library_1][ - show_key_1 - ] - - for user_1 in watched_list_1: - for library_1 in watched_list_1[user_1]: - if library_1 in modified_watched_list_1[user_1]: - # If library is empty then remove it - if len(modified_watched_list_1[user_1][library_1]) == 0: - logger(f"Removing {library_1} from {user_1} because it is empty", 1) - del modified_watched_list_1[user_1][library_1] - - if user_1 in modified_watched_list_1: - # If user is empty delete user - if len(modified_watched_list_1[user_1]) == 0: - logger(f"Removing {user_1} from watched list 1 because it is empty", 1) - del modified_watched_list_1[user_1] - - return modified_watched_list_1 - - def setup_black_white_lists( blacklist_library: str, whitelist_library: str, From d0746cec5ade00b0d3bd246e047fb050c453f7b8 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sun, 18 Dec 2022 22:27:42 -0700 Subject: [PATCH 03/11] Fix server 2 always running async runner. Speedup plex get watched --- src/functions.py | 45 +++++++------ src/jellyfin.py | 14 +++-- src/main.py | 58 ++++++++++++----- src/plex.py | 77 +++++++++++------------ test/test_main.py | 156 +++++++++++++++++++++++----------------------- 5 files changed, 185 insertions(+), 165 deletions(-) diff --git a/src/functions.py b/src/functions.py index 51e4293..80ee608 100644 --- a/src/functions.py +++ b/src/functions.py @@ -178,28 +178,6 @@ def combine_watched_dicts(dicts: list): return combined_dict -def future_thread_executor(args: list, workers: int = -1): - futures_list = [] - results = [] - - if workers == -1: - workers = min(32, os.cpu_count() * 1.25) - - with ThreadPoolExecutor(max_workers=workers) as executor: - for arg in args: - # * arg unpacks the list into actual arguments - futures_list.append(executor.submit(*arg)) - - for future in futures_list: - try: - result = future.result() - results.append(result) - except Exception as e: - raise Exception(e) - - return results - - def cleanup_watched( watched_list_1, watched_list_2, user_mapping=None, library_mapping=None ): @@ -326,4 +304,25 @@ def is_episode_in_dict(episode, episode_watched_list_2_keys_dict): return True # If the loop completes without finding a match, return False - return False \ No newline at end of file + return False + +def future_thread_executor(args: list, workers: int = -1): + futures_list = [] + results = [] + + if workers == -1: + workers = min(32, os.cpu_count() * 2) + + with ThreadPoolExecutor(max_workers=workers) as executor: + for arg in args: + # * arg unpacks the list into actual arguments + futures_list.append(executor.submit(*arg)) + + for future in futures_list: + try: + result = future.result() + results.append(result) + except Exception as e: + raise Exception(e) + + return results diff --git a/src/jellyfin.py b/src/jellyfin.py index 364b12e..ace4958 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -239,7 +239,8 @@ class Jellyfin: ][episode_identifiers["season_name"]].append( episode_guids ) - + + logger(f"Jellyfin: Got watched for {user_name} in library {library_title}") return user_watched except Exception as e: logger( @@ -342,7 +343,7 @@ class Jellyfin: for user_name, user_id in users.items(): watched.append( - await self.get_users_watched( + self.get_users_watched( user_name, user_id, blacklist_library, @@ -353,6 +354,7 @@ class Jellyfin: ) ) + watched = await asyncio.gather(*watched, return_exceptions=True) for user_watched in watched: user_watched_temp = combine_watched_dicts(user_watched) for user, user_watched_temp in user_watched_temp.items(): @@ -534,12 +536,12 @@ class Jellyfin: else: logger( f"Jellyfin: Skipping episode {jellyfin_episode['Name']} as it is not in mark list for {user_name}", - 1, + 3, ) else: logger( f"Jellyfin: Skipping show {jellyfin_show['Name']} as it is not in mark list for {user_name}", - 1, + 3, ) if ( @@ -618,13 +620,13 @@ class Jellyfin: else: logger( f"Jellyfin: Library {library} or {library_other} not found in library list", - 2, + 1, ) continue else: logger( f"Jellyfin: Library {library} not found in library list", - 2, + 1, ) continue diff --git a/src/main.py b/src/main.py index b758d03..1f690ca 100644 --- a/src/main.py +++ b/src/main.py @@ -310,6 +310,32 @@ def generate_server_connections(): return servers +def get_server_watched(server_connection: list, users: dict, blacklist_library: list, whitelist_library: list, blacklist_library_type: list, whitelist_library_type: list, library_mapping: dict): + if server_connection[0] == "plex": + return server_connection[1].get_watched( + users, + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + library_mapping, + ) + elif server_connection[0] == "jellyfin": + return asyncio.run(server_connection[1].get_watched( + users, + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + library_mapping, + )) + + +def update_server_watched(server_connection: list, server_watched_filtered: dict, user_mapping: dict, library_mapping: dict, dryrun: bool): + if server_connection[0] == "plex": + server_connection[1].update_watched(server_watched_filtered, user_mapping, library_mapping, dryrun) + elif server_connection[0] == "jellyfin": + asyncio.run(server_connection[1].update_watched(server_watched_filtered, user_mapping, library_mapping, dryrun)) def main_loop(): logfile = os.getenv("LOGFILE", "log.log") @@ -379,7 +405,8 @@ def main_loop(): ) logger("Creating watched lists", 1) - server_1_watched = server_1_connection.get_watched( + server_1_watched = get_server_watched( + server_1, server_1_users, blacklist_library, whitelist_library, @@ -388,15 +415,14 @@ def main_loop(): library_mapping, ) logger("Finished creating watched list server 1", 1) - server_2_watched = asyncio.run( - server_2_connection.get_watched( - server_2_users, - blacklist_library, - whitelist_library, - blacklist_library_type, - whitelist_library_type, - library_mapping, - ) + server_2_watched = get_server_watched( + server_2, + server_2_users, + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + library_mapping, ) logger("Finished creating watched list server 2", 1) logger(f"Server 1 watched: {server_1_watched}", 3) @@ -421,13 +447,12 @@ def main_loop(): 1, ) - server_1_connection.update_watched( - server_2_watched_filtered, user_mapping, library_mapping, dryrun + update_server_watched( + server_1, server_2_watched_filtered, user_mapping, library_mapping, dryrun ) - asyncio.run( - server_2_connection.update_watched( - server_1_watched_filtered, user_mapping, library_mapping, dryrun - ) + + update_server_watched( + server_2, server_1_watched_filtered, user_mapping, library_mapping, dryrun ) @@ -455,6 +480,7 @@ def main(): logger(error, log_type=2) logger(traceback.format_exc(), 2) + logger(f"Retrying in {sleep_duration}", log_type=0) sleep(sleep_duration) diff --git a/src/plex.py b/src/plex.py index 30fb391..267f22f 100644 --- a/src/plex.py +++ b/src/plex.py @@ -54,14 +54,13 @@ def get_user_watched(user, user_plex, library): user_watched[user_name][library.title] = {} library_videos = user_plex.library.section(library.title) - for show in library_videos.search(unwatched=False): + shows = library_videos.search(unwatched=False) + for show in shows: show_guids = {} for show_guid in show.guids: - # Extract after :// from guid.id - show_guid_source = ( - re.search(r"(.*)://", show_guid.id).group(1).lower() - ) - show_guid_id = re.search(r"://(.*)", show_guid.id).group(1) + # Extract source and id from guid.id + m = re.match(r"(.*)://(.*)", show_guid.id) + show_guid_source, show_guid_id = m.group(1).lower(), m.group(2) show_guids[show_guid_source] = show_guid_id show_guids["title"] = show.title @@ -69,40 +68,33 @@ def get_user_watched(user, user_plex, library): [x.split("/")[-1] for x in show.locations] ) show_guids = frozenset(show_guids.items()) + + # Get all watched episodes for show + episode_guids = {} + for episode in show.watched(): + if episode.viewCount > 0: + episode_guids_temp = {} + for guid in episode.guids: + # Extract after :// from guid.id + m = re.match(r"(.*)://(.*)", guid.id) + guid_source, guid_id = m.group(1).lower(), m.group(2) + episode_guids_temp[guid_source] = guid_id - for season in show.seasons(): - episode_guids = [] - for episode in season.episodes(): - if episode.viewCount > 0: - episode_guids_temp = {} - for guid in episode.guids: - # Extract after :// from guid.id - guid_source = ( - re.search(r"(.*)://", guid.id).group(1).lower() - ) - guid_id = re.search(r"://(.*)", guid.id).group(1) - episode_guids_temp[guid_source] = guid_id - - episode_guids_temp["locations"] = tuple( - [x.split("/")[-1] for x in episode.locations] - ) - episode_guids.append(episode_guids_temp) - - if episode_guids: - # append show, season, episode - if show_guids not in user_watched[user_name][library.title]: - user_watched[user_name][library.title][show_guids] = {} - if ( - season.title - not in user_watched[user_name][library.title][show_guids] - ): - user_watched[user_name][library.title][show_guids][ - season.title - ] = {} - user_watched[user_name][library.title][show_guids][ - season.title - ] = episode_guids + episode_guids_temp["locations"] = tuple( + [x.split("/")[-1] for x in episode.locations] + ) + if episode.parentTitle not in episode_guids: + episode_guids[episode.parentTitle] = [] + episode_guids[episode.parentTitle].append(episode_guids_temp) + + if episode_guids: + # append show, season, episode + if show_guids not in user_watched[user_name][library.title]: + user_watched[user_name][library.title][show_guids] = {} + + user_watched[user_name][library.title][show_guids] = episode_guids + logger(f"Plex: Got watched for {user_name} in library {library.title}", 0) return user_watched except Exception as e: logger( @@ -223,12 +215,12 @@ def update_user_watched(user, user_plex, library, videos, dryrun): else: logger( f"Plex: Skipping episode {episode_search.title} as it is not in mark list for {user.title}", - 1, + 3, ) else: logger( f"Plex: Skipping show {show_search.title} as it is not in mark list for {user.title}", - 1, + 3, ) if not videos_movies_ids and not videos_shows_ids and not videos_episodes_ids: @@ -415,12 +407,13 @@ class Plex: else: logger( f"Plex: Library {library} or {library_other} not found in library list", - 2, + 1, ) continue else: logger( - f"Plex: Library {library} not found in library list", 2 + f"Plex: Library {library} not found in library list", + 1, ) continue diff --git a/test/test_main.py b/test/test_main.py index 8352f10..fbe990b 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,78 +1,78 @@ -import sys -import os - -# getting the name of the directory -# where the this file is present. -current = os.path.dirname(os.path.realpath(__file__)) - -# Getting the parent directory name -# where the current directory is present. -parent = os.path.dirname(current) - -# adding the parent directory to -# the sys.path. -sys.path.append(parent) - -from src.main import setup_black_white_lists - - -def test_setup_black_white_lists(): - # Simple - blacklist_library = "library1, library2" - whitelist_library = "library1, library2" - blacklist_library_type = "library_type1, library_type2" - whitelist_library_type = "library_type1, library_type2" - blacklist_users = "user1, user2" - whitelist_users = "user1, user2" - - ( - results_blacklist_library, - return_whitelist_library, - return_blacklist_library_type, - return_whitelist_library_type, - return_blacklist_users, - return_whitelist_users, - ) = setup_black_white_lists( - blacklist_library, - whitelist_library, - blacklist_library_type, - whitelist_library_type, - blacklist_users, - whitelist_users, - ) - - assert results_blacklist_library == ["library1", "library2"] - assert return_whitelist_library == ["library1", "library2"] - assert return_blacklist_library_type == ["library_type1", "library_type2"] - assert return_whitelist_library_type == ["library_type1", "library_type2"] - assert return_blacklist_users == ["user1", "user2"] - assert return_whitelist_users == ["user1", "user2"] - - # Library Mapping and user mapping - library_mapping = {"library1": "library3"} - user_mapping = {"user1": "user3"} - - ( - results_blacklist_library, - return_whitelist_library, - return_blacklist_library_type, - return_whitelist_library_type, - return_blacklist_users, - return_whitelist_users, - ) = setup_black_white_lists( - blacklist_library, - whitelist_library, - blacklist_library_type, - whitelist_library_type, - blacklist_users, - whitelist_users, - library_mapping, - user_mapping, - ) - - assert results_blacklist_library == ["library1", "library2", "library3"] - assert return_whitelist_library == ["library1", "library2", "library3"] - assert return_blacklist_library_type == ["library_type1", "library_type2"] - assert return_whitelist_library_type == ["library_type1", "library_type2"] - assert return_blacklist_users == ["user1", "user2", "user3"] - assert return_whitelist_users == ["user1", "user2", "user3"] +import sys +import os + +# getting the name of the directory +# where the this file is present. +current = os.path.dirname(os.path.realpath(__file__)) + +# Getting the parent directory name +# where the current directory is present. +parent = os.path.dirname(current) + +# adding the parent directory to +# the sys.path. +sys.path.append(parent) + +from src.main import setup_black_white_lists + + +def test_setup_black_white_lists(): + # Simple + blacklist_library = "library1, library2" + whitelist_library = "library1, library2" + blacklist_library_type = "library_type1, library_type2" + whitelist_library_type = "library_type1, library_type2" + blacklist_users = "user1, user2" + whitelist_users = "user1, user2" + + ( + results_blacklist_library, + return_whitelist_library, + return_blacklist_library_type, + return_whitelist_library_type, + return_blacklist_users, + return_whitelist_users, + ) = setup_black_white_lists( + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + blacklist_users, + whitelist_users, + ) + + assert results_blacklist_library == ["library1", "library2"] + assert return_whitelist_library == ["library1", "library2"] + assert return_blacklist_library_type == ["library_type1", "library_type2"] + assert return_whitelist_library_type == ["library_type1", "library_type2"] + assert return_blacklist_users == ["user1", "user2"] + assert return_whitelist_users == ["user1", "user2"] + + # Library Mapping and user mapping + library_mapping = {"library1": "library3"} + user_mapping = {"user1": "user3"} + + ( + results_blacklist_library, + return_whitelist_library, + return_blacklist_library_type, + return_whitelist_library_type, + return_blacklist_users, + return_whitelist_users, + ) = setup_black_white_lists( + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + blacklist_users, + whitelist_users, + library_mapping, + user_mapping, + ) + + assert results_blacklist_library == ["library1", "library2", "library3"] + assert return_whitelist_library == ["library1", "library2", "library3"] + assert return_blacklist_library_type == ["library_type1", "library_type2"] + assert return_whitelist_library_type == ["library_type1", "library_type2"] + assert return_blacklist_users == ["user1", "user2", "user3"] + assert return_whitelist_users == ["user1", "user2", "user3"] From 370e9bac63e1eaedfafcde7cc346056cac1d2363 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sun, 18 Dec 2022 22:39:03 -0700 Subject: [PATCH 04/11] change get user watched name to avoid mistakes --- src/jellyfin.py | 4 ++-- src/plex.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index ace4958..6abe55e 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -73,7 +73,7 @@ class Jellyfin: logger(f"Jellyfin: Get users failed {e}", 2) raise Exception(e) - async def get_user_watched( + async def get_user_library_watched( self, user_name, user_id, library_type, library_id, library_title ): try: @@ -316,7 +316,7 @@ class Jellyfin: # Get watched for user task = asyncio.ensure_future( - self.get_user_watched( + self.get_user_library_watched( user_name, user_id, library_type, library_id, library_title ) ) diff --git a/src/plex.py b/src/plex.py index 267f22f..f3681df 100644 --- a/src/plex.py +++ b/src/plex.py @@ -21,7 +21,7 @@ class HostNameIgnoringAdapter(requests.adapters.HTTPAdapter): assert_hostname=False, **pool_kwargs) -def get_user_watched(user, user_plex, library): +def get_user_library_watched(user, user_plex, library): try: user_name = user.title.lower() user_watched = {} @@ -341,7 +341,7 @@ class Plex: ) continue - args.append([get_user_watched, user, user_plex, library]) + args.append([get_user_library_watched, user, user_plex, library]) for user_watched in future_thread_executor(args): for user, user_watched_temp in user_watched.items(): From e8faf52b2b910dc2522cef4e1f05576016b51dfe Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 19 Dec 2022 01:35:16 -0700 Subject: [PATCH 05/11] Do not mark shows/movies that do not exist --- src/functions.py | 61 +++++++++++++---- src/jellyfin.py | 168 ++++++++++++++++++++++++++++------------------- src/main.py | 65 +++++++++++++----- src/plex.py | 32 +++++---- 4 files changed, 217 insertions(+), 109 deletions(-) diff --git a/src/functions.py b/src/functions.py index 80ee608..13a0495 100644 --- a/src/functions.py +++ b/src/functions.py @@ -170,7 +170,9 @@ def combine_watched_dicts(dicts: list): # If the subkey already exists in the combined dictionary, # check if the values are different and raise an exception if they are if combined_dict[key][subkey] != subvalue: - raise ValueError(f"Conflicting values for subkey '{subkey}' under key '{key}'") + raise ValueError( + f"Conflicting values for subkey '{subkey}' under key '{key}'" + ) else: # If the subkey does not exist in the combined dictionary, add it combined_dict[key][subkey] = subvalue @@ -218,22 +220,56 @@ def cleanup_watched( for show_key_1 in watched_list_1[user_1][library_1].keys(): show_key_dict = dict(show_key_1) for season in watched_list_1[user_1][library_1][show_key_1]: - for episode in watched_list_1[user_1][library_1][show_key_1][season]: - if is_episode_in_dict(episode, episode_watched_list_2_keys_dict): - if episode in modified_watched_list_1[user_1][library_1][show_key_1][season]: - logger(f"Removing {episode} from {show_key_dict['title']}", 3) - modified_watched_list_1[user_1][library_1][show_key_1][season].remove(episode) + for episode in watched_list_1[user_1][library_1][show_key_1][ + season + ]: + if is_episode_in_dict( + episode, episode_watched_list_2_keys_dict + ): + if ( + episode + in modified_watched_list_1[user_1][library_1][ + show_key_1 + ][season] + ): + logger( + f"Removing {episode} from {show_key_dict['title']}", + 3, + ) + modified_watched_list_1[user_1][library_1][ + show_key_1 + ][season].remove(episode) # Remove empty seasons - if len(modified_watched_list_1[user_1][library_1][show_key_1][season]) == 0: - if season in modified_watched_list_1[user_1][library_1][show_key_1]: - logger(f"Removing {season} from {show_key_dict['title']} because it is empty", 3) - del modified_watched_list_1[user_1][library_1][show_key_1][season] + if ( + len( + modified_watched_list_1[user_1][library_1][show_key_1][ + season + ] + ) + == 0 + ): + if ( + season + in modified_watched_list_1[user_1][library_1][ + show_key_1 + ] + ): + logger( + f"Removing {season} from {show_key_dict['title']} because it is empty", + 3, + ) + del modified_watched_list_1[user_1][library_1][ + show_key_1 + ][season] # Remove empty shows if len(modified_watched_list_1[user_1][library_1][show_key_1]) == 0: if show_key_1 in modified_watched_list_1[user_1][library_1]: - logger(f"Removing {show_key_dict['title']} because it is empty", 3) + logger( + f"Removing {show_key_dict['title']} because it is empty", + 3, + ) del modified_watched_list_1[user_1][library_1][show_key_1] for user_1 in watched_list_1: @@ -280,7 +316,7 @@ def is_movie_in_dict(movie, movies_watched_list_2_keys_dict): # If the movie_value is in the movies_watched_list_2_keys_dict dictionary, return True if movie_value in movies_watched_list_2_keys_dict[movie_key]: return True - + # If the loop completes without finding a match, return False return False @@ -306,6 +342,7 @@ def is_episode_in_dict(episode, episode_watched_list_2_keys_dict): # If the loop completes without finding a match, return False return False + def future_thread_executor(args: list, workers: int = -1): futures_list = [] results = [] diff --git a/src/jellyfin.py b/src/jellyfin.py index 6abe55e..2ef065d 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -85,8 +85,9 @@ class Jellyfin: f"Jellyfin: Generating watched for {user_name} in library {library_title}", 0, ) - # Movies + async with aiohttp.ClientSession() as session: + # Movies if library_type == "Movie": user_watched[user_name][library_title] = [] watched = await self.query( @@ -95,16 +96,27 @@ class Jellyfin: "get", session, ) + for movie in watched["Items"]: - if movie["UserData"]["Played"] is True: - movie_guids = {} - movie_guids["title"] = movie["Name"] + # Check if the movie has been played + if ( + movie["UserData"]["Played"] is True + and "MediaSources" in movie + and movie["MediaSources"] is not {} + ): + # Create a dictionary for the movie with its title + movie_guids = {"title": movie["Name"]} + + # If the movie has provider IDs, add them to the dictionary if "ProviderIds" in movie: - # Lowercase movie["ProviderIds"] keys - movie_guids = { - k.lower(): v - for k, v in movie["ProviderIds"].items() - } + movie_guids.update( + { + k.lower(): v + for k, v in movie["ProviderIds"].items() + } + ) + + # If the movie has media sources, add them to the dictionary if "MediaSources" in movie: movie_guids["locations"] = tuple( [ @@ -112,22 +124,31 @@ class Jellyfin: for x in movie["MediaSources"] ] ) + + # Append the movie dictionary to the list for the given user and library user_watched[user_name][library_title].append(movie_guids) # TV Shows if library_type == "Series": + # Initialize an empty dictionary for the given user and library user_watched[user_name][library_title] = {} + + # Retrieve a list of watched TV shows watched_shows = await self.query( f"/Users/{user_id}/Items" + f"?ParentId={library_id}&isPlaceHolder=false&Fields=ProviderIds,Path,RecursiveItemCount", "get", session, ) + + # Filter the list of shows to only include those that have been partially or fully watched watched_shows_filtered = [] for show in watched_shows["Items"]: if "PlayedPercentage" in show["UserData"]: if show["UserData"]["PlayedPercentage"] > 0: watched_shows_filtered.append(show) + + # Create a list of tasks to retrieve the seasons of each watched show seasons_tasks = [] for show in watched_shows_filtered: show_guids = { @@ -136,21 +157,26 @@ class Jellyfin: show_guids["title"] = show["Name"] show_guids["locations"] = tuple([show["Path"].split("/")[-1]]) show_guids = frozenset(show_guids.items()) - identifiers = {"show_guids": show_guids, "show_id": show["Id"]} - task = asyncio.ensure_future( + show_identifiers = { + "show_guids": show_guids, + "show_id": show["Id"], + } + season_task = asyncio.ensure_future( self.query( f"/Shows/{show['Id']}/Seasons" + f"?userId={user_id}&isPlaceHolder=false&Fields=ProviderIds,RecursiveItemCount", "get", session, - frozenset(identifiers.items()), + frozenset(show_identifiers.items()), ) ) - seasons_tasks.append(task) + seasons_tasks.append(season_task) + # Retrieve the seasons for each watched show seasons_watched = await asyncio.gather(*seasons_tasks) - seasons_watched_filtered = [] + # Filter the list of seasons to only include those that have been partially or fully watched + seasons_watched_filtered = [] for seasons in seasons_watched: seasons_watched_filtered_dict = {} seasons_watched_filtered_dict["Identifiers"] = seasons[ @@ -169,6 +195,7 @@ class Jellyfin: seasons_watched_filtered_dict ) + # Create a list of tasks to retrieve the episodes of each watched season episodes_tasks = [] for seasons in seasons_watched_filtered: if len(seasons["Items"]) > 0: @@ -176,7 +203,7 @@ class Jellyfin: season_identifiers = dict(seasons["Identifiers"]) season_identifiers["season_id"] = season["Id"] season_identifiers["season_name"] = season["Name"] - task = asyncio.ensure_future( + episode_task = asyncio.ensure_future( self.query( f"/Shows/{season_identifiers['show_id']}/Episodes" + f"?seasonId={season['Id']}&userId={user_id}&isPlaceHolder=false&isPlayed=true&Fields=ProviderIds,MediaSources", @@ -185,62 +212,67 @@ class Jellyfin: frozenset(season_identifiers.items()), ) ) - episodes_tasks.append(task) + episodes_tasks.append(episode_task) + # Retrieve the episodes for each watched season watched_episodes = await asyncio.gather(*episodes_tasks) - for episodes in watched_episodes: - if len(episodes["Items"]) > 0: - for episode in episodes["Items"]: - if episode["UserData"]["Played"] is True: - if ( - "ProviderIds" in episode - or "MediaSources" in episode - ): - episode_identifiers = dict( - episodes["Identifiers"] - ) - show_guids = episode_identifiers["show_guids"] - if ( - show_guids - not in user_watched[user_name][ - library_title - ] - ): - user_watched[user_name][library_title][ - show_guids - ] = {} - if ( - episode_identifiers["season_name"] - not in user_watched[user_name][ - library_title - ][show_guids] - ): - user_watched[user_name][library_title][ - show_guids - ][episode_identifiers["season_name"]] = [] - episode_guids = {} - if "ProviderIds" in episode: - episode_guids = { - k.lower(): v - for k, v in episode[ - "ProviderIds" - ].items() - } - if "MediaSources" in episode: - episode_guids["locations"] = tuple( - [ - x["Path"].split("/")[-1] - for x in episode["MediaSources"] - ] - ) - user_watched[user_name][library_title][ - show_guids - ][episode_identifiers["season_name"]].append( - episode_guids - ) - - logger(f"Jellyfin: Got watched for {user_name} in library {library_title}") + # Iterate through the watched episodes + for episodes in watched_episodes: + # If the season has any watched episodes + if len(episodes["Items"]) > 0: + # Create a dictionary for the season with its identifier and episodes + season_dict = {} + season_dict["Identifiers"] = dict(episodes["Identifiers"]) + season_dict["Episodes"] = [] + for episode in episodes["Items"]: + if ( + episode["UserData"]["Played"] is True + and "MediaSources" in episode + and episode["MediaSources"] is not {} + ): + # Create a dictionary for the episode with its provider IDs and media sources + episode_dict = { + k.lower(): v + for k, v in episode["ProviderIds"].items() + } + episode_dict["title"] = episode["Name"] + episode_dict["locations"] = tuple( + [ + x["Path"].split("/")[-1] + for x in episode["MediaSources"] + ] + ) + # Add the episode dictionary to the season's list of episodes + season_dict["Episodes"].append(episode_dict) + # Add the season dictionary to the show's list of seasons + if ( + season_dict["Identifiers"]["show_guids"] + not in user_watched[user_name][library_title] + ): + user_watched[user_name][library_title][ + season_dict["Identifiers"]["show_guids"] + ] = {} + + if ( + season_dict["Identifiers"]["season_name"] + not in user_watched[user_name][library_title][ + season_dict["Identifiers"]["show_guids"] + ] + ): + user_watched[user_name][library_title][ + season_dict["Identifiers"]["show_guids"] + ][season_dict["Identifiers"]["season_name"]] = [] + + user_watched[user_name][library_title][ + season_dict["Identifiers"]["show_guids"] + ][season_dict["Identifiers"]["season_name"]] = season_dict[ + "Episodes" + ] + + logger( + f"Jellyfin: Got watched for {user_name} in library {library_title}", 1 + ) return user_watched except Exception as e: logger( diff --git a/src/main.py b/src/main.py index 1f690ca..8dc5f17 100644 --- a/src/main.py +++ b/src/main.py @@ -13,6 +13,7 @@ from src.jellyfin import Jellyfin load_dotenv(override=True) + def setup_black_white_lists( blacklist_library: str, whitelist_library: str, @@ -310,9 +311,28 @@ def generate_server_connections(): return servers -def get_server_watched(server_connection: list, users: dict, blacklist_library: list, whitelist_library: list, blacklist_library_type: list, whitelist_library_type: list, library_mapping: dict): + +def get_server_watched( + server_connection: list, + users: dict, + blacklist_library: list, + whitelist_library: list, + blacklist_library_type: list, + whitelist_library_type: list, + library_mapping: dict, +): if server_connection[0] == "plex": return server_connection[1].get_watched( + users, + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + library_mapping, + ) + elif server_connection[0] == "jellyfin": + return asyncio.run( + server_connection[1].get_watched( users, blacklist_library, whitelist_library, @@ -320,22 +340,27 @@ def get_server_watched(server_connection: list, users: dict, blacklist_library: whitelist_library_type, library_mapping, ) - elif server_connection[0] == "jellyfin": - return asyncio.run(server_connection[1].get_watched( - users, - blacklist_library, - whitelist_library, - blacklist_library_type, - whitelist_library_type, - library_mapping, - )) + ) -def update_server_watched(server_connection: list, server_watched_filtered: dict, user_mapping: dict, library_mapping: dict, dryrun: bool): +def update_server_watched( + server_connection: list, + server_watched_filtered: dict, + user_mapping: dict, + library_mapping: dict, + dryrun: bool, +): if server_connection[0] == "plex": - server_connection[1].update_watched(server_watched_filtered, user_mapping, library_mapping, dryrun) + server_connection[1].update_watched( + server_watched_filtered, user_mapping, library_mapping, dryrun + ) elif server_connection[0] == "jellyfin": - asyncio.run(server_connection[1].update_watched(server_watched_filtered, user_mapping, library_mapping, dryrun)) + asyncio.run( + server_connection[1].update_watched( + server_watched_filtered, user_mapping, library_mapping, dryrun + ) + ) + def main_loop(): logfile = os.getenv("LOGFILE", "log.log") @@ -448,11 +473,19 @@ def main_loop(): ) update_server_watched( - server_1, server_2_watched_filtered, user_mapping, library_mapping, dryrun + server_1, + server_2_watched_filtered, + user_mapping, + library_mapping, + dryrun, ) - + update_server_watched( - server_2, server_1_watched_filtered, user_mapping, library_mapping, dryrun + server_2, + server_1_watched_filtered, + user_mapping, + library_mapping, + dryrun, ) diff --git a/src/plex.py b/src/plex.py index f3681df..30b5d1c 100644 --- a/src/plex.py +++ b/src/plex.py @@ -15,11 +15,14 @@ from src.functions import ( # Bypass hostname validation for ssl. Taken from https://github.com/pkkid/python-plexapi/issues/143#issuecomment-775485186 class HostNameIgnoringAdapter(requests.adapters.HTTPAdapter): def init_poolmanager(self, connections, maxsize, block=..., **pool_kwargs): - self.poolmanager = PoolManager(num_pools=connections, - maxsize=maxsize, - block=block, - assert_hostname=False, - **pool_kwargs) + self.poolmanager = PoolManager( + num_pools=connections, + maxsize=maxsize, + block=block, + assert_hostname=False, + **pool_kwargs, + ) + def get_user_library_watched(user, user_plex, library): try: @@ -39,8 +42,9 @@ def get_user_library_watched(user, user_plex, library): for video in library_videos.search(unwatched=False): movie_guids = {} for guid in video.guids: - guid_source = re.search(r"(.*)://", guid.id).group(1).lower() - guid_id = re.search(r"://(.*)", guid.id).group(1) + # Extract source and id from guid.id + m = re.match(r"(.*)://(.*)", guid.id) + guid_source, guid_id = m.group(1).lower(), m.group(2) movie_guids[guid_source] = guid_id movie_guids["title"] = video.title @@ -68,7 +72,7 @@ def get_user_library_watched(user, user_plex, library): [x.split("/")[-1] for x in show.locations] ) show_guids = frozenset(show_guids.items()) - + # Get all watched episodes for show episode_guids = {} for episode in show.watched(): @@ -86,15 +90,15 @@ def get_user_library_watched(user, user_plex, library): if episode.parentTitle not in episode_guids: episode_guids[episode.parentTitle] = [] episode_guids[episode.parentTitle].append(episode_guids_temp) - + if episode_guids: # append show, season, episode if show_guids not in user_watched[user_name][library.title]: user_watched[user_name][library.title][show_guids] = {} - + user_watched[user_name][library.title][show_guids] = episode_guids - logger(f"Plex: Got watched for {user_name} in library {library.title}", 0) + logger(f"Plex: Got watched for {user_name} in library {library.title}", 1) return user_watched except Exception as e: logger( @@ -316,7 +320,9 @@ class Plex: user_plex = self.plex else: user_plex = self.login( - self.plex._baseurl, user.get_token(self.plex.machineIdentifier), self.ssl_bypass + self.plex._baseurl, + user.get_token(self.plex.machineIdentifier), + self.ssl_bypass, ) libraries = user_plex.library.sections() @@ -412,7 +418,7 @@ class Plex: continue else: logger( - f"Plex: Library {library} not found in library list", + f"Plex: Library {library} not found in library list", 1, ) continue From 39b33f3d43bafa18e653ef4c01f110236ac622f1 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 19 Dec 2022 13:22:42 -0700 Subject: [PATCH 06/11] Fix missing logging when using debug level --- src/functions.py | 2 +- src/main.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/functions.py b/src/functions.py index 13a0495..49312cf 100644 --- a/src/functions.py +++ b/src/functions.py @@ -14,7 +14,7 @@ def logger(message: str, log_type=0): output = str(message) if log_type == 0: pass - elif log_type == 1 and (debug and debug_level == "info"): + elif log_type == 1 and (debug and debug_level in ("info", "debug")): output = f"[INFO]: {output}" elif log_type == 2: output = f"[ERROR]: {output}" diff --git a/src/main.py b/src/main.py index 8dc5f17..69ae3ce 100644 --- a/src/main.py +++ b/src/main.py @@ -125,8 +125,8 @@ def setup_users( server_1_connection = server_1[1] server_2_type = server_2[0] server_2_connection = server_2[1] - print(f"Server 1: {server_1_type} {server_1_connection}") - print(f"Server 2: {server_2_type} {server_2_connection}") + logger(f"Server 1: {server_1_type} {server_1_connection}", 0) + logger(f"Server 2: {server_2_type} {server_2_connection}", 0) server_1_users = [] if server_1_type == "plex": @@ -419,7 +419,6 @@ def main_loop(): # Start server_2 at the next server in the list for server_2 in servers[servers.index(server_1) + 1 :]: - server_1_connection = server_1[1] server_2_connection = server_2[1] From 59c6d278e321f7dbed1da8ce8e61b3fa9673e002 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 19 Dec 2022 13:57:20 -0700 Subject: [PATCH 07/11] Add more logging to debug --- src/jellyfin.py | 9 +++++++++ src/plex.py | 14 ++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 2ef065d..1b139d5 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -104,6 +104,7 @@ class Jellyfin: and "MediaSources" in movie and movie["MediaSources"] is not {} ): + logger(f"Jellyfin: Adding {movie['Name']} to {user_name} watched list", 3) # Create a dictionary for the movie with its title movie_guids = {"title": movie["Name"]} @@ -127,6 +128,7 @@ class Jellyfin: # Append the movie dictionary to the list for the given user and library user_watched[user_name][library_title].append(movie_guids) + logger(f"Jellyfin: Added {movie_guids} to {user_name} watched list", 3) # TV Shows if library_type == "Series": @@ -151,6 +153,7 @@ class Jellyfin: # Create a list of tasks to retrieve the seasons of each watched show seasons_tasks = [] for show in watched_shows_filtered: + logger(f"Jellyfin: Adding {show['Name']} to {user_name} watched list", 3) show_guids = { k.lower(): v for k, v in show["ProviderIds"].items() } @@ -269,10 +272,16 @@ class Jellyfin: ][season_dict["Identifiers"]["season_name"]] = season_dict[ "Episodes" ] + logger(f"Jellyfin: Added {season_dict['Episodes']} to {user_name} {season_dict['Identifiers']['show_guids']} watched list", 1) logger( f"Jellyfin: Got watched for {user_name} in library {library_title}", 1 ) + if library_title in user_watched[user_name]: + logger( + f"Jellyfin: {user_watched[user_name][library_title]}", 3 + ) + return user_watched except Exception as e: logger( diff --git a/src/plex.py b/src/plex.py index 30b5d1c..919d8a3 100644 --- a/src/plex.py +++ b/src/plex.py @@ -35,11 +35,13 @@ def get_user_library_watched(user, user_plex, library): 0, ) + library_videos = user_plex.library.section(library.title) + if library.type == "movie": user_watched[user_name][library.title] = [] - library_videos = user_plex.library.section(library.title) for video in library_videos.search(unwatched=False): + logger(f"Plex: Adding {video.title} to {user_name} watched list", 3) movie_guids = {} for guid in video.guids: # Extract source and id from guid.id @@ -53,13 +55,13 @@ def get_user_library_watched(user, user_plex, library): ) user_watched[user_name][library.title].append(movie_guids) + logger(f"Plex: Added {movie_guids} to {user_name} watched list", 3) elif library.type == "show": user_watched[user_name][library.title] = {} - library_videos = user_plex.library.section(library.title) - shows = library_videos.search(unwatched=False) - for show in shows: + for show in library_videos.search(unwatched=False): + logger(f"Plex: Adding {show.title} to {user_name} watched list", 3) show_guids = {} for show_guid in show.guids: # Extract source and id from guid.id @@ -97,8 +99,12 @@ def get_user_library_watched(user, user_plex, library): user_watched[user_name][library.title][show_guids] = {} user_watched[user_name][library.title][show_guids] = episode_guids + logger(f"Plex: Added {episode_guids} to {user_name} {show_guids} watched list", 3) logger(f"Plex: Got watched for {user_name} in library {library.title}", 1) + if library.title in user_watched[user_name]: + logger(f"Plex: {user_watched[user_name][library.title]}", 3) + return user_watched except Exception as e: logger( From 4066228e570408d5d395f2646cd84b6fba13fc93 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 19 Dec 2022 14:07:56 -0700 Subject: [PATCH 08/11] Add more debug logging. Do not enable debug by default --- src/functions.py | 2 +- src/jellyfin.py | 5 +++++ src/plex.py | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/functions.py b/src/functions.py index 49312cf..9afec6e 100644 --- a/src/functions.py +++ b/src/functions.py @@ -8,7 +8,7 @@ logfile = os.getenv("LOGFILE", "log.log") def logger(message: str, log_type=0): - debug = str_to_bool(os.getenv("DEBUG", "True")) + debug = str_to_bool(os.getenv("DEBUG", "False")) debug_level = os.getenv("DEBUG_LEVEL", "info").lower() output = str(message) diff --git a/src/jellyfin.py b/src/jellyfin.py index 1b139d5..3d98b86 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -105,6 +105,11 @@ class Jellyfin: and movie["MediaSources"] is not {} ): logger(f"Jellyfin: Adding {movie['Name']} to {user_name} watched list", 3) + if "ProviderIds" in movie: + logger(f"Jellyfin: {movie['Name']} {movie['ProviderIds']} {movie['MediaSources']}", 3) + else: + logger(f"Jellyfin: {movie['Name']} {movie['MediaSources']['Path']}", 3) + # Create a dictionary for the movie with its title movie_guids = {"title": movie["Name"]} diff --git a/src/plex.py b/src/plex.py index 919d8a3..1948bcc 100644 --- a/src/plex.py +++ b/src/plex.py @@ -1,4 +1,4 @@ -import re, requests +import re, requests, json from urllib3.poolmanager import PoolManager from plexapi.server import PlexServer @@ -42,6 +42,8 @@ def get_user_library_watched(user, user_plex, library): for video in library_videos.search(unwatched=False): logger(f"Plex: Adding {video.title} to {user_name} watched list", 3) + logger(f"Plex: {video.title} {video.guids} {video.locations}", 3) + movie_guids = {} for guid in video.guids: # Extract source and id from guid.id From 1a4e3f4ec466b39235818f4f9a682ce3434e76e3 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Fri, 23 Dec 2022 23:02:53 -0700 Subject: [PATCH 09/11] Move setup_black_white_list to functions. Fix trailing slash on jellyfin baseurl --- src/functions.py | 102 ++++++++++++++++++++++++++++++++++++++++++++ src/main.py | 108 +++-------------------------------------------- 2 files changed, 108 insertions(+), 102 deletions(-) diff --git a/src/functions.py b/src/functions.py index 9afec6e..83acc6c 100644 --- a/src/functions.py +++ b/src/functions.py @@ -55,6 +55,108 @@ def search_mapping(dictionary: dict, key_value: str): return None +def setup_black_white_lists( + blacklist_library: str, + whitelist_library: str, + blacklist_library_type: str, + whitelist_library_type: str, + blacklist_users: str, + whitelist_users: str, + library_mapping=None, + user_mapping=None, +): + if blacklist_library: + if len(blacklist_library) > 0: + blacklist_library = blacklist_library.split(",") + blacklist_library = [x.strip() for x in blacklist_library] + if library_mapping: + temp_library = [] + for library in blacklist_library: + library_other = search_mapping(library_mapping, library) + if library_other: + temp_library.append(library_other) + + blacklist_library = blacklist_library + temp_library + else: + blacklist_library = [] + logger(f"Blacklist Library: {blacklist_library}", 1) + + if whitelist_library: + if len(whitelist_library) > 0: + whitelist_library = whitelist_library.split(",") + whitelist_library = [x.strip() for x in whitelist_library] + if library_mapping: + temp_library = [] + for library in whitelist_library: + library_other = search_mapping(library_mapping, library) + if library_other: + temp_library.append(library_other) + + whitelist_library = whitelist_library + temp_library + else: + whitelist_library = [] + logger(f"Whitelist Library: {whitelist_library}", 1) + + if blacklist_library_type: + if len(blacklist_library_type) > 0: + blacklist_library_type = blacklist_library_type.split(",") + blacklist_library_type = [x.lower().strip() for x in blacklist_library_type] + else: + blacklist_library_type = [] + logger(f"Blacklist Library Type: {blacklist_library_type}", 1) + + if whitelist_library_type: + if len(whitelist_library_type) > 0: + whitelist_library_type = whitelist_library_type.split(",") + whitelist_library_type = [x.lower().strip() for x in whitelist_library_type] + else: + whitelist_library_type = [] + logger(f"Whitelist Library Type: {whitelist_library_type}", 1) + + if blacklist_users: + if len(blacklist_users) > 0: + blacklist_users = blacklist_users.split(",") + blacklist_users = [x.lower().strip() for x in blacklist_users] + if user_mapping: + temp_users = [] + for user in blacklist_users: + user_other = search_mapping(user_mapping, user) + if user_other: + temp_users.append(user_other) + + blacklist_users = blacklist_users + temp_users + else: + blacklist_users = [] + logger(f"Blacklist Users: {blacklist_users}", 1) + + if whitelist_users: + if len(whitelist_users) > 0: + whitelist_users = whitelist_users.split(",") + whitelist_users = [x.lower().strip() for x in whitelist_users] + if user_mapping: + temp_users = [] + for user in whitelist_users: + user_other = search_mapping(user_mapping, user) + if user_other: + temp_users.append(user_other) + + whitelist_users = whitelist_users + temp_users + else: + whitelist_users = [] + else: + whitelist_users = [] + logger(f"Whitelist Users: {whitelist_users}", 1) + + return ( + blacklist_library, + whitelist_library, + blacklist_library_type, + whitelist_library_type, + blacklist_users, + whitelist_users, + ) + + def check_skip_logic( library_title, library_type, diff --git a/src/main.py b/src/main.py index 69ae3ce..39027f0 100644 --- a/src/main.py +++ b/src/main.py @@ -7,6 +7,7 @@ from src.functions import ( str_to_bool, search_mapping, cleanup_watched, + setup_black_white_lists, ) from src.plex import Plex from src.jellyfin import Jellyfin @@ -14,107 +15,6 @@ from src.jellyfin import Jellyfin load_dotenv(override=True) -def setup_black_white_lists( - blacklist_library: str, - whitelist_library: str, - blacklist_library_type: str, - whitelist_library_type: str, - blacklist_users: str, - whitelist_users: str, - library_mapping=None, - user_mapping=None, -): - if blacklist_library: - if len(blacklist_library) > 0: - blacklist_library = blacklist_library.split(",") - blacklist_library = [x.strip() for x in blacklist_library] - if library_mapping: - temp_library = [] - for library in blacklist_library: - library_other = search_mapping(library_mapping, library) - if library_other: - temp_library.append(library_other) - - blacklist_library = blacklist_library + temp_library - else: - blacklist_library = [] - logger(f"Blacklist Library: {blacklist_library}", 1) - - if whitelist_library: - if len(whitelist_library) > 0: - whitelist_library = whitelist_library.split(",") - whitelist_library = [x.strip() for x in whitelist_library] - if library_mapping: - temp_library = [] - for library in whitelist_library: - library_other = search_mapping(library_mapping, library) - if library_other: - temp_library.append(library_other) - - whitelist_library = whitelist_library + temp_library - else: - whitelist_library = [] - logger(f"Whitelist Library: {whitelist_library}", 1) - - if blacklist_library_type: - if len(blacklist_library_type) > 0: - blacklist_library_type = blacklist_library_type.split(",") - blacklist_library_type = [x.lower().strip() for x in blacklist_library_type] - else: - blacklist_library_type = [] - logger(f"Blacklist Library Type: {blacklist_library_type}", 1) - - if whitelist_library_type: - if len(whitelist_library_type) > 0: - whitelist_library_type = whitelist_library_type.split(",") - whitelist_library_type = [x.lower().strip() for x in whitelist_library_type] - else: - whitelist_library_type = [] - logger(f"Whitelist Library Type: {whitelist_library_type}", 1) - - if blacklist_users: - if len(blacklist_users) > 0: - blacklist_users = blacklist_users.split(",") - blacklist_users = [x.lower().strip() for x in blacklist_users] - if user_mapping: - temp_users = [] - for user in blacklist_users: - user_other = search_mapping(user_mapping, user) - if user_other: - temp_users.append(user_other) - - blacklist_users = blacklist_users + temp_users - else: - blacklist_users = [] - logger(f"Blacklist Users: {blacklist_users}", 1) - - if whitelist_users: - if len(whitelist_users) > 0: - whitelist_users = whitelist_users.split(",") - whitelist_users = [x.lower().strip() for x in whitelist_users] - if user_mapping: - temp_users = [] - for user in whitelist_users: - user_other = search_mapping(user_mapping, user) - if user_other: - temp_users.append(user_other) - - whitelist_users = whitelist_users + temp_users - else: - whitelist_users = [] - else: - whitelist_users = [] - logger(f"Whitelist Users: {whitelist_users}", 1) - - return ( - blacklist_library, - whitelist_library, - blacklist_library_type, - whitelist_library_type, - blacklist_users, - whitelist_users, - ) - def setup_users( server_1, server_2, blacklist_users, whitelist_users, user_mapping=None @@ -302,13 +202,17 @@ def generate_server_connections(): ) for i, baseurl in enumerate(jellyfin_baseurl): + baseurl = baseurl.strip() + if baseurl[-1] == '/': + baseurl = baseurl[:-1] servers.append( ( "jellyfin", - Jellyfin(baseurl=baseurl.strip(), token=jellyfin_token[i].strip()), + Jellyfin(baseurl=baseurl, token=jellyfin_token[i].strip()), ) ) + return servers From 111e284cc86725ca248094641f9229b7104575a4 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Fri, 23 Dec 2022 23:10:51 -0700 Subject: [PATCH 10/11] Cleanup --- src/jellyfin.py | 2 +- src/main.py | 3 --- src/plex.py | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 3d98b86..84fb130 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -286,7 +286,7 @@ class Jellyfin: logger( f"Jellyfin: {user_watched[user_name][library_title]}", 3 ) - + return user_watched except Exception as e: logger( diff --git a/src/main.py b/src/main.py index 39027f0..9302f3e 100644 --- a/src/main.py +++ b/src/main.py @@ -323,9 +323,6 @@ def main_loop(): # Start server_2 at the next server in the list for server_2 in servers[servers.index(server_1) + 1 :]: - server_1_connection = server_1[1] - server_2_connection = server_2[1] - # Create users list logger("Creating users list", 1) server_1_users, server_2_users = setup_users( diff --git a/src/plex.py b/src/plex.py index 1948bcc..87724da 100644 --- a/src/plex.py +++ b/src/plex.py @@ -1,4 +1,4 @@ -import re, requests, json +import re, requests from urllib3.poolmanager import PoolManager from plexapi.server import PlexServer @@ -106,7 +106,7 @@ def get_user_library_watched(user, user_plex, library): logger(f"Plex: Got watched for {user_name} in library {library.title}", 1) if library.title in user_watched[user_name]: logger(f"Plex: {user_watched[user_name][library.title]}", 3) - + return user_watched except Exception as e: logger( From 1eb92cf7c17ee126a2b2630f03352acd76ef49ab Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Fri, 23 Dec 2022 23:11:38 -0700 Subject: [PATCH 11/11] black formatting --- src/jellyfin.py | 34 +++++++++++++++++++++++++--------- src/main.py | 4 +--- src/plex.py | 5 ++++- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 84fb130..9658427 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -104,11 +104,20 @@ class Jellyfin: and "MediaSources" in movie and movie["MediaSources"] is not {} ): - logger(f"Jellyfin: Adding {movie['Name']} to {user_name} watched list", 3) + logger( + f"Jellyfin: Adding {movie['Name']} to {user_name} watched list", + 3, + ) if "ProviderIds" in movie: - logger(f"Jellyfin: {movie['Name']} {movie['ProviderIds']} {movie['MediaSources']}", 3) + logger( + f"Jellyfin: {movie['Name']} {movie['ProviderIds']} {movie['MediaSources']}", + 3, + ) else: - logger(f"Jellyfin: {movie['Name']} {movie['MediaSources']['Path']}", 3) + logger( + f"Jellyfin: {movie['Name']} {movie['MediaSources']['Path']}", + 3, + ) # Create a dictionary for the movie with its title movie_guids = {"title": movie["Name"]} @@ -133,7 +142,10 @@ class Jellyfin: # Append the movie dictionary to the list for the given user and library user_watched[user_name][library_title].append(movie_guids) - logger(f"Jellyfin: Added {movie_guids} to {user_name} watched list", 3) + logger( + f"Jellyfin: Added {movie_guids} to {user_name} watched list", + 3, + ) # TV Shows if library_type == "Series": @@ -158,7 +170,10 @@ class Jellyfin: # Create a list of tasks to retrieve the seasons of each watched show seasons_tasks = [] for show in watched_shows_filtered: - logger(f"Jellyfin: Adding {show['Name']} to {user_name} watched list", 3) + logger( + f"Jellyfin: Adding {show['Name']} to {user_name} watched list", + 3, + ) show_guids = { k.lower(): v for k, v in show["ProviderIds"].items() } @@ -277,15 +292,16 @@ class Jellyfin: ][season_dict["Identifiers"]["season_name"]] = season_dict[ "Episodes" ] - logger(f"Jellyfin: Added {season_dict['Episodes']} to {user_name} {season_dict['Identifiers']['show_guids']} watched list", 1) + logger( + f"Jellyfin: Added {season_dict['Episodes']} to {user_name} {season_dict['Identifiers']['show_guids']} watched list", + 1, + ) logger( f"Jellyfin: Got watched for {user_name} in library {library_title}", 1 ) if library_title in user_watched[user_name]: - logger( - f"Jellyfin: {user_watched[user_name][library_title]}", 3 - ) + logger(f"Jellyfin: {user_watched[user_name][library_title]}", 3) return user_watched except Exception as e: diff --git a/src/main.py b/src/main.py index 9302f3e..153ec71 100644 --- a/src/main.py +++ b/src/main.py @@ -15,7 +15,6 @@ from src.jellyfin import Jellyfin load_dotenv(override=True) - def setup_users( server_1, server_2, blacklist_users, whitelist_users, user_mapping=None ): @@ -203,7 +202,7 @@ def generate_server_connections(): for i, baseurl in enumerate(jellyfin_baseurl): baseurl = baseurl.strip() - if baseurl[-1] == '/': + if baseurl[-1] == "/": baseurl = baseurl[:-1] servers.append( ( @@ -212,7 +211,6 @@ def generate_server_connections(): ) ) - return servers diff --git a/src/plex.py b/src/plex.py index 87724da..a91b81d 100644 --- a/src/plex.py +++ b/src/plex.py @@ -101,7 +101,10 @@ def get_user_library_watched(user, user_plex, library): user_watched[user_name][library.title][show_guids] = {} user_watched[user_name][library.title][show_guids] = episode_guids - logger(f"Plex: Added {episode_guids} to {user_name} {show_guids} watched list", 3) + logger( + f"Plex: Added {episode_guids} to {user_name} {show_guids} watched list", + 3, + ) logger(f"Plex: Got watched for {user_name} in library {library.title}", 1) if library.title in user_watched[user_name]: