diff --git a/.gitignore b/.gitignore index 5ac8951..06e876a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +*.prof # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/main.py b/main.py index aa32ca2..ea9f4c2 100644 --- a/main.py +++ b/main.py @@ -2,7 +2,8 @@ import copy, os, traceback, json from dotenv import load_dotenv from time import sleep -from src.functions import logger, str_to_bool, search_mapping, generate_library_guids_dict + +from src.functions import logger, str_to_bool, search_mapping, generate_library_guids_dict, future_thread_executor from src.plex import Plex from src.jellyfin import Jellyfin @@ -346,8 +347,12 @@ def main(): # Create users list server_1_users, server_2_users = setup_users(server_1, server_2, blacklist_users, whitelist_users, user_mapping) - server_1_watched = server_1_connection.get_watched(server_1_users, blacklist_library, whitelist_library, blacklist_library_type, whitelist_library_type, library_mapping) - server_2_watched = server_2_connection.get_watched(server_2_users, blacklist_library, whitelist_library, blacklist_library_type, whitelist_library_type, library_mapping) + args = [[server_1_connection.get_watched, server_1_users, blacklist_library, whitelist_library, blacklist_library_type, whitelist_library_type, library_mapping] + , [server_2_connection.get_watched, server_2_users, blacklist_library, whitelist_library, blacklist_library_type, whitelist_library_type, library_mapping]] + + results = future_thread_executor(args) + server_1_watched = results[0] + server_2_watched = results[1] # clone watched so it isnt modified in the cleanup function so all duplicates are actually removed server_1_watched_filtered = copy.deepcopy(server_1_watched) @@ -362,10 +367,10 @@ def main(): logger(f"server 1 watched that needs to be synced to server 2:\n{server_1_watched_filtered}", 1) logger(f"server 2 watched that needs to be synced to server 1:\n{server_2_watched_filtered}", 1) - # Update watched status - server_1_connection.update_watched(server_2_watched_filtered, user_mapping, library_mapping, dryrun) - server_2_connection.update_watched(server_1_watched_filtered, user_mapping, library_mapping, dryrun) + args= [[server_1_connection.update_watched, server_2_watched_filtered, user_mapping, library_mapping, dryrun] + , [server_2_connection.update_watched, server_1_watched_filtered, user_mapping, library_mapping, dryrun]] + future_thread_executor(args) if __name__ == "__main__": sleep_timer = float(os.getenv("SLEEP_TIMER", "3600")) diff --git a/src/functions.py b/src/functions.py index 0ef75f4..8e3d81c 100644 --- a/src/functions.py +++ b/src/functions.py @@ -1,10 +1,12 @@ import os +from concurrent.futures import ThreadPoolExecutor from dotenv import load_dotenv + load_dotenv(override=True) logfile = os.getenv("LOGFILE","log.log") -def logger(message, log_type=0): +def logger(message: str, log_type=0): debug = str_to_bool(os.getenv("DEBUG", "True")) debug_level = os.getenv("DEBUG_LEVEL", "info").lower() @@ -114,3 +116,20 @@ def generate_library_guids_dict(user_list: dict, generate_output: int): return show_output_dict, episode_output_dict, movies_output_dict +def future_thread_executor(args: list): + futures_list = [] + results = [] + + with ThreadPoolExecutor() as executor: + for arg in args: + # * arg unpacks the list into actual arguments + futures_list.append(executor.submit(*arg)) + + for future in futures_list: + try: + result = future.result() + results.append(result) + except Exception as e: + raise Exception(e) + + return results diff --git a/src/jellyfin.py b/src/jellyfin.py index 1052714..9df880d 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -1,10 +1,11 @@ import requests -from src.functions import logger, search_mapping, str_to_bool, check_skip_logic, generate_library_guids_dict +from src.functions import logger, search_mapping, str_to_bool, check_skip_logic, generate_library_guids_dict, future_thread_executor class Jellyfin(): def __init__(self, baseurl, token): self.baseurl = baseurl self.token = token + self.session = requests.Session() if not self.baseurl: raise Exception("Jellyfin baseurl not set") @@ -19,8 +20,12 @@ class Jellyfin(): try: response = None + headers = { + "Accept": "application/json", + "X-Emby-Token": self.token + } if query_type == "get": - response = requests.get(self.baseurl + query, headers={"accept":"application/json", "X-Emby-Token": self.token}) + response = self.session.get(self.baseurl + query, headers=headers) elif query_type == "post": authorization = ( @@ -30,7 +35,8 @@ class Jellyfin(): 'DeviceId="script", ' 'Version="0.0.0"' ) - response = requests.post(self.baseurl + query, headers={"accept":"application/json", "X-Emby-Authorization": authorization, "X-Emby-Token": self.token}) + headers["X-Emby-Authorization"] = authorization + response = self.session.post(self.baseurl + query, headers=headers) return response.json() except Exception as e: @@ -50,9 +56,61 @@ class Jellyfin(): return users + def get_user_watched(self, user_name, user_id, library_type, library_id, library_title): + user_watched = {} + + logger(f"Jellyfin: Generating watched for {user_name} in library {library_title}", 0) + # Movies + if library_type == "Movie": + watched = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=true&ParentId={library_id}&Filters=IsPlayed&Fields=ItemCounts,ProviderIds", "get") + for movie in watched["Items"]: + if movie["UserData"]["Played"] == True: + if movie["ProviderIds"]: + if user_name not in user_watched: + user_watched[user_name] = {} + if library_title not in user_watched[user_name]: + user_watched[user_name][library_title] = [] + # Lowercase movie["ProviderIds"] keys + movie["ProviderIds"] = {k.lower(): v for k, v in movie["ProviderIds"].items()} + user_watched[user_name][library_title].append(movie["ProviderIds"]) + + # TV Shows + if library_type == "Episode": + watched = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=true&ParentId={library_id}&Fields=ItemCounts,ProviderIds", "get") + watched_shows = [x for x in watched["Items"] if x["Type"] == "Series"] + + for show in watched_shows: + show_guids = {k.lower(): v for k, v in show["ProviderIds"].items()} + show_guids["title"] = show["Name"] + show_guids = frozenset(show_guids.items()) + seasons = self.query(f"/Shows/{show['Id']}/Seasons?userId={user_id}&Fields=ItemCounts,ProviderIds", "get") + if len(seasons["Items"]) > 0: + for season in seasons["Items"]: + episodes = self.query(f"/Shows/{show['Id']}/Episodes?seasonId={season['Id']}&userId={user_id}&Fields=ItemCounts,ProviderIds", "get") + if len(episodes["Items"]) > 0: + for episode in episodes["Items"]: + if episode["UserData"]["Played"] == True: + if episode["ProviderIds"]: + if user_name not in user_watched: + user_watched[user_name] = {} + if library_title not in user_watched[user_name]: + user_watched[user_name][library_title] = {} + if show_guids not in user_watched[user_name][library_title]: + user_watched[user_name][library_title][show_guids] = {} + if season["Name"] not in user_watched[user_name][library_title][show_guids]: + user_watched[user_name][library_title][show_guids][season["Name"]] = [] + + # Lowercase episode["ProviderIds"] keys + episode["ProviderIds"] = {k.lower(): v for k, v in episode["ProviderIds"].items()} + user_watched[user_name][library_title][show_guids][season["Name"]].append(episode["ProviderIds"]) + + return user_watched + + def get_watched(self, users, blacklist_library, whitelist_library, blacklist_library_type, whitelist_library_type, library_mapping=None): users_watched = {} - + args = [] + for user_name, user_id in users.items(): # Get all libraries user_name = user_name.lower() @@ -76,54 +134,75 @@ class Jellyfin(): logger(f"Jellyfin: Skipping library {library_title} {skip_reason}", 1) continue - logger(f"Jellyfin: Generating watched for {user_name} in library {library_title}", 0) - # Movies - if library_type == "Movie": - watched = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=true&ParentId={library_id}&Filters=IsPlayed&Fields=ItemCounts,ProviderIds", "get") - for movie in watched["Items"]: - if movie["UserData"]["Played"] == True: - if movie["ProviderIds"]: - if user_name not in users_watched: - users_watched[user_name] = {} - if library_title not in users_watched[user_name]: - users_watched[user_name][library_title] = [] - # Lowercase movie["ProviderIds"] keys - movie["ProviderIds"] = {k.lower(): v for k, v in movie["ProviderIds"].items()} - users_watched[user_name][library_title].append(movie["ProviderIds"]) + args.append([self.get_user_watched, user_name, user_id, library_type, library_id, library_title]) - # TV Shows - if library_type == "Episode": - watched = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=true&ParentId={library_id}&Fields=ItemCounts,ProviderIds", "get") - watched_shows = [x for x in watched["Items"] if x["Type"] == "Series"] - - for show in watched_shows: - show_guids = {k.lower(): v for k, v in show["ProviderIds"].items()} - show_guids["title"] = show["Name"] - show_guids = frozenset(show_guids.items()) - seasons = self.query(f"/Shows/{show['Id']}/Seasons?userId={user_id}&Fields=ItemCounts,ProviderIds", "get") - if len(seasons["Items"]) > 0: - for season in seasons["Items"]: - episodes = self.query(f"/Shows/{show['Id']}/Episodes?seasonId={season['Id']}&userId={user_id}&Fields=ItemCounts,ProviderIds", "get") - if len(episodes["Items"]) > 0: - for episode in episodes["Items"]: - if episode["UserData"]["Played"] == True: - if episode["ProviderIds"]: - if user_name not in users_watched: - users_watched[user_name] = {} - if library_title not in users_watched[user_name]: - users_watched[user_name][library_title] = {} - if show_guids not in users_watched[user_name][library_title]: - users_watched[user_name][library_title][show_guids] = {} - if season["Name"] not in users_watched[user_name][library_title][show_guids]: - users_watched[user_name][library_title][show_guids][season["Name"]] = [] - - # Lowercase episode["ProviderIds"] keys - episode["ProviderIds"] = {k.lower(): v for k, v in episode["ProviderIds"].items()} - users_watched[user_name][library_title][show_guids][season["Name"]].append(episode["ProviderIds"]) + for user_watched in future_thread_executor(args): + users_watched.update(user_watched) return users_watched + + def update_user_watched(self, user, user_id, library, library_id, videos, dryrun): + logger(f"Jellyfin: Updating watched for {user} in library {library}", 1) + library_search = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=true&ParentId={library_id}&limit=1", "get") + library_type = library_search["Items"][0]["Type"] + + # Movies + if library_type == "Movie": + _, _, videos_movies_ids = generate_library_guids_dict(videos, 2) + + jellyfin_search = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=false&ParentId={library_id}&isPlayed=false&Fields=ItemCounts,ProviderIds", "get") + for jellyfin_video in jellyfin_search["Items"]: + if str_to_bool(jellyfin_video["UserData"]["Played"]) == False: + jellyfin_video_id = jellyfin_video["Id"] + + for movie_provider_source, movie_provider_id in jellyfin_video["ProviderIds"].items(): + if movie_provider_source.lower() in videos_movies_ids: + if movie_provider_id.lower() in videos_movies_ids[movie_provider_source.lower()]: + msg = f"{jellyfin_video['Name']} as watched for {user} in {library} for Jellyfin" + if not dryrun: + logger(f"Marking {msg}", 0) + self.query(f"/Users/{user_id}/PlayedItems/{jellyfin_video_id}", "post") + else: + logger(f"Dryrun {msg}", 0) + break + + # TV Shows + if library_type == "Episode": + videos_shows_ids, videos_episode_ids, _ = generate_library_guids_dict(videos, 3) + + jellyfin_search = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=false&ParentId={library_id}&isPlayed=false&Fields=ItemCounts,ProviderIds", "get") + jellyfin_shows = [x for x in jellyfin_search["Items"]] + + for jellyfin_show in jellyfin_shows: + show_found = False + for show_provider_source, show_provider_id in jellyfin_show["ProviderIds"].items(): + if show_provider_source.lower() in videos_shows_ids: + if show_provider_id.lower() in videos_shows_ids[show_provider_source.lower()]: + show_found = True + jellyfin_show_id = jellyfin_show["Id"] + jellyfin_episodes = self.query(f"/Shows/{jellyfin_show_id}/Episodes?userId={user_id}&Fields=ItemCounts,ProviderIds", "get") + for jellyfin_episode in jellyfin_episodes["Items"]: + if str_to_bool(jellyfin_episode["UserData"]["Played"]) == False: + jellyfin_episode_id = jellyfin_episode["Id"] + + for episode_provider_source, episode_provider_id in jellyfin_episode["ProviderIds"].items(): + if episode_provider_source.lower() in videos_episode_ids: + if episode_provider_id.lower() in videos_episode_ids[episode_provider_source.lower()]: + msg = f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode['IndexNumber']} {jellyfin_episode['Name']} as watched for {user} in {library} for Jellyfin" + if not dryrun: + logger(f"Marked {msg}", 0) + self.query(f"/Users/{user_id}/PlayedItems/{jellyfin_episode_id}", "post") + else: + logger(f"Dryrun {msg}", 0) + break + + if show_found: + break + + def update_watched(self, watched_list, user_mapping=None, library_mapping=None, dryrun=False): + args = [] for user, libraries in watched_list.items(): user_other = None if user_mapping: @@ -143,7 +222,7 @@ class Jellyfin(): if not user_id: logger(f"{user} {user_other} not found in Jellyfin", 2) - break + continue jellyfin_libraries = self.query(f"/Users/{user_id}/Views", "get")["Items"] @@ -172,59 +251,6 @@ class Jellyfin(): continue if library_id: - logger(f"Jellyfin: Updating watched for {user} in library {library}", 1) - library_search = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=true&ParentId={library_id}&limit=1", "get") - library_type = library_search["Items"][0]["Type"] + args.append([self.update_user_watched, user, user_id, library, library_id, videos, dryrun]) - # Movies - if library_type == "Movie": - _, _, videos_movies_ids = generate_library_guids_dict(videos, 2) - - jellyfin_search = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=false&ParentId={library_id}&isPlayed=false&Fields=ItemCounts,ProviderIds", "get") - for jellyfin_video in jellyfin_search["Items"]: - if str_to_bool(jellyfin_video["UserData"]["Played"]) == False: - jellyfin_video_id = jellyfin_video["Id"] - - for movie_provider_source, movie_provider_id in jellyfin_video["ProviderIds"].items(): - if movie_provider_source.lower() in videos_movies_ids: - if movie_provider_id.lower() in videos_movies_ids[movie_provider_source.lower()]: - msg = f"{jellyfin_video['Name']} as watched for {user} in {library} for Jellyfin" - if not dryrun: - logger(f"Marking {msg}", 0) - self.query(f"/Users/{user_id}/PlayedItems/{jellyfin_video_id}", "post") - else: - logger(f"Dryrun {msg}", 0) - break - - # TV Shows - if library_type == "Episode": - videos_shows_ids, videos_episode_ids, _ = generate_library_guids_dict(videos, 3) - - jellyfin_search = self.query(f"/Users/{user_id}/Items?SortBy=SortName&SortOrder=Ascending&Recursive=false&ParentId={library_id}&isPlayed=false&Fields=ItemCounts,ProviderIds", "get") - jellyfin_shows = [x for x in jellyfin_search["Items"]] - - for jellyfin_show in jellyfin_shows: - show_found = False - for show_provider_source, show_provider_id in jellyfin_show["ProviderIds"].items(): - if show_provider_source.lower() in videos_shows_ids: - if show_provider_id.lower() in videos_shows_ids[show_provider_source.lower()]: - show_found = True - jellyfin_show_id = jellyfin_show["Id"] - jellyfin_episodes = self.query(f"/Shows/{jellyfin_show_id}/Episodes?userId={user_id}&Fields=ItemCounts,ProviderIds", "get") - for jellyfin_episode in jellyfin_episodes["Items"]: - if str_to_bool(jellyfin_episode["UserData"]["Played"]) == False: - jellyfin_episode_id = jellyfin_episode["Id"] - - for episode_provider_source, episode_provider_id in jellyfin_episode["ProviderIds"].items(): - if episode_provider_source.lower() in videos_episode_ids: - if episode_provider_id.lower() in videos_episode_ids[episode_provider_source.lower()]: - msg = f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode['IndexNumber']} {jellyfin_episode['Name']} as watched for {user} in {library} for Jellyfin" - if not dryrun: - logger(f"Marked {msg}", 0) - self.query(f"/Users/{user_id}/PlayedItems/{jellyfin_episode_id}", "post") - else: - logger(f"Dryrun {msg}", 0) - break - - if show_found: - break + future_thread_executor(args) diff --git a/src/plex.py b/src/plex.py index 0c43e49..e398c19 100644 --- a/src/plex.py +++ b/src/plex.py @@ -1,9 +1,11 @@ import re -from src.functions import logger, search_mapping, check_skip_logic, generate_library_guids_dict from plexapi.server import PlexServer from plexapi.myplex import MyPlexAccount +from src.functions import logger, search_mapping, check_skip_logic, generate_library_guids_dict, future_thread_executor + + # class plex accept base url and token and username and password but default with none class Plex: def __init__(self, baseurl=None, token=None, username=None, password=None, servername=None): @@ -117,20 +119,81 @@ class Plex: logger(f"Plex: Skipping library {library_title} {skip_reason}", 1) continue + args = [] for user in users: logger(f"Plex: Generating watched for {user.title} in library {library_title}", 0) user_name = user.title.lower() - watched = self.get_user_watched(user, library) - if watched: + watched = args.append([self.get_user_watched, user, library]) + + for user_watched in future_thread_executor(args): + if user_watched: if user_name not in users_watched: users_watched[user_name] = {} - if library_title not in users_watched[user_name]: - users_watched[user_name][library_title] = [] - users_watched[user_name][library_title] = watched - + users_watched[user_name][library_title] = user_watched return users_watched + def update_user_watched (self, user, user_plex, library, videos, dryrun): + logger(f"Plex: Updating watched for {user.title} in library {library}", 1) + library_videos = user_plex.library.section(library) + + if library_videos.type == "movie": + _, _, videos_movies_ids = generate_library_guids_dict(videos, 2) + for movies_search in library_videos.search(unmatched=False, unwatched=True): + for movie_guid in movies_search.guids: + movie_guid_source = re.search(r'(.*)://', movie_guid.id).group(1).lower() + movie_guid_id = re.search(r'://(.*)', movie_guid.id).group(1) + + # If movie provider source and movie provider id are in videos_movie_ids exactly, then the movie is in the list + if movie_guid_source in videos_movies_ids.keys(): + if movie_guid_id in videos_movies_ids[movie_guid_source]: + if movies_search.viewCount == 0: + msg = f"{movies_search.title} as watched for {user.title} in {library} for Plex" + if not dryrun: + logger(f"Marked {msg}", 0) + movies_search.markWatched() + else: + logger(f"Dryrun {msg}", 0) + break + + + elif library_videos.type == "show": + videos_shows_ids, videos_episode_ids, _ = generate_library_guids_dict(videos, 3) + + for show_search in library_videos.search(unmatched=False, unwatched=True): + show_found = False + for show_guid in show_search.guids: + show_guid_source = re.search(r'(.*)://', show_guid.id).group(1).lower() + show_guid_id = re.search(r'://(.*)', show_guid.id).group(1) + + # If show provider source and show provider id are in videos_shows_ids exactly, then the show is in the list + if show_guid_source in videos_shows_ids.keys(): + if show_guid_id in videos_shows_ids[show_guid_source]: + show_found = True + for episode_search in show_search.episodes(): + for episode_guid in episode_search.guids: + episode_guid_source = re.search(r'(.*)://', episode_guid.id).group(1).lower() + episode_guid_id = re.search(r'://(.*)', episode_guid.id).group(1) + + # If episode provider source and episode provider id are in videos_episode_ids exactly, then the episode is in the list + if episode_guid_source in videos_episode_ids.keys(): + if episode_guid_id in videos_episode_ids[episode_guid_source]: + if episode_search.viewCount == 0: + msg = f"{show_search.title} {episode_search.title} as watched for {user.title} in {library} for Plex" + if not dryrun: + logger(f"Marked {msg}", 0) + episode_search.markWatched() + else: + logger(f"Dryrun {msg}", 0) + break + + if show_found: + break + + + def update_watched(self, watched_list, user_mapping=None, library_mapping=None, dryrun=False): + args = [] + for user, libraries in watched_list.items(): user_other = None # If type of user is dict @@ -171,58 +234,7 @@ class Plex: logger(f"Library {library} {library_other} not found in Plex library list", 2) continue - logger(f"Plex: Updating watched for {user.title} in library {library}", 1) - library_videos = user_plex.library.section(library) - if library_videos.type == "movie": - _, _, videos_movies_ids = generate_library_guids_dict(videos, 2) - for movies_search in library_videos.search(unmatched=False, unwatched=True): - for movie_guid in movies_search.guids: - movie_guid_source = re.search(r'(.*)://', movie_guid.id).group(1).lower() - movie_guid_id = re.search(r'://(.*)', movie_guid.id).group(1) + args.append([self.update_user_watched, user, user_plex, library, videos, dryrun]) - # If movie provider source and movie provider id are in videos_movie_ids exactly, then the movie is in the list - if movie_guid_source in videos_movies_ids.keys(): - if movie_guid_id in videos_movies_ids[movie_guid_source]: - if movies_search.viewCount == 0: - msg = f"{movies_search.title} as watched for {user.title} in {library} for Plex" - if not dryrun: - logger(f"Marked {msg}", 0) - movies_search.markWatched() - else: - logger(f"Dryrun {msg}", 0) - break - - - elif library_videos.type == "show": - videos_shows_ids, videos_episode_ids, _ = generate_library_guids_dict(videos, 3) - - for show_search in library_videos.search(unmatched=False, unwatched=True): - show_found = False - for show_guid in show_search.guids: - show_guid_source = re.search(r'(.*)://', show_guid.id).group(1).lower() - show_guid_id = re.search(r'://(.*)', show_guid.id).group(1) - - # If show provider source and show provider id are in videos_shows_ids exactly, then the show is in the list - if show_guid_source in videos_shows_ids.keys(): - if show_guid_id in videos_shows_ids[show_guid_source]: - show_found = True - for episode_search in show_search.episodes(): - for episode_guid in episode_search.guids: - episode_guid_source = re.search(r'(.*)://', episode_guid.id).group(1).lower() - episode_guid_id = re.search(r'://(.*)', episode_guid.id).group(1) - - # If episode provider source and episode provider id are in videos_episode_ids exactly, then the episode is in the list - if episode_guid_source in videos_episode_ids.keys(): - if episode_guid_id in videos_episode_ids[episode_guid_source]: - if episode_search.viewCount == 0: - msg = f"{show_search.title} {episode_search.title} as watched for {user.title} in {library} for Plex" - if not dryrun: - logger(f"Marked {msg}", 0) - episode_search.markWatched() - else: - logger(f"Dryrun {msg}", 0) - break - - if show_found: - break + future_thread_executor(args)