From f8ef4fe6c990dbd5f09bd1b9b6cc4bab234be2a3 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sat, 29 Apr 2023 20:31:24 -0600 Subject: [PATCH 01/62] Add docker-compose file --- docker-compose.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e3783f1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +version: '3' + +services: + jellyplex-watched: + image: luigi311/jellyplex-watched:latest + container_name: jellyplex-watched + restart: always + environment: + - DRYRUN=True + - DEBUG=True + - DEBUG_LEVEL=info + - RUN_ONLY_ONCE=False + - SLEEP_DURATION=3600 + - LOGFILE=/tmp/log.log + - USER_MAPPING= + - LIBRARY_MAPPING={"TV Shows":"Shows"} + - BLACKLIST_LIBRARY= + - WHITELIST_LIBRARY= + - BLACKLIST_LIBRARY_TYPE= + - WHITELIST_LIBRARY_TYPE= + - BLACKLIST_USERS= + - WHITELIST_USERS= + - PLEX_BASEURL= + - PLEX_TOKEN= + - JELLYFIN_BASEURL= + - JELLYFIN_TOKEN= + - SSL_BYPASS=True + - SYNC_FROM_PLEX_TO_JELLYFIN=True + - SYNC_FROM_JELLYFIN_TO_PLEX=True + - SYNC_FROM_PLEX_TO_PLEX=True + - SYNC_FROM_JELLYFIN_TO_JELLYFIN=True From fd281a50b63ccd9323989f1194b5d58adc926b7b Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 15 May 2023 10:44:30 -0600 Subject: [PATCH 02/62] Log both servers users instead of exiting immediately --- src/main.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main.py b/src/main.py index f308310..715e5d8 100644 --- a/src/main.py +++ b/src/main.py @@ -40,15 +40,23 @@ def setup_users( # Check if users is none or empty if output_server_1_users is None or len(output_server_1_users) == 0: - raise Exception( - f"No users found for server 1 {server_1[0]}, users found {users}, filtered users {users_filtered}, server 1 users {server_1[1].users}" + logger( + f"No users found for server 1 {server_1[0]}, users: {server_1_users}, overlapping users {users}, filtered users {users_filtered}, server 1 users {server_1[1].users}" ) if output_server_2_users is None or len(output_server_2_users) == 0: - raise Exception( - f"No users found for server 2 {server_2[0]}, users found {users} filtered users {users_filtered}, server 2 users {server_2[1].users}" + logger( + f"No users found for server 2 {server_2[0]}, users: {server_2_users}, overlapping users {users} filtered users {users_filtered}, server 2 users {server_2[1].users}" ) + if ( + output_server_1_users is None + or len(output_server_1_users) == 0 + or output_server_2_users is None + or len(output_server_2_users) == 0 + ): + raise Exception("No users found for one or both servers") + logger(f"Server 1 users: {output_server_1_users}", 1) logger(f"Server 2 users: {output_server_2_users}", 1) From c4a2f8af39bb50580715abdb6bd3e215b2b7acfa Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 15 May 2023 11:10:03 -0600 Subject: [PATCH 03/62] Users: Default to username and fall back to title --- src/users.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/users.py b/src/users.py index 6075be8..fc8962b 100644 --- a/src/users.py +++ b/src/users.py @@ -11,7 +11,12 @@ def generate_user_list(server): server_users = [] if server_type == "plex": - server_users = [x.title.lower() for x in server_connection.users] + for user in server_connection.users: + if user.username: + server_users.append(user.username.lower()) + else: + server_users.append(user.title.lower()) + elif server_type == "jellyfin": server_users = [key.lower() for key in server_connection.users.keys()] From 43ead4bb0f8c44a255f5335580e3043e8d417c57 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 15 May 2023 11:17:28 -0600 Subject: [PATCH 04/62] Plex: Fix username/title selection --- src/plex.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/plex.py b/src/plex.py index 889078d..0402016 100644 --- a/src/plex.py +++ b/src/plex.py @@ -126,7 +126,9 @@ def get_user_library_watched_show(show): def get_user_library_watched(user, user_plex, library): try: - user_name = user.title.lower() + user_name = user.username.lower() + if user_name == "": + user_name = user.title.lower() user_watched = {} user_watched[user_name] = {} @@ -509,10 +511,14 @@ class Plex: user_other = search_mapping(user_mapping, user) for index, value in enumerate(self.users): - if user.lower() == value.title.lower(): + username_title = value.username.lower() + if username_title == "": + username_title = value.title.lower() + + if user.lower() == username_title: user = self.users[index] break - elif user_other and user_other.lower() == value.title.lower(): + elif user_other and user_other.lower() == username_title: user = self.users[index] break From e4b4c7ba390ce289432f7336a57007a4524c44ec Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 15 May 2023 14:57:46 -0600 Subject: [PATCH 05/62] plex: Fix username/title --- src/users.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/users.py b/src/users.py index fc8962b..e6e2309 100644 --- a/src/users.py +++ b/src/users.py @@ -71,9 +71,13 @@ def generate_server_users(server, users): if server[0] == "plex": server_users = [] for plex_user in server[1].users: + username_title = ( + plex_user.username if plex_user.username != "" else plex_user.title + ) + if ( - plex_user.title.lower() in users.keys() - or plex_user.title.lower() in users.values() + username_title.lower() in users.keys() + or username_title.lower() in users.values() ): server_users.append(plex_user) elif server[0] == "jellyfin": From da808ba25e4560b69c8b6478cfdbd29b234b377c Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 15 May 2023 15:12:25 -0600 Subject: [PATCH 06/62] CI: Add back in dev based on alpine --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63fec60..826a647 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,8 @@ jobs: # Do not push to ghcr.io on PRs due to permission issues ghcr.io/${{ github.repository }},enable=${{ github.event_name != 'pull_request' }} tags: | - type=raw,value=latest,enable=${{ matrix.variant == 'alpine' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + type=raw,value=latest,enable=${{ matrix.variant == 'alpine' && github.ref_name == github.event.repository.default_branch }} + type=raw,value=dev,enable=${{ matrix.variant == 'alpine' && github.ref_name == 'dev' }} type=raw,value=latest,suffix=-${{ matrix.variant }},enable={{ is_default_branch }} type=ref,event=branch,suffix=-${{ matrix.variant }} type=ref,event=pr,suffix=-${{ matrix.variant }} From 945ffb2fb3b6986fa5bfc45e8644517300808425 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Wed, 17 May 2023 13:22:00 -0600 Subject: [PATCH 07/62] Plex: Cleanup username_title --- src/plex.py | 12 ++++++------ src/users.py | 9 ++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/plex.py b/src/plex.py index 0402016..0bcdf7b 100644 --- a/src/plex.py +++ b/src/plex.py @@ -126,9 +126,7 @@ def get_user_library_watched_show(show): def get_user_library_watched(user, user_plex, library): try: - user_name = user.username.lower() - if user_name == "": - user_name = user.title.lower() + user_name = user.username.lower() if user.username else user.title.lower() user_watched = {} user_watched[user_name] = {} @@ -511,9 +509,11 @@ class Plex: user_other = search_mapping(user_mapping, user) for index, value in enumerate(self.users): - username_title = value.username.lower() - if username_title == "": - username_title = value.title.lower() + username_title = ( + value.username.lower() + if value.username + else value.title.lower() + ) if user.lower() == username_title: user = self.users[index] diff --git a/src/users.py b/src/users.py index e6e2309..4f1e280 100644 --- a/src/users.py +++ b/src/users.py @@ -12,10 +12,9 @@ def generate_user_list(server): server_users = [] if server_type == "plex": for user in server_connection.users: - if user.username: - server_users.append(user.username.lower()) - else: - server_users.append(user.title.lower()) + server_users.append( + user.username.lower() if user.username else user.title.lower() + ) elif server_type == "jellyfin": server_users = [key.lower() for key in server_connection.users.keys()] @@ -72,7 +71,7 @@ def generate_server_users(server, users): server_users = [] for plex_user in server[1].users: username_title = ( - plex_user.username if plex_user.username != "" else plex_user.title + plex_user.username if plex_user.username else plex_user.title ) if ( From d87542ab789cd3ac33b09db87701ac9f0050274f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 May 2023 06:45:31 +0000 Subject: [PATCH 08/62] Bump requests from 2.28.2 to 2.31.0 Bumps [requests](https://github.com/psf/requests) from 2.28.2 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.28.2...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index de75d94..2f36597 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ PlexAPI==4.13.4 -requests==2.28.2 +requests==2.31.0 python-dotenv==1.0.0 aiohttp==3.8.4 From e94a8fb2c318a47e75e1a7086dae073544ac12fc Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 22 May 2023 01:22:34 -0600 Subject: [PATCH 09/62] Jellyfin: Fix errors with missing matches Signed-off-by: Luigi311 --- src/jellyfin.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index f17f445..11184b9 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -582,9 +582,8 @@ class Jellyfin: ] ): for video in videos: - if ( - movie_provider_id.lower() - in video[movie_provider_source.lower()] + if movie_provider_id.lower() in video.get( + movie_provider_source.lower(), [] ): movie_status = video["status"] break @@ -671,9 +670,8 @@ class Jellyfin: episode_videos = [] for show, seasons in videos.items(): show = {k: v for k, v in show} - if ( - show_provider_id.lower() - in show[show_provider_source.lower()] + if show_provider_id.lower() in show.get( + show_provider_source.lower(), [] ): for season in seasons.values(): for episode in season: @@ -752,7 +750,7 @@ class Jellyfin: if episode_status["completed"]: jellyfin_episode_id = jellyfin_episode["Id"] msg = ( - f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode['IndexNumber']} {jellyfin_episode['Name']}" + f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode['Name']}" + f" as watched for {user_name} in {library} for Jellyfin" ) if not dryrun: @@ -768,7 +766,7 @@ class Jellyfin: # TODO add support for partially watched episodes jellyfin_episode_id = jellyfin_episode["Id"] msg = ( - f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode['IndexNumber']} {jellyfin_episode['Name']}" + f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode['Name']}" + f" as partially watched for {floor(episode_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" ) if not dryrun: From da5360938540b861b9aea64ba8d097745f8ccda3 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 23 May 2023 14:33:42 -0600 Subject: [PATCH 10/62] Jellyfin: Remove reassigning jellyfin_episode_id Signed-off-by: Luigi311 --- src/jellyfin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 11184b9..78f0484 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -747,8 +747,8 @@ class Jellyfin: break if episode_status: + jellyfin_episode_id = jellyfin_episode["Id"] if episode_status["completed"]: - jellyfin_episode_id = jellyfin_episode["Id"] msg = ( f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode['Name']}" + f" as watched for {user_name} in {library} for Jellyfin" @@ -764,7 +764,6 @@ class Jellyfin: logger(f"Dryrun {msg}", 0) else: # TODO add support for partially watched episodes - jellyfin_episode_id = jellyfin_episode["Id"] msg = ( f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode['Name']}" + f" as partially watched for {floor(episode_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" From a227c01a7ff0b77424ec67a624643beb44217e6f Mon Sep 17 00:00:00 2001 From: Lai Jiang Date: Mon, 29 May 2023 21:08:12 -0400 Subject: [PATCH 11/62] Fix a type --- src/black_white.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black_white.py b/src/black_white.py index e32ee08..5b44c03 100644 --- a/src/black_white.py +++ b/src/black_white.py @@ -15,7 +15,7 @@ def setup_black_white_lists( blacklist_library, blacklist_library_type, blacklist_users, - "White", + "Black", library_mapping, user_mapping, ) @@ -24,7 +24,7 @@ def setup_black_white_lists( whitelist_library, whitelist_library_type, whitelist_users, - "Black", + "White", library_mapping, user_mapping, ) From dd64617cbd4cfb139ce67c9a0d863e8f271c0385 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Wed, 28 Jun 2023 16:21:07 -0600 Subject: [PATCH 12/62] Jellyfin: Handle missing paths Signed-off-by: Luigi311 --- src/jellyfin.py | 97 ++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 41 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 78f0484..4f71157 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -49,8 +49,11 @@ def get_episode_guids(episode): # Create a dictionary for the episode with its provider IDs and media sources episode_dict = {k.lower(): v for k, v in episode["ProviderIds"].items()} episode_dict["title"] = episode["Name"] - episode_dict["locations"] = tuple( - [x["Path"].split("/")[-1] for x in episode["MediaSources"]] + + episode_dict["locations"] = ( + tuple([x["Path"].split("/")[-1] for x in ["MediaSources"] if "Path" in x]) + if "MediaSources" in episode + else tuple() ) episode_dict["status"] = { @@ -238,7 +241,11 @@ class Jellyfin: k.lower(): v for k, v in show["ProviderIds"].items() } show_guids["title"] = show["Name"] - show_guids["locations"] = tuple([show["Path"].split("/")[-1]]) + show_guids["locations"] = ( + tuple([show["Path"].split("/")[-1]]) + if "Path" in show + else tuple() + ) show_guids = frozenset(show_guids.items()) show_identifiers = { "show_guids": show_guids, @@ -550,24 +557,27 @@ class Jellyfin: if "MediaSources" in jellyfin_video: for movie_location in jellyfin_video["MediaSources"]: - if ( - contains_nested( - movie_location["Path"].split("/")[-1], - videos_movies_ids["locations"], - ) - is not None - ): - for video in videos: - if ( - contains_nested( - movie_location["Path"].split("/")[-1], - video["locations"], - ) - is not None - ): - movie_status = video["status"] - break - break + if "Path" in movie_location: + if ( + contains_nested( + movie_location["Path"].split("/")[-1], + videos_movies_ids["locations"], + ) + is not None + ): + for video in videos: + if ( + contains_nested( + movie_location["Path"].split("/")[ + -1 + ], + video["locations"], + ) + is not None + ): + movie_status = video["status"] + break + break if not movie_status: for ( @@ -697,26 +707,31 @@ class Jellyfin: for episode_location in jellyfin_episode[ "MediaSources" ]: - if ( - contains_nested( - episode_location["Path"].split("/")[-1], - videos_episodes_ids["locations"], - ) - is not None - ): - for episode in episode_videos: - if ( - contains_nested( - episode_location["Path"].split( - "/" - )[-1], - episode["locations"], - ) - is not None - ): - episode_status = episode["status"] - break - break + if "Path" in episode_location: + if ( + contains_nested( + episode_location["Path"].split("/")[ + -1 + ], + videos_episodes_ids["locations"], + ) + is not None + ): + for episode in episode_videos: + if ( + contains_nested( + episode_location[ + "Path" + ].split("/")[-1], + episode["locations"], + ) + is not None + ): + episode_status = episode[ + "status" + ] + break + break if not episode_status: for ( From 7ef37fe848c0d6dbddb9a1ad24181dc951fc873c Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Wed, 28 Jun 2023 16:52:23 -0600 Subject: [PATCH 13/62] Jellyfin: Check for provider_source in episode Signed-off-by: Luigi311 --- src/jellyfin.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 4f71157..32078bc 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -750,15 +750,19 @@ class Jellyfin: ): for episode in episode_videos: if ( - episode_provider_id.lower() - in episode[ - episode_provider_source.lower() - ] + episode_provider_source.lower() + in episode ): - episode_status = episode[ - "status" - ] - break + if ( + episode_provider_id.lower() + in episode[ + episode_provider_source.lower() + ] + ): + episode_status = episode[ + "status" + ] + break break if episode_status: From d9e6a554f6f4f4a663bdb0b595f253d896442218 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Wed, 28 Jun 2023 16:55:56 -0600 Subject: [PATCH 14/62] Disable fast fail Signed-off-by: Luigi311 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 826a647..4dcee0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,7 @@ jobs: runs-on: ubuntu-latest needs: pytest strategy: + fail-fast: false matrix: include: - dockerfile: Dockerfile.alpine From e6b33f1bc9d2634e3a53796e99409f2f239e0584 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 18 Jul 2023 16:27:13 -0600 Subject: [PATCH 15/62] Jellyfin: Fix locations logic Signed-off-by: Luigi311 --- src/jellyfin.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 32078bc..03beeba 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -35,6 +35,8 @@ def get_movie_guids(movie): movie_guids["locations"] = tuple( [x["Path"].split("/")[-1] for x in movie["MediaSources"]] ) + else: + movie_guids["locations"] = tuple() movie_guids["status"] = { "completed": movie["UserData"]["Played"], @@ -50,11 +52,11 @@ def get_episode_guids(episode): episode_dict = {k.lower(): v for k, v in episode["ProviderIds"].items()} episode_dict["title"] = episode["Name"] - episode_dict["locations"] = ( - tuple([x["Path"].split("/")[-1] for x in ["MediaSources"] if "Path" in x]) - if "MediaSources" in episode - else tuple() - ) + episode_dict["locations"] = tuple() + if "MediaSources" in episode: + for x in episode["MediaSources"]: + if "Path" in x: + episode_dict["locations"] += (x["Path"].split("/")[-1],) episode_dict["status"] = { "completed": episode["UserData"]["Played"], From 03617dacfc850c971ea064145d4a1ffa011b91e1 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Wed, 16 Aug 2023 19:00:17 -0600 Subject: [PATCH 16/62] Jellyfin: Remove isPlayed, Use get for name Signed-off-by: Luigi311 --- src/jellyfin.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 03beeba..e4bd4b5 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -14,12 +14,12 @@ from src.watched import ( def get_movie_guids(movie): if "ProviderIds" in movie: logger( - f"Jellyfin: {movie['Name']} {movie['ProviderIds']} {movie['MediaSources']}", + f"Jellyfin: {movie.get('Name')} {movie['ProviderIds']} {movie['MediaSources']}", 3, ) else: logger( - f"Jellyfin: {movie['Name']} {movie['MediaSources']['Path']}", + f"Jellyfin: {movie.get('Name')} {movie['MediaSources']['Path']}", 3, ) @@ -177,7 +177,7 @@ class Jellyfin: for movie in watched["Items"]: if "MediaSources" in movie and movie["MediaSources"] != {}: logger( - f"Jellyfin: Adding {movie['Name']} to {user_name} watched list", + f"Jellyfin: Adding {movie.get('Name')} to {user_name} watched list", 3, ) @@ -198,7 +198,7 @@ class Jellyfin: continue logger( - f"Jellyfin: Adding {movie['Name']} to {user_name} watched list", + f"Jellyfin: Adding {movie.get('Name')} to {user_name} watched list", 3, ) @@ -236,7 +236,7 @@ class Jellyfin: seasons_tasks = [] for show in watched_shows_filtered: logger( - f"Jellyfin: Adding {show['Name']} to {user_name} watched list", + f"Jellyfin: Adding {show.get('Name')} to {user_name} watched list", 3, ) show_guids = { @@ -604,7 +604,7 @@ class Jellyfin: if movie_status: jellyfin_video_id = jellyfin_video["Id"] if movie_status["completed"]: - msg = f"{jellyfin_video['Name']} as watched for {user_name} in {library} for Jellyfin" + msg = f"{jellyfin_video.get('Name')} as watched for {user_name} in {library} for Jellyfin" if not dryrun: logger(f"Marking {msg}", 0) await self.query( @@ -616,7 +616,7 @@ class Jellyfin: logger(f"Dryrun {msg}", 0) else: # TODO add support for partially watched movies - msg = f"{jellyfin_video['Name']} as partially watched for {floor(movie_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" + msg = f"{jellyfin_video.get('Name')} as partially watched for {floor(movie_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" if not dryrun: pass # logger(f"Marked {msg}", 0) @@ -625,7 +625,7 @@ class Jellyfin: # logger(f"Dryrun {msg}", 0) else: logger( - f"Jellyfin: Skipping movie {jellyfin_video['Name']} as it is not in mark list for {user_name}", + f"Jellyfin: Skipping movie {jellyfin_video.get('Name')} as it is not in mark list for {user_name}", 1, ) @@ -634,7 +634,7 @@ class Jellyfin: jellyfin_search = await self.query( f"/Users/{user_id}/Items" + f"?SortBy=SortName&SortOrder=Ascending&Recursive=True&ParentId={library_id}" - + "&isPlayed=false&Fields=ItemCounts,ProviderIds,Path&IncludeItemTypes=Series", + + "&Fields=ItemCounts,ProviderIds,Path&IncludeItemTypes=Series", "get", session, ) @@ -691,7 +691,7 @@ class Jellyfin: if show_found: logger( - f"Jellyfin: Updating watched for {user_name} in library {library} for show {jellyfin_show['Name']}", + f"Jellyfin: Updating watched for {user_name} in library {library} for show {jellyfin_show.get('Name')}", 1, ) jellyfin_show_id = jellyfin_show["Id"] @@ -771,7 +771,7 @@ class Jellyfin: jellyfin_episode_id = jellyfin_episode["Id"] if episode_status["completed"]: msg = ( - f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode['Name']}" + f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode.get('Name')}" + f" as watched for {user_name} in {library} for Jellyfin" ) if not dryrun: @@ -786,7 +786,7 @@ class Jellyfin: else: # TODO add support for partially watched episodes msg = ( - f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode['Name']}" + f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode.get('Name')}" + f" as partially watched for {floor(episode_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" ) if not dryrun: @@ -797,12 +797,12 @@ class Jellyfin: # logger(f"Dryrun {msg}", 0) else: logger( - f"Jellyfin: Skipping episode {jellyfin_episode['Name']} as it is not in mark list for {user_name}", + f"Jellyfin: Skipping episode {jellyfin_episode.get('Name')} as it is not in mark list for {user_name}", 3, ) else: logger( - f"Jellyfin: Skipping show {jellyfin_show['Name']} as it is not in mark list for {user_name}", + f"Jellyfin: Skipping show {jellyfin_show.get('Name')} as it is not in mark list for {user_name}", 3, ) From e1fb365096e08d4a4ee19aeef4df53f02c6b07ee Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Thu, 28 Sep 2023 09:47:34 -0600 Subject: [PATCH 17/62] Update apis Signed-off-by: Luigi311 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2f36597..027232b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -PlexAPI==4.13.4 +PlexAPI==4.15.2 requests==2.31.0 python-dotenv==1.0.0 -aiohttp==3.8.4 +aiohttp==3.8.5 From 116d50a75aa8ad21e075efe57508581c8b780e4e Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Thu, 28 Sep 2023 10:00:07 -0600 Subject: [PATCH 18/62] Add max_threads Signed-off-by: Luigi311 --- .env.sample | 6 ++++++ src/functions.py | 5 ++--- src/plex.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.env.sample b/.env.sample index 2f231ea..2953a0e 100644 --- a/.env.sample +++ b/.env.sample @@ -18,6 +18,12 @@ SLEEP_DURATION = "3600" ## Log file where all output will be written to LOGFILE = "log.log" +## Timeout for requests for jellyfin +REQUEST_TIMEOUT = 300 + +## Max threads for processing +MAX_THREADS = 32 + ## Map usernames between servers in the event that they are different, order does not matter ## Comma separated for multiple options #USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" } diff --git a/src/functions.py b/src/functions.py index 2e7529b..49dc67e 100644 --- a/src/functions.py +++ b/src/functions.py @@ -63,12 +63,11 @@ def search_mapping(dictionary: dict, key_value: str): return None -def future_thread_executor(args: list, workers: int = -1): +def future_thread_executor(args: list, threads: int = 32): futures_list = [] results = [] - if workers == -1: - workers = min(32, os.cpu_count() * 2) + workers = min(int(os.getenv("MAX_THREADS", 32)), os.cpu_count() * 2, threads) with ThreadPoolExecutor(max_workers=workers) as executor: for arg in args: diff --git a/src/plex.py b/src/plex.py index 0bcdf7b..715daae 100644 --- a/src/plex.py +++ b/src/plex.py @@ -174,7 +174,7 @@ def get_user_library_watched(user, user_plex, library): args.append([get_user_library_watched_show, show]) for show_guids, episode_guids in future_thread_executor( - args, workers=min(os.cpu_count(), 4) + args, threads=4 ): if show_guids and episode_guids: # append show, season, episode From b53d7c9eccaf646f77a90c403f6d0cec975ef8f0 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Thu, 28 Sep 2023 10:45:02 -0600 Subject: [PATCH 19/62] Add docker compose to types Signed-off-by: Luigi311 --- .github/ISSUE_TEMPLATE/bug_report.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 298ced2..08d0947 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,7 +24,8 @@ A clear and concise description of what you expected to happen. If applicable, add logs to help explain your problem ideally with DEBUG set to true, be sure to remove sensitive information **Type:** -- [ ] Docker +- [ ] Docker Compose +- [ ] Docker - [ ] Native **Additional context** From 43d6bc0d82bb478dab56ef1d00ba592ef5eb32f6 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 25 Sep 2023 01:59:16 -0600 Subject: [PATCH 20/62] Timeout issues (#103) * Add timeout support for jellyfin requests Signed-off-by: Luigi311 --- .env.sample | 3 --- README.md | 3 +++ src/jellyfin.py | 21 +++++++++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.env.sample b/.env.sample index 2953a0e..af6e63e 100644 --- a/.env.sample +++ b/.env.sample @@ -21,9 +21,6 @@ LOGFILE = "log.log" ## Timeout for requests for jellyfin REQUEST_TIMEOUT = 300 -## Max threads for processing -MAX_THREADS = 32 - ## Map usernames between servers in the event that they are different, order does not matter ## Comma separated for multiple options #USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" } diff --git a/README.md b/README.md index 35e1c76..899425d 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,9 @@ SLEEP_DURATION = "3600" ## Log file where all output will be written to LOGFILE = "log.log" +## Timeout for requests for jellyfin +REQUEST_TIMEOUT = 300 + ## Map usernames between servers in the event that they are different, order does not matter ## Comma separated for multiple options USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" } diff --git a/src/jellyfin.py b/src/jellyfin.py index e4bd4b5..2dbd7f3 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -1,5 +1,6 @@ -import asyncio, aiohttp, traceback +import asyncio, aiohttp, traceback, os from math import floor +from dotenv import load_dotenv from src.functions import logger, search_mapping, contains_nested from src.library import ( @@ -10,6 +11,8 @@ from src.watched import ( combine_watched_dicts, ) +load_dotenv(override=True) + def get_movie_guids(movie): if "ProviderIds" in movie: @@ -70,6 +73,12 @@ class Jellyfin: def __init__(self, baseurl, token): self.baseurl = baseurl self.token = token + self.timeout = aiohttp.ClientTimeout( + total = int(os.getenv("REQUEST_TIMEOUT", 300)), + connect=None, + sock_connect=None, + sock_read=None, + ) if not self.baseurl: raise Exception("Jellyfin baseurl not set") @@ -130,7 +139,7 @@ class Jellyfin: users = {} query_string = "/Users" - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=self.timeout) as session: response = await self.query(query_string, "get", session) # If response is not empty @@ -156,7 +165,7 @@ class Jellyfin: 0, ) - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=self.timeout) as session: # Movies if library_type == "Movie": user_watched[user_name][library_title] = [] @@ -404,7 +413,7 @@ class Jellyfin: tasks_watched = [] tasks_libraries = [] - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=self.timeout) as session: libraries = await self.query(f"/Users/{user_id}/Views", "get", session) for library in libraries["Items"]: library_id = library["Id"] @@ -545,7 +554,7 @@ class Jellyfin: f"Jellyfin: mark list\nShows: {videos_shows_ids}\nEpisodes: {videos_episodes_ids}\nMovies: {videos_movies_ids}", 1, ) - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=self.timeout) as session: if videos_movies_ids: jellyfin_search = await self.query( f"/Users/{user_id}/Items" @@ -829,7 +838,7 @@ class Jellyfin: ): try: tasks = [] - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(timeout=self.timeout) as session: for user, libraries in watched_list.items(): logger(f"Jellyfin: Updating for entry {user}, {libraries}", 1) user_other = None From 4de25a0d4a902e3de2a657df5d5d5bac27a0638d Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Thu, 28 Sep 2023 19:41:53 -0600 Subject: [PATCH 21/62] Print server info Signed-off-by: Luigi311 --- src/black_white.py | 40 +--------------------------------------- src/jellyfin.py | 23 +++++++++++++++++++++-- src/main.py | 3 +++ src/plex.py | 9 +++++---- 4 files changed, 30 insertions(+), 45 deletions(-) diff --git a/src/black_white.py b/src/black_white.py index 5b44c03..494818a 100644 --- a/src/black_white.py +++ b/src/black_white.py @@ -38,6 +38,7 @@ def setup_black_white_lists( whitelist_users, ) + def setup_x_lists( xlist_library, xlist_library_type, @@ -89,42 +90,3 @@ def setup_x_lists( logger(f"{xlist_type}list Users: {xlist_users}", 1) return xlist_library, xlist_library_type, xlist_users - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/jellyfin.py b/src/jellyfin.py index 2dbd7f3..50555f9 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -74,7 +74,7 @@ class Jellyfin: self.baseurl = baseurl self.token = token self.timeout = aiohttp.ClientTimeout( - total = int(os.getenv("REQUEST_TIMEOUT", 300)), + total=int(os.getenv("REQUEST_TIMEOUT", 300)), connect=None, sock_connect=None, sock_read=None, @@ -88,8 +88,12 @@ class Jellyfin: self.users = asyncio.run(self.get_users()) - async def query(self, query, query_type, session, identifiers=None): + async def query(self, query, query_type, session=None, identifiers=None): try: + if not session: + async with aiohttp.ClientSession(timeout=self.timeout) as session: + return await self.query(query, query_type, session, identifiers) + results = None headers = {"Accept": "application/json", "X-Emby-Token": self.token} authorization = ( @@ -134,6 +138,21 @@ class Jellyfin: logger(f"Jellyfin: Query {query_type} {query}\nResults {results}\n{e}", 2) raise Exception(e) + def info(self) -> str: + try: + query_string = "/System/Info/Public" + + response = asyncio.run(self.query(query_string, "get")) + + if response: + return f"{response['ServerName']}: {response['Version']}" + else: + return None + + except Exception as e: + logger(f"Jellyfin: Get server name failed {e}", 2) + raise Exception(e) + async def get_users(self): try: users = {} diff --git a/src/main.py b/src/main.py index 715e5d8..562dc46 100644 --- a/src/main.py +++ b/src/main.py @@ -304,6 +304,9 @@ def main_loop(): # Start server_2 at the next server in the list for server_2 in servers[servers.index(server_1) + 1 :]: + logger(f"Server 1: {server_1[0].capitalize()}: {server_1[1].info()}", 0) + logger(f"Server 2: {server_2[0].capitalize()}: {server_2[1].info()}", 0) + # Create users list logger("Creating users list", 1) server_1_users, server_2_users = setup_users( diff --git a/src/plex.py b/src/plex.py index 715daae..df80adf 100644 --- a/src/plex.py +++ b/src/plex.py @@ -1,4 +1,4 @@ -import re, requests, os, traceback +import re, requests, traceback from urllib3.poolmanager import PoolManager from math import floor @@ -173,9 +173,7 @@ def get_user_library_watched(user, user_plex, library): for show in library_videos.search(inProgress=True): args.append([get_user_library_watched_show, show]) - for show_guids, episode_guids in future_thread_executor( - args, threads=4 - ): + for show_guids, episode_guids in future_thread_executor(args, threads=4): if show_guids and episode_guids: # append show, season, episode if show_guids not in user_watched[user_name][library.title]: @@ -414,6 +412,9 @@ class Plex: logger(f"Plex: Failed to login, Error: {e}", 2) raise Exception(e) + def info(self) -> str: + return f"{self.plex.friendlyName}: {self.plex.version}" + def get_users(self): try: users = self.plex.myPlexAccount().users() From 466f292febcd90c26ac8c6d88f5d0acc17a84ea1 Mon Sep 17 00:00:00 2001 From: neofright <68615872+neofright@users.noreply.github.com> Date: Fri, 3 Nov 2023 13:14:46 +0000 Subject: [PATCH 22/62] Typo in .env.sample --- .env.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index af6e63e..4a7bdb0 100644 --- a/.env.sample +++ b/.env.sample @@ -25,7 +25,7 @@ REQUEST_TIMEOUT = 300 ## Comma separated for multiple options #USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" } -## Map libraries between servers in the even that they are different, order does not matter +## Map libraries between servers in the event that they are different, order does not matter ## Comma separated for multiple options #LIBRARY_MAPPING = { "Shows": "TV Shows", "Movie": "Movies" } From 58c1eb7004bc7956fa70f3448ef3fffc5d995a39 Mon Sep 17 00:00:00 2001 From: neofright <68615872+neofright@users.noreply.github.com> Date: Fri, 3 Nov 2023 13:21:07 +0000 Subject: [PATCH 23/62] Improve README.md - Inprogress is not a word in English, but two separate words. - Many words are unnecessarily captialised as they are not names or at the beginning of the sentences. - Prefer 'usernames' to 'usersnames' --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 899425d..5fbb898 100644 --- a/README.md +++ b/README.md @@ -12,33 +12,33 @@ Keep in sync all your users watched history between jellyfin and plex servers lo ### Plex -* \[x] Match via Filenames +* \[x] Match via filenames * \[x] Match via provider ids -* \[x] Map usersnames +* \[x] Map usernames * \[x] Use single login -* \[x] One Way/Multi Way sync -* \[x] Sync Watched -* \[x] Sync Inprogress +* \[x] One way/multi way sync +* \[x] Sync watched +* \[x] Sync in progress ### Jellyfin -* \[x] Match via Filenames +* \[x] Match via filenames * \[x] Match via provider ids -* \[x] Map usersnames +* \[x] Map usernames * \[x] Use single login -* \[x] One Way/Multi Way sync -* \[x] Sync Watched -* \[ ] Sync Inprogress +* \[x] One way/multi way sync +* \[x] Sync watched +* \[ ] Sync in progress ### Emby -* \[ ] Match via Filenames +* \[ ] Match via filenames * \[ ] Match via provider ids -* \[ ] Map usersnames +* \[ ] Map usernames * \[ ] Use single login -* \[ ] One Way/Multi Way sync -* \[ ] Sync Watched -* \[ ] Sync Inprogress +* \[ ] One way/multi Way sync +* \[ ] Sync watched +* \[ ] Sync in progress ## Configuration From 79325b8c614b7981a9b1f428803e2e8103db856f Mon Sep 17 00:00:00 2001 From: neofright <68615872+neofright@users.noreply.github.com> Date: Fri, 3 Nov 2023 13:21:41 +0000 Subject: [PATCH 24/62] Update README.md Remove another unnecessary captialisation. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5fbb898..e0f3f9a 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Keep in sync all your users watched history between jellyfin and plex servers lo * \[ ] Match via provider ids * \[ ] Map usernames * \[ ] Use single login -* \[ ] One way/multi Way sync +* \[ ] One way/multi way sync * \[ ] Sync watched * \[ ] Sync in progress From 93bc94add59745cb568afea21f70bbf1b194a8eb Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 00:16:44 -0700 Subject: [PATCH 25/62] Pin to 3.11 due to 3.12 issues --- Dockerfile.alpine | 2 +- Dockerfile.slim | 2 +- requirements.txt | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 5b1a049..9437333 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -1,4 +1,4 @@ -FROM python:3-alpine +FROM python:3.11-alpine ENV DRYRUN 'True' ENV DEBUG 'True' diff --git a/Dockerfile.slim b/Dockerfile.slim index f667fe7..7cc3c0d 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -1,4 +1,4 @@ -FROM python:3-slim +FROM python:3.11-slim ENV DRYRUN 'True' ENV DEBUG 'True' diff --git a/requirements.txt b/requirements.txt index 027232b..332dbb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -PlexAPI==4.15.2 +PlexAPI==4.15.5 requests==2.31.0 python-dotenv==1.0.0 -aiohttp==3.8.5 +aiohttp==3.8.6 From 76ac264b25f19741b5bd9d1c02e0ac52274a4999 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 01:09:11 -0700 Subject: [PATCH 26/62] Add example baseurl/token to docker-compose --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e3783f1..9a061c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,10 +20,10 @@ services: - WHITELIST_LIBRARY_TYPE= - BLACKLIST_USERS= - WHITELIST_USERS= - - PLEX_BASEURL= - - PLEX_TOKEN= - - JELLYFIN_BASEURL= - - JELLYFIN_TOKEN= + - PLEX_BASEURL=https://localhost:32400 + - PLEX_TOKEN=plex_token + - JELLYFIN_BASEURL=http://localhost:8096 + - JELLYFIN_TOKEN=jelly_token - SSL_BYPASS=True - SYNC_FROM_PLEX_TO_JELLYFIN=True - SYNC_FROM_JELLYFIN_TO_PLEX=True From df13cef760e1107e7ef8c6565e411e2e04a206a3 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 01:12:08 -0700 Subject: [PATCH 27/62] Add mark list support --- .env.sample | 166 ++++++++++++++++++++++++----------------------- src/functions.py | 19 ++++++ src/jellyfin.py | 36 +++++++++- src/plex.py | 25 +++++++ 4 files changed, 165 insertions(+), 81 deletions(-) diff --git a/.env.sample b/.env.sample index 4a7bdb0..b90d0e1 100644 --- a/.env.sample +++ b/.env.sample @@ -1,80 +1,86 @@ -# Global Settings - -## Do not mark any shows/movies as played and instead just output to log if they would of been marked. -DRYRUN = "True" - -## Additional logging information -DEBUG = "False" - -## Debugging level, "info" is default, "debug" is more verbose -DEBUG_LEVEL = "info" - -## If set to true then the script will only run once and then exit -RUN_ONLY_ONCE = "False" - -## How often to run the script in seconds -SLEEP_DURATION = "3600" - -## Log file where all output will be written to -LOGFILE = "log.log" - -## Timeout for requests for jellyfin -REQUEST_TIMEOUT = 300 - -## Map usernames between servers in the event that they are different, order does not matter -## Comma separated for multiple options -#USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" } - -## Map libraries between servers in the event that they are different, order does not matter -## Comma separated for multiple options -#LIBRARY_MAPPING = { "Shows": "TV Shows", "Movie": "Movies" } - -## Blacklisting/Whitelisting libraries, library types such as Movies/TV Shows, and users. Mappings apply so if the mapping for the user or library exist then both will be excluded. -## Comma separated for multiple options -#BLACKLIST_LIBRARY = "" -#WHITELIST_LIBRARY = "" -#BLACKLIST_LIBRARY_TYPE = "" -#WHITELIST_LIBRARY_TYPE = "" -#BLACKLIST_USERS = "" -WHITELIST_USERS = "testuser1,testuser2" - - - -# Plex - -## Recommended to use token as it is faster to connect as it is direct to the server instead of going through the plex servers -## URL of the plex server, use hostname or IP address if the hostname is not resolving correctly -## Comma separated list for multiple servers -PLEX_BASEURL = "http://localhost:32400, https://nas:32400" - -## Plex token https://support.plex.tv/articles/204059436-finding-an-authentication-token-x-plex-token/ -## Comma separated list for multiple servers -PLEX_TOKEN = "SuperSecretToken, SuperSecretToken2" - -## If not using plex token then use username and password of the server admin along with the servername -## Comma separated for multiple options -#PLEX_USERNAME = "PlexUser, PlexUser2" -#PLEX_PASSWORD = "SuperSecret, SuperSecret2" -#PLEX_SERVERNAME = "Plex Server1, Plex Server2" - -## Skip hostname validation for ssl certificates. -## Set to True if running into ssl certificate errors -SSL_BYPASS = "False" - -## control the direction of syncing. e.g. SYNC_FROM_PLEX_TO_JELLYFIN set to true will cause the updates from plex -## to be updated in jellyfin. SYNC_FROM_PLEX_TO_PLEX set to true will sync updates between multiple plex servers -SYNC_FROM_PLEX_TO_JELLYFIN = "True" -SYNC_FROM_JELLYFIN_TO_PLEX = "True" -SYNC_FROM_PLEX_TO_PLEX = "True" -SYNC_FROM_JELLYFIN_TO_JELLYFIN = "True" - - -# Jellyfin - -## Jellyfin server URL, use hostname or IP address if the hostname is not resolving correctly -## Comma separated list for multiple servers -JELLYFIN_BASEURL = "http://localhost:8096, http://nas:8096" - -## Jellyfin api token, created manually by logging in to the jellyfin server admin dashboard and creating an api key -## Comma separated list for multiple servers -JELLYFIN_TOKEN = "SuperSecretToken, SuperSecretToken2" +# Global Settings + +## Do not mark any shows/movies as played and instead just output to log if they would of been marked. +DRYRUN = "True" + +## Additional logging information +DEBUG = "False" + +## Debugging level, "info" is default, "debug" is more verbose +DEBUG_LEVEL = "info" + +## If set to true then the script will only run once and then exit +RUN_ONLY_ONCE = "False" + +## How often to run the script in seconds +SLEEP_DURATION = "3600" + +## Log file where all output will be written to +LOGFILE = "log.log" + +## Mark file where all shows/movies that have been marked as played will be written to +MARK_FILE = "mark.log" + +## Timeout for requests for jellyfin +REQUEST_TIMEOUT = 300 + +## Max threads for processing +MAX_THREADS = 32 + +## Map usernames between servers in the event that they are different, order does not matter +## Comma separated for multiple options +#USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" } + +## Map libraries between servers in the event that they are different, order does not matter +## Comma separated for multiple options +#LIBRARY_MAPPING = { "Shows": "TV Shows", "Movie": "Movies" } + +## Blacklisting/Whitelisting libraries, library types such as Movies/TV Shows, and users. Mappings apply so if the mapping for the user or library exist then both will be excluded. +## Comma separated for multiple options +#BLACKLIST_LIBRARY = "" +#WHITELIST_LIBRARY = "" +#BLACKLIST_LIBRARY_TYPE = "" +#WHITELIST_LIBRARY_TYPE = "" +#BLACKLIST_USERS = "" +WHITELIST_USERS = "testuser1,testuser2" + + + +# Plex + +## Recommended to use token as it is faster to connect as it is direct to the server instead of going through the plex servers +## URL of the plex server, use hostname or IP address if the hostname is not resolving correctly +## Comma separated list for multiple servers +PLEX_BASEURL = "http://localhost:32400, https://nas:32400" + +## Plex token https://support.plex.tv/articles/204059436-finding-an-authentication-token-x-plex-token/ +## Comma separated list for multiple servers +PLEX_TOKEN = "SuperSecretToken, SuperSecretToken2" + +## If not using plex token then use username and password of the server admin along with the servername +## Comma separated for multiple options +#PLEX_USERNAME = "PlexUser, PlexUser2" +#PLEX_PASSWORD = "SuperSecret, SuperSecret2" +#PLEX_SERVERNAME = "Plex Server1, Plex Server2" + +## Skip hostname validation for ssl certificates. +## Set to True if running into ssl certificate errors +SSL_BYPASS = "False" + +## control the direction of syncing. e.g. SYNC_FROM_PLEX_TO_JELLYFIN set to true will cause the updates from plex +## to be updated in jellyfin. SYNC_FROM_PLEX_TO_PLEX set to true will sync updates between multiple plex servers +SYNC_FROM_PLEX_TO_JELLYFIN = "True" +SYNC_FROM_JELLYFIN_TO_PLEX = "True" +SYNC_FROM_PLEX_TO_PLEX = "True" +SYNC_FROM_JELLYFIN_TO_JELLYFIN = "True" + + +# Jellyfin + +## Jellyfin server URL, use hostname or IP address if the hostname is not resolving correctly +## Comma separated list for multiple servers +JELLYFIN_BASEURL = "http://localhost:8096, http://nas:8096" + +## Jellyfin api token, created manually by logging in to the jellyfin server admin dashboard and creating an api key +## Comma separated list for multiple servers +JELLYFIN_TOKEN = "SuperSecretToken, SuperSecretToken2" diff --git a/src/functions.py b/src/functions.py index 49dc67e..588d5e1 100644 --- a/src/functions.py +++ b/src/functions.py @@ -5,6 +5,7 @@ from dotenv import load_dotenv load_dotenv(override=True) logfile = os.getenv("LOGFILE", "log.log") +markfile = os.getenv("MARK_FILE", "mark.log") def logger(message: str, log_type=0): @@ -31,6 +32,24 @@ def logger(message: str, log_type=0): file.write(output + "\n") +def log_marked( + username: str, library: str, movie_show: str, episode: str = None, duration=None +): + if markfile is None: + return + + output = f"{username}/{library}/{movie_show}" + + if episode: + output += f"/{episode}" + + if duration: + output += f"/{duration}" + + file = open(f"{markfile}", "a", encoding="utf-8") + file.write(output + "\n") + + # Reimplementation of distutils.util.strtobool due to it being deprecated # Source: https://github.com/PostHog/posthog/blob/01e184c29d2c10c43166f1d40a334abbc3f99d8a/posthog/utils.py#L668 def str_to_bool(value: any) -> bool: diff --git a/src/jellyfin.py b/src/jellyfin.py index 50555f9..c3f26a9 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -2,7 +2,12 @@ import asyncio, aiohttp, traceback, os from math import floor from dotenv import load_dotenv -from src.functions import logger, search_mapping, contains_nested +from src.functions import ( + logger, + search_mapping, + contains_nested, + log_marked, +) from src.library import ( check_skip_logic, generate_library_guids_dict, @@ -642,6 +647,12 @@ class Jellyfin: ) else: logger(f"Dryrun {msg}", 0) + + log_marked( + user_name, + library, + jellyfin_video.get("Name"), + ) else: # TODO add support for partially watched movies msg = f"{jellyfin_video.get('Name')} as partially watched for {floor(movie_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" @@ -651,6 +662,13 @@ class Jellyfin: else: pass # logger(f"Dryrun {msg}", 0) + + log_marked( + user_name, + library, + jellyfin_video.get("Name"), + duration=floor(movie_status["time"] / 60_000), + ) else: logger( f"Jellyfin: Skipping movie {jellyfin_video.get('Name')} as it is not in mark list for {user_name}", @@ -811,18 +829,34 @@ class Jellyfin: ) else: logger(f"Dryrun {msg}", 0) + + log_marked( + user_name, + library, + jellyfin_episode.get("SeriesName"), + jellyfin_episode.get("Name"), + ) else: # TODO add support for partially watched episodes msg = ( f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode.get('Name')}" + f" as partially watched for {floor(episode_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" ) + """ if not dryrun: pass # logger(f"Marked {msg}", 0) else: pass # logger(f"Dryrun {msg}", 0) + + log_marked( + user_name, + library, + jellyfin_episode.get("SeriesName"), + jellyfin_episode.get('Name'), + duration=floor(episode_status["time"] / 60_000), + )""" else: logger( f"Jellyfin: Skipping episode {jellyfin_episode.get('Name')} as it is not in mark list for {user_name}", diff --git a/src/plex.py b/src/plex.py index df80adf..c9cc9ae 100644 --- a/src/plex.py +++ b/src/plex.py @@ -10,6 +10,7 @@ from src.functions import ( search_mapping, future_thread_executor, contains_nested, + log_marked, ) from src.library import ( check_skip_logic, @@ -301,6 +302,8 @@ def update_user_watched(user, user_plex, library, videos, dryrun): movies_search.markWatched() else: logger(f"Dryrun {msg}", 0) + + log_marked(user.title, library, movies_search.title, None, None) elif video_status["time"] > 60_000: msg = f"{movies_search.title} as partially watched for {floor(video_status['time'] / 60_000)} minutes for {user.title} in {library} for Plex" if not dryrun: @@ -308,6 +311,13 @@ def update_user_watched(user, user_plex, library, videos, dryrun): movies_search.updateProgress(video_status["time"]) else: logger(f"Dryrun {msg}", 0) + + log_marked( + user.title, + library, + movies_search.title, + duration=video_status["time"], + ) else: logger( f"Plex: Skipping movie {movies_search.title} as it is not in mark list for {user.title}", @@ -332,6 +342,13 @@ def update_user_watched(user, user_plex, library, videos, dryrun): episode_search.markWatched() else: logger(f"Dryrun {msg}", 0) + + log_marked( + user.title, + library, + show_search.title, + episode_search.title, + ) else: msg = f"{show_search.title} {episode_search.title} as partially watched for {floor(video_status['time'] / 60_000)} minutes for {user.title} in {library} for Plex" if not dryrun: @@ -339,6 +356,14 @@ def update_user_watched(user, user_plex, library, videos, dryrun): episode_search.updateProgress(video_status["time"]) else: logger(f"Dryrun {msg}", 0) + + log_marked( + user.title, + library, + show_search.title, + episode_search.title, + video_status["time"], + ) else: logger( f"Plex: Skipping episode {episode_search.title} as it is not in mark list for {user.title}", From b311bf2770d674f9734cbba03a6068591218742c Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 01:48:07 -0700 Subject: [PATCH 28/62] Add MARK/DRYRUN logger levels --- src/functions.py | 4 ++++ src/jellyfin.py | 27 ++++++++++++++------------- src/plex.py | 24 ++++++++++++------------ 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/functions.py b/src/functions.py index 588d5e1..94169b2 100644 --- a/src/functions.py +++ b/src/functions.py @@ -23,6 +23,10 @@ def logger(message: str, log_type=0): output = f"[DEBUG]: {output}" elif log_type == 4: output = f"[WARNING]: {output}" + elif log_type == 5: + output = f"[MARK]: {output}" + elif log_type == 6: + output = f"[DRYRUN]: {output}" else: output = None diff --git a/src/jellyfin.py b/src/jellyfin.py index c3f26a9..7d750f9 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -637,16 +637,16 @@ class Jellyfin: if movie_status: jellyfin_video_id = jellyfin_video["Id"] if movie_status["completed"]: - msg = f"{jellyfin_video.get('Name')} as watched for {user_name} in {library} for Jellyfin" + msg = f"Jellyfin: {jellyfin_video.get('Name')} as watched for {user_name} in {library}" if not dryrun: - logger(f"Marking {msg}", 0) + logger(msg, 5) await self.query( f"/Users/{user_id}/PlayedItems/{jellyfin_video_id}", "post", session, ) else: - logger(f"Dryrun {msg}", 0) + logger(msg, 6) log_marked( user_name, @@ -655,20 +655,21 @@ class Jellyfin: ) else: # TODO add support for partially watched movies - msg = f"{jellyfin_video.get('Name')} as partially watched for {floor(movie_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" + msg = f"Jellyfin: {jellyfin_video.get('Name')} as partially watched for {floor(movie_status['time'] / 60_000)} minutes for {user_name} in {library}" + """ if not dryrun: pass - # logger(f"Marked {msg}", 0) + # logger(msg, 5) else: pass - # logger(f"Dryrun {msg}", 0) + # logger(msg, 6) log_marked( user_name, library, jellyfin_video.get("Name"), duration=floor(movie_status["time"] / 60_000), - ) + )""" else: logger( f"Jellyfin: Skipping movie {jellyfin_video.get('Name')} as it is not in mark list for {user_name}", @@ -817,18 +818,18 @@ class Jellyfin: jellyfin_episode_id = jellyfin_episode["Id"] if episode_status["completed"]: msg = ( - f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode.get('Name')}" - + f" as watched for {user_name} in {library} for Jellyfin" + f"Jellyfin: {jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode.get('Name')}" + + f" as watched for {user_name} in {library}" ) if not dryrun: - logger(f"Marked {msg}", 0) + logger(msg, 5) await self.query( f"/Users/{user_id}/PlayedItems/{jellyfin_episode_id}", "post", session, ) else: - logger(f"Dryrun {msg}", 0) + logger(msg, 6) log_marked( user_name, @@ -839,8 +840,8 @@ class Jellyfin: else: # TODO add support for partially watched episodes msg = ( - f"{jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode.get('Name')}" - + f" as partially watched for {floor(episode_status['time'] / 60_000)} minutes for {user_name} in {library} for Jellyfin" + f"Jellyfin: {jellyfin_episode['SeriesName']} {jellyfin_episode['SeasonName']} Episode {jellyfin_episode.get('IndexNumber')} {jellyfin_episode.get('Name')}" + + f" as partially watched for {floor(episode_status['time'] / 60_000)} minutes for {user_name} in {library}" ) """ if not dryrun: diff --git a/src/plex.py b/src/plex.py index c9cc9ae..87b2b3a 100644 --- a/src/plex.py +++ b/src/plex.py @@ -296,21 +296,21 @@ def update_user_watched(user, user_plex, library, videos, dryrun): ) if video_status: if video_status["completed"]: - msg = f"{movies_search.title} as watched for {user.title} in {library} for Plex" + msg = f"Plex: {movies_search.title} as watched for {user.title} in {library}" if not dryrun: - logger(f"Marked {msg}", 0) + logger(msg, 5) movies_search.markWatched() else: - logger(f"Dryrun {msg}", 0) + logger(msg, 6) log_marked(user.title, library, movies_search.title, None, None) elif video_status["time"] > 60_000: - msg = f"{movies_search.title} as partially watched for {floor(video_status['time'] / 60_000)} minutes for {user.title} in {library} for Plex" + msg = f"Plex: {movies_search.title} as partially watched for {floor(video_status['time'] / 60_000)} minutes for {user.title} in {library}" if not dryrun: - logger(f"Marked {msg}", 0) + logger(msg, 5) movies_search.updateProgress(video_status["time"]) else: - logger(f"Dryrun {msg}", 0) + logger(msg, 6) log_marked( user.title, @@ -336,12 +336,12 @@ def update_user_watched(user, user_plex, library, videos, dryrun): ) if video_status: if video_status["completed"]: - msg = f"{show_search.title} {episode_search.title} as watched for {user.title} in {library} for Plex" + msg = f"Plex: {show_search.title} {episode_search.title} as watched for {user.title} in {library}" if not dryrun: - logger(f"Marked {msg}", 0) + logger(msg, 5) episode_search.markWatched() else: - logger(f"Dryrun {msg}", 0) + logger(msg, 6) log_marked( user.title, @@ -350,12 +350,12 @@ def update_user_watched(user, user_plex, library, videos, dryrun): episode_search.title, ) else: - msg = f"{show_search.title} {episode_search.title} as partially watched for {floor(video_status['time'] / 60_000)} minutes for {user.title} in {library} for Plex" + msg = f"Plex: {show_search.title} {episode_search.title} as partially watched for {floor(video_status['time'] / 60_000)} minutes for {user.title} in {library}" if not dryrun: - logger(f"Marked {msg}", 0) + logger(msg, 5) episode_search.updateProgress(video_status["time"]) else: - logger(f"Dryrun {msg}", 0) + logger(msg, 6) log_marked( user.title, From 1841b0dea6ddab55bd9b1f3cc84379abbf2581bf Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 01:59:18 -0700 Subject: [PATCH 29/62] Jellyfin: Remove headers append --- src/jellyfin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 7d750f9..faaa144 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -100,7 +100,7 @@ class Jellyfin: return await self.query(query, query_type, session, identifiers) results = None - headers = {"Accept": "application/json", "X-Emby-Token": self.token} + authorization = ( "MediaBrowser , " 'Client="other", ' @@ -108,8 +108,8 @@ class Jellyfin: 'DeviceId="script", ' 'Version="0.0.0"' ) - headers["X-Emby-Authorization"] = authorization - + headers = {"Accept": "application/json", "X-Emby-Token": self.token, "X-Emby-Authorization": authorization} + if query_type == "get": async with session.get( self.baseurl + query, headers=headers From a1b11ab03951a19fc400c72566c35e1d80699322 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 02:05:47 -0700 Subject: [PATCH 30/62] Add unraid to type --- .github/ISSUE_TEMPLATE/bug_report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 08d0947..853ff22 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -26,6 +26,7 @@ If applicable, add logs to help explain your problem ideally with DEBUG set to t **Type:** - [ ] Docker Compose - [ ] Docker +- [ ] Unraid - [ ] Native **Additional context** From dbea28e9c606207c4ec35106533a7783523ce871 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 02:28:40 -0700 Subject: [PATCH 31/62] Docker: Add RUN_ONLY_ONCE and MARKFILE --- Dockerfile.alpine | 2 ++ Dockerfile.slim | 3 +++ 2 files changed, 5 insertions(+) diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 9437333..22cc341 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -3,8 +3,10 @@ FROM python:3.11-alpine ENV DRYRUN 'True' ENV DEBUG 'True' ENV DEBUG_LEVEL 'INFO' +ENV RUN_ONLY_ONCE 'False' ENV SLEEP_DURATION '3600' ENV LOGFILE 'log.log' +ENV MARKFILE 'mark.log' ENV USER_MAPPING '' ENV LIBRARY_MAPPING '' diff --git a/Dockerfile.slim b/Dockerfile.slim index 7cc3c0d..ee524fe 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -3,8 +3,10 @@ FROM python:3.11-slim ENV DRYRUN 'True' ENV DEBUG 'True' ENV DEBUG_LEVEL 'INFO' +ENV RUN_ONLY_ONCE 'False' ENV SLEEP_DURATION '3600' ENV LOGFILE 'log.log' +ENV MARKFILE 'mark.log' ENV USER_MAPPING '' ENV LIBRARY_MAPPING '' @@ -33,6 +35,7 @@ ENV WHITELIST_USERS '' WORKDIR /app COPY ./requirements.txt ./ + RUN pip install --no-cache-dir -r requirements.txt COPY . . From b8273f50c2f02ee063acbc6babc0508149f5a0fb Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 02:30:11 -0700 Subject: [PATCH 32/62] MARKFILE match LOGFILE --- src/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functions.py b/src/functions.py index 94169b2..b7ae693 100644 --- a/src/functions.py +++ b/src/functions.py @@ -5,7 +5,7 @@ from dotenv import load_dotenv load_dotenv(override=True) logfile = os.getenv("LOGFILE", "log.log") -markfile = os.getenv("MARK_FILE", "mark.log") +markfile = os.getenv("MARKFILE", "mark.log") def logger(message: str, log_type=0): From 3dc50fff95a35cadf85ef053ba85446d9a4b9a4e Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 02:49:14 -0700 Subject: [PATCH 33/62] Docker-compose: Add markfile. Add user mapping ex --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9a061c8..c3f0703 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,8 @@ services: - RUN_ONLY_ONCE=False - SLEEP_DURATION=3600 - LOGFILE=/tmp/log.log - - USER_MAPPING= + - MARKFILE=/tmp/mark.log + - USER_MAPPING={"user1":"user2"} - LIBRARY_MAPPING={"TV Shows":"Shows"} - BLACKLIST_LIBRARY= - WHITELIST_LIBRARY= From 0c1579bae77a328d02c45cdd7563ea0e14d963f4 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 03:36:10 -0700 Subject: [PATCH 34/62] Use non root for containers --- Dockerfile.alpine | 12 ++++++++++-- Dockerfile.slim | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 22cc341..03f432a 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -32,12 +32,20 @@ ENV WHITELIST_LIBRARY_TYPE '' ENV BLACKLIST_USERS '' ENV WHITELIST_USERS '' + +RUN addgroup --system jellyplex_user && \ + adduser --system --no-create-home jellyplex_user --ingroup jellyplex_user && \ + mkdir -p /app && \ + chown -R jellyplex_user:jellyplex_user /app + WORKDIR /app -COPY ./requirements.txt ./ +COPY --chown=jellyplex_user:jellyplex_user ./requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt -COPY . . +COPY --chown=jellyplex_user:jellyplex_user . . + +USER jellyplex_user CMD ["python", "-u", "main.py"] diff --git a/Dockerfile.slim b/Dockerfile.slim index ee524fe..59c9a02 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -32,12 +32,20 @@ ENV WHITELIST_LIBRARY_TYPE '' ENV BLACKLIST_USERS '' ENV WHITELIST_USERS '' + +RUN addgroup --system jellyplex_user && \ + adduser --system --no-create-home jellyplex_user --ingroup jellyplex_user && \ + mkdir -p /app && \ + chown -R jellyplex_user:jellyplex_user /app + WORKDIR /app -COPY ./requirements.txt ./ +COPY --chown=jellyplex_user:jellyplex_user ./requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt -COPY . . +COPY --chown=jellyplex_user:jellyplex_user . . + +USER jellyplex_user CMD ["python", "-u", "main.py"] From 8e23847c791185f9afa3abc74276733b4880b4c3 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 03:39:29 -0700 Subject: [PATCH 35/62] README: Change configuration to point to .env.sample --- README.md | 84 +------------------------------------------------------ 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/README.md b/README.md index e0f3f9a..8b6cfdd 100644 --- a/README.md +++ b/README.md @@ -42,89 +42,7 @@ Keep in sync all your users watched history between jellyfin and plex servers lo ## Configuration -```bash -# Global Settings - -## Do not mark any shows/movies as played and instead just output to log if they would of been marked. -DRYRUN = "True" - -## Additional logging information -DEBUG = "False" - -## Debugging level, "info" is default, "debug" is more verbose -DEBUG_LEVEL = "info" - -## If set to true then the script will only run once and then exit -RUN_ONLY_ONCE = "False" - -## How often to run the script in seconds -SLEEP_DURATION = "3600" - -## Log file where all output will be written to -LOGFILE = "log.log" - -## Timeout for requests for jellyfin -REQUEST_TIMEOUT = 300 - -## Map usernames between servers in the event that they are different, order does not matter -## Comma separated for multiple options -USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" } - -## Map libraries between servers in the even that they are different, order does not matter -## Comma separated for multiple options -LIBRARY_MAPPING = { "Shows": "TV Shows", "Movie": "Movies" } - -## Blacklisting/Whitelisting libraries, library types such as Movies/TV Shows, and users. Mappings apply so if the mapping for the user or library exist then both will be excluded. -## Comma separated for multiple options -BLACKLIST_LIBRARY = "" -WHITELIST_LIBRARY = "" -BLACKLIST_LIBRARY_TYPE = "" -WHITELIST_LIBRARY_TYPE = "" -BLACKLIST_USERS = "" -WHITELIST_USERS = "testuser1,testuser2" - - - -# Plex - -## Recommended to use token as it is faster to connect as it is direct to the server instead of going through the plex servers -## URL of the plex server, use hostname or IP address if the hostname is not resolving correctly -## Comma separated list for multiple servers -PLEX_BASEURL = "http://localhost:32400, https://nas:32400" - -## Plex token https://support.plex.tv/articles/204059436-finding-an-authentication-token-x-plex-token/ -## Comma separated list for multiple servers -PLEX_TOKEN = "SuperSecretToken, SuperSecretToken2" - -## If not using plex token then use username and password of the server admin along with the servername -## Comma separated for multiple options -#PLEX_USERNAME = "PlexUser, PlexUser2" -#PLEX_PASSWORD = "SuperSecret, SuperSecret2" -#PLEX_SERVERNAME = "Plex Server1, Plex Server2" - -## Skip hostname validation for ssl certificates. -## Set to True if running into ssl certificate errors -SSL_BYPASS = "False" - - -## control the direction of syncing. e.g. SYNC_FROM_PLEX_TO_JELLYFIN set to true will cause the updates from plex -## to be updated in jellyfin. SYNC_FROM_PLEX_TO_PLEX set to true will sync updates between multiple plex servers -SYNC_FROM_PLEX_TO_JELLYFIN = "True" -SYNC_FROM_JELLYFIN_TO_PLEX = "True" -SYNC_FROM_PLEX_TO_PLEX = "True" -SYNC_FROM_JELLYFIN_TO_JELLYFIN = "True" - - -# Jellyfin - -## Jellyfin server URL, use hostname or IP address if the hostname is not resolving correctly -## Comma separated list for multiple servers -JELLYFIN_BASEURL = "http://localhost:8096, http://nas:8096" - -## Jellyfin api token, created manually by logging in to the jellyfin server admin dashboard and creating an api key -## Comma separated list for multiple servers -JELLYFIN_TOKEN = "SuperSecretToken, SuperSecretToken2" -``` +Full list of configuration options can be found in the [.env.sample](.env.sample) ## Installation From d5b6859bf8a5a138a1297b9eb2fa7fd681d24806 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Mon, 13 Nov 2023 03:48:05 -0700 Subject: [PATCH 36/62] Action: Add default variant --- .github/workflows/ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dcee0f..a5c5bdb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: docker: runs-on: ubuntu-latest needs: pytest + env: + DEFAULT_VARIANT: alpine strategy: fail-fast: false matrix: @@ -45,14 +47,19 @@ jobs: # Do not push to ghcr.io on PRs due to permission issues ghcr.io/${{ github.repository }},enable=${{ github.event_name != 'pull_request' }} tags: | - type=raw,value=latest,enable=${{ matrix.variant == 'alpine' && github.ref_name == github.event.repository.default_branch }} - type=raw,value=dev,enable=${{ matrix.variant == 'alpine' && github.ref_name == 'dev' }} + type=raw,value=latest,enable=${{ matrix.variant == env.DEFAULT_VARIANT && github.ref_name == github.event.repository.default_branch }} + type=raw,value=dev,enable=${{ matrix.variant == env.DEFAULT_VARIANT && github.ref_name == 'dev' }} type=raw,value=latest,suffix=-${{ matrix.variant }},enable={{ is_default_branch }} type=ref,event=branch,suffix=-${{ matrix.variant }} + type=ref,event=branch,enable=${{ matrix.variant == env.DEFAULT_VARIANT }} type=ref,event=pr,suffix=-${{ matrix.variant }} + type=ref,event=pr,enable=${{ matrix.variant == env.DEFAULT_VARIANT }} type=semver,pattern={{ version }},suffix=-${{ matrix.variant }} + type=semver,pattern={{ version }},enable=${{ matrix.variant == env.DEFAULT_VARIANT }} type=semver,pattern={{ major }}.{{ minor }},suffix=-${{ matrix.variant }} + type=semver,pattern={{ major }}.{{ minor }},enable=${{ matrix.variant == env.DEFAULT_VARIANT }} type=sha,suffix=-${{ matrix.variant }} + type=sha,enable=${{ matrix.variant == env.DEFAULT_VARIANT }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 From 5472baab5162295d07ef7e229d2024b72f5abd90 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Wed, 6 Dec 2023 12:04:32 -0700 Subject: [PATCH 37/62] Action: Limit ghcr push to luigi311 Signed-off-by: Luigi311 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5c5bdb..5b9be46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,8 +44,8 @@ jobs: with: images: | ${{ secrets.DOCKER_USERNAME }}/jellyplex-watched,enable=${{ secrets.DOCKER_USERNAME != '' }} - # Do not push to ghcr.io on PRs due to permission issues - ghcr.io/${{ github.repository }},enable=${{ github.event_name != 'pull_request' }} + # Do not push to ghcr.io on PRs due to permission issues, only push if the owner is luigi311 so it doesnt fail on forks + ghcr.io/${{ github.repository }},enable=${{ github.event_name != 'pull_request' && github.repository_owner == 'luigi311'}} tags: | type=raw,value=latest,enable=${{ matrix.variant == env.DEFAULT_VARIANT && github.ref_name == github.event.repository.default_branch }} type=raw,value=dev,enable=${{ matrix.variant == env.DEFAULT_VARIANT && github.ref_name == 'dev' }} From 6b7f8b04e6568eb983c9b716055617f0d5f80f17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 01:04:37 +0000 Subject: [PATCH 38/62] Bump aiohttp from 3.8.6 to 3.9.0 Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.8.6 to 3.9.0. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.8.6...v3.9.0) --- updated-dependencies: - dependency-name: aiohttp dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 332dbb6..de8801d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ PlexAPI==4.15.5 requests==2.31.0 python-dotenv==1.0.0 -aiohttp==3.8.6 +aiohttp==3.9.0 From c91ba0b1b32cedabdfaf71527a85d0498b211422 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Wed, 6 Dec 2023 12:54:33 -0700 Subject: [PATCH 39/62] Action: Add test Spins up jellyfin and plex containers to test against --- .github/workflows/ci.yml | 43 +++++++++++++++++++- src/jellyfin.py | 10 +++-- src/main.py | 45 +++++++++++++-------- test/ci.env | 86 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 test/ci.env diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b9be46..dad43c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,9 +21,50 @@ jobs: - name: "Run tests" run: pytest -vvv + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: "Install dependencies" + run: | + pip install -r requirements.txt + sudo apt update && sudo apt install -y docker-compose + + - name: "Checkout JellyPlex-Watched-CI" + uses: actions/checkout@v2 + with: + repository: luigi311/JellyPlex-Watched-CI + path: JellyPlex-Watched-CI + + - name: "Start containers" + run: | + export PGID=$(id -g) + export PUID=$(id -u) + + sudo chown -R $PUID:$PGID JellyPlex-Watched-CI + + docker-compose -f JellyPlex-Watched-CI/plex/docker-compose.yml up -d + docker-compose -f JellyPlex-Watched-CI/jellyfin/docker-compose.yml up -d + # Wait for containers to start + sleep 15 + docker-compose -f JellyPlex-Watched-CI/plex/docker-compose.yml logs + docker-compose -f JellyPlex-Watched-CI/jellyfin/docker-compose.yml logs + + - name: "Run tests" + run: | + # Move test/.env to root + mv test/ci.env .env + # Run script + python main.py + + cat mark.log + docker: runs-on: ubuntu-latest - needs: pytest + needs: + - pytest + - test env: DEFAULT_VARIANT: alpine strategy: diff --git a/src/jellyfin.py b/src/jellyfin.py index faaa144..41ddbc9 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -100,7 +100,7 @@ class Jellyfin: return await self.query(query, query_type, session, identifiers) results = None - + authorization = ( "MediaBrowser , " 'Client="other", ' @@ -108,8 +108,12 @@ class Jellyfin: 'DeviceId="script", ' 'Version="0.0.0"' ) - headers = {"Accept": "application/json", "X-Emby-Token": self.token, "X-Emby-Authorization": authorization} - + headers = { + "Accept": "application/json", + "X-Emby-Token": self.token, + "X-Emby-Authorization": authorization, + } + if query_type == "get": async with session.get( self.baseurl + query, headers=headers diff --git a/src/main.py b/src/main.py index 562dc46..05b5c96 100644 --- a/src/main.py +++ b/src/main.py @@ -83,17 +83,21 @@ def generate_server_connections(): ) for i, url in enumerate(plex_baseurl): + server = Plex( + baseurl=url.strip(), + token=plex_token[i].strip(), + username=None, + password=None, + servername=None, + ssl_bypass=ssl_bypass, + ) + + logger(f"Plex Server {i} info: {server.info()}", 3) + servers.append( ( "plex", - Plex( - baseurl=url.strip(), - token=plex_token[i].strip(), - username=None, - password=None, - servername=None, - ssl_bypass=ssl_bypass, - ), + server, ) ) @@ -110,17 +114,20 @@ def generate_server_connections(): ) for i, username in enumerate(plex_username): + server = Plex( + baseurl=None, + token=None, + username=username.strip(), + password=plex_password[i].strip(), + servername=plex_servername[i].strip(), + ssl_bypass=ssl_bypass, + ) + + logger(f"Plex Server {i} info: {server.info()}", 3) servers.append( ( "plex", - Plex( - baseurl=None, - token=None, - username=username.strip(), - password=plex_password[i].strip(), - servername=plex_servername[i].strip(), - ssl_bypass=ssl_bypass, - ), + server, ) ) @@ -140,10 +147,14 @@ def generate_server_connections(): baseurl = baseurl.strip() if baseurl[-1] == "/": baseurl = baseurl[:-1] + + server = Jellyfin(baseurl=baseurl, token=jellyfin_token[i].strip()) + + logger(f"Jellyfin Server {i} info: {server.info()}", 3) servers.append( ( "jellyfin", - Jellyfin(baseurl=baseurl, token=jellyfin_token[i].strip()), + server, ) ) diff --git a/test/ci.env b/test/ci.env new file mode 100644 index 0000000..ba9bd28 --- /dev/null +++ b/test/ci.env @@ -0,0 +1,86 @@ +# Global Settings + +## Do not mark any shows/movies as played and instead just output to log if they would of been marked. +DRYRUN = "False" + +## Additional logging information +DEBUG = "True" + +## Debugging level, "info" is default, "debug" is more verbose +DEBUG_LEVEL = "debug" + +## If set to true then the script will only run once and then exit +RUN_ONLY_ONCE = "True" + +## How often to run the script in seconds +SLEEP_DURATION = 10 + +## Log file where all output will be written to +LOG_FILE = "log.log" + +## Mark file where all shows/movies that have been marked as played will be written to +MARK_FILE = "mark.log" + +## Timeout for requests for jellyfin +REQUEST_TIMEOUT = 300 + +## Max threads for processing +MAX_THREADS = 2 + +## Map usernames between servers in the event that they are different, order does not matter +## Comma seperated for multiple options +USER_MAPPING = {"JellyUser":"jellyplex_watched"} + +## Map libraries between servers in the even that they are different, order does not matter +## Comma seperated for multiple options +LIBRARY_MAPPING = { "Shows": "TV Shows" } + + +## Blacklisting/Whitelisting libraries, library types such as Movies/TV Shows, and users. Mappings apply so if the mapping for the user or library exist then both will be excluded. +## Comma seperated for multiple options +#BLACKLIST_LIBRARY = "" +#WHITELIST_LIBRARY = "Movies" +#BLACKLIST_LIBRARY_TYPE = "Series" +#WHITELIST_LIBRARY_TYPE = "Movies, movie" +#BLACKLIST_USERS = "" +WHITELIST_USERS = "jellyplex_watched" + + + +# Plex + +## Recommended to use token as it is faster to connect as it is direct to the server instead of going through the plex servers +## URL of the plex server, use hostname or IP address if the hostname is not resolving correctly +## Comma seperated list for multiple servers +PLEX_BASEURL = "https://localhost:32400" + +## Plex token https://support.plex.tv/articles/204059436-finding-an-authentication-token-x-plex-token/ +## Comma seperated list for multiple servers +PLEX_TOKEN = "mVaCzSyd78uoWkCBzZ_Y" + +## If not using plex token then use username and password of the server admin along with the servername +## Comma seperated for multiple options +#PLEX_USERNAME = "PlexUser, PlexUser2" +#PLEX_PASSWORD = "SuperSecret, SuperSecret2" +#PLEX_SERVERNAME = "Plex Server1, Plex Server2" + +## Skip hostname validation for ssl certificates. +## Set to True if running into ssl certificate errors +SSL_BYPASS = "True" + +## control the direction of syncing. e.g. SYNC_FROM_PLEX_TO_JELLYFIN set to true will cause the updates from plex +## to be updated in jellyfin. SYNC_FROM_PLEX_TO_PLEX set to true will sync updates between multiple plex servers +SYNC_FROM_PLEX_TO_JELLYFIN = "True" +SYNC_FROM_JELLYFIN_TO_PLEX = "True" +SYNC_FROM_PLEX_TO_PLEX = "True" +SYNC_FROM_JELLYFIN_TO_JELLYFIN = "True" + +# Jellyfin + +## Jellyfin server URL, use hostname or IP address if the hostname is not resolving correctly +## Comma seperated list for multiple servers +JELLYFIN_BASEURL = "http://localhost:8096" + +## Jellyfin api token, created manually by logging in to the jellyfin server admin dashboard and creating an api key +## Comma seperated list for multiple servers +JELLYFIN_TOKEN = "d773c4db3ecc4b028fc0904d9694804c" From 4b02aae88915bd2869a19bb436c311a4516285a7 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Fri, 8 Dec 2023 11:57:30 -0700 Subject: [PATCH 40/62] Show average time on exit Signed-off-by: Luigi311 --- src/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.py b/src/main.py index 05b5c96..fa99a63 100644 --- a/src/main.py +++ b/src/main.py @@ -422,5 +422,7 @@ def main(): sleep(sleep_duration) except KeyboardInterrupt: + if len(times) > 0: + logger(f"Average time: {sum(times) / len(times)}", 0) logger("Exiting", log_type=0) os._exit(0) From 2e0ec9aa38bfa50ca7c537507b5158f76762d53a Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Fri, 8 Dec 2023 14:16:43 -0700 Subject: [PATCH 41/62] Plex: Use updateTimeline instead of updateProgress Not all accounts have access to updateProgress, so use updateTimeline instead Signed-off-by: Luigi311 --- src/plex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plex.py b/src/plex.py index 87b2b3a..01049c2 100644 --- a/src/plex.py +++ b/src/plex.py @@ -308,7 +308,7 @@ def update_user_watched(user, user_plex, library, videos, dryrun): msg = f"Plex: {movies_search.title} as partially watched for {floor(video_status['time'] / 60_000)} minutes for {user.title} in {library}" if not dryrun: logger(msg, 5) - movies_search.updateProgress(video_status["time"]) + movies_search.updateTimeline(video_status["time"]) else: logger(msg, 6) @@ -353,7 +353,7 @@ def update_user_watched(user, user_plex, library, videos, dryrun): msg = f"Plex: {show_search.title} {episode_search.title} as partially watched for {floor(video_status['time'] / 60_000)} minutes for {user.title} in {library}" if not dryrun: logger(msg, 5) - episode_search.updateProgress(video_status["time"]) + episode_search.updateTimeline(video_status["time"]) else: logger(msg, 6) From 03cad668aa84a2095712f5189d6cdbd8766e5a19 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sat, 9 Dec 2023 18:37:23 -0700 Subject: [PATCH 42/62] README: Add troubleshooting/Issues Signed-off-by: Luigi311 --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 8b6cfdd..476f933 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,15 @@ Full list of configuration options can be found in the [.env.sample](.env.sample docker run --rm -it -v "$(pwd)/.env:/app/.env" luigi311/jellyplex-watched:latest ``` +## Troubleshooting/Issues + +* Jellyfin + * Attempt to decode JSON with unexpected mimetype, make sure you enable remote access or add your docker subnet to lan networks in jellyfin settings + +* Configuration + * Do not use quotes around variables in docker compose + + ## Contributing I am open to receiving pull requests. If you are submitting a pull request, please make sure run it locally for a day or two to make sure it is working as expected and stable. Make all pull requests against the dev branch and nothing will be merged into the main without going through the lower branches. From b8b627be1a4041b0e6927b47141ff84b5e78314f Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sun, 10 Dec 2023 10:41:59 -0700 Subject: [PATCH 43/62] Use season number instead of season name Using season name is not reliable as it can vary between servers and can be overridden by the user. Signed-off-by: Luigi311 --- src/jellyfin.py | 11 ++++++----- src/plex.py | 12 ++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/jellyfin.py b/src/jellyfin.py index 41ddbc9..9ab4942 100644 --- a/src/jellyfin.py +++ b/src/jellyfin.py @@ -331,8 +331,9 @@ class Jellyfin: if len(seasons["Items"]) > 0: for season in seasons["Items"]: season_identifiers = dict(seasons["Identifiers"]) - season_identifiers["season_id"] = season["Id"] - season_identifiers["season_name"] = season["Name"] + season_identifiers["season_index"] = season[ + "IndexNumber" + ] watched_task = asyncio.ensure_future( self.query( f"/Shows/{season_identifiers['show_id']}/Episodes" @@ -390,18 +391,18 @@ class Jellyfin: ] = {} if ( - season_dict["Identifiers"]["season_name"] + season_dict["Identifiers"]["season_index"] not in user_watched[user_name][library_title][ season_dict["Identifiers"]["show_guids"] ] ): user_watched[user_name][library_title][ season_dict["Identifiers"]["show_guids"] - ][season_dict["Identifiers"]["season_name"]] = [] + ][season_dict["Identifiers"]["season_index"]] = [] user_watched[user_name][library_title][ season_dict["Identifiers"]["show_guids"] - ][season_dict["Identifiers"]["season_name"]] = season_dict[ + ][season_dict["Identifiers"]["season_index"]] = season_dict[ "Episodes" ] logger( diff --git a/src/plex.py b/src/plex.py index 01049c2..ba4fb41 100644 --- a/src/plex.py +++ b/src/plex.py @@ -105,17 +105,17 @@ def get_user_library_watched_show(show): for episode in show.episodes(): if episode in watched: - if episode.parentTitle not in episode_guids: - episode_guids[episode.parentTitle] = [] + if episode.parentIndex not in episode_guids: + episode_guids[episode.parentIndex] = [] - episode_guids[episode.parentTitle].append( + episode_guids[episode.parentIndex].append( get_episode_guids(episode, show, completed=True) ) elif episode.viewOffset > 0: - if episode.parentTitle not in episode_guids: - episode_guids[episode.parentTitle] = [] + if episode.parentIndex not in episode_guids: + episode_guids[episode.parentIndex] = [] - episode_guids[episode.parentTitle].append( + episode_guids[episode.parentIndex].append( get_episode_guids(episode, show, completed=False) ) From bdf6476689b7a08d646bdf97496a0e3d4b9e76cd Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sun, 10 Dec 2023 12:20:53 -0700 Subject: [PATCH 44/62] Watched: combine_watched_dicts check types Signed-off-by: Luigi311 --- src/watched.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/watched.py b/src/watched.py index 2498eae..7890ca0 100644 --- a/src/watched.py +++ b/src/watched.py @@ -6,11 +6,17 @@ from src.library import generate_library_guids_dict def combine_watched_dicts(dicts: list): + # Ensure that the input is a list of dictionaries + if not all(isinstance(d, dict) for d in dicts): + raise ValueError("Input must be a list of dictionaries") + combined_dict = {} + for single_dict in dicts: for key, value in single_dict.items(): if key not in combined_dict: combined_dict[key] = {} + for subkey, subvalue in value.items(): if subkey in combined_dict[key]: # If the subkey already exists in the combined dictionary, From 9739b2771812c37caaafc21e29e269429bb1c323 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sun, 10 Dec 2023 22:03:54 -0700 Subject: [PATCH 45/62] Remove failed message from show/episode/movie dict Signed-off-by: Luigi311 --- src/library.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/library.py b/src/library.py index 6754f14..6112539 100644 --- a/src/library.py +++ b/src/library.py @@ -158,7 +158,7 @@ def show_title_dict(user_list: dict): return show_output_dict except Exception: - logger("Generating show_output_dict failed, skipping", 1) + logger("Skipping show_output_dict ", 1) return {} @@ -213,7 +213,7 @@ def episode_title_dict(user_list: dict): return episode_output_dict except Exception: - logger("Generating episode_output_dict failed, skipping", 1) + logger("Skipping episode_output_dict", 1) return {} @@ -246,7 +246,7 @@ def movies_title_dict(user_list: dict): return movies_output_dict except Exception: - logger("Generating movies_output_dict failed, skipping", 1) + logger("Skipping movies_output_dict failed", 1) return {} From a2b802a5deeef68f477ec506eb40c0fd418f3e13 Mon Sep 17 00:00:00 2001 From: Jan Willhaus Date: Thu, 14 Dec 2023 09:04:38 +0100 Subject: [PATCH 46/62] Add tini for sigterm handling --- Dockerfile.alpine | 4 +++- Dockerfile.slim | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 03f432a..afa5d6e 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -33,7 +33,8 @@ ENV BLACKLIST_USERS '' ENV WHITELIST_USERS '' -RUN addgroup --system jellyplex_user && \ +RUN apk add --no-cache tini && \ + addgroup --system jellyplex_user && \ adduser --system --no-create-home jellyplex_user --ingroup jellyplex_user && \ mkdir -p /app && \ chown -R jellyplex_user:jellyplex_user /app @@ -48,4 +49,5 @@ COPY --chown=jellyplex_user:jellyplex_user . . USER jellyplex_user +ENTRYPOINT ["/sbin/tini", "--"] CMD ["python", "-u", "main.py"] diff --git a/Dockerfile.slim b/Dockerfile.slim index 59c9a02..b15d237 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -33,7 +33,11 @@ ENV BLACKLIST_USERS '' ENV WHITELIST_USERS '' -RUN addgroup --system jellyplex_user && \ +RUN apt-get update && \ + apt-get install tini --yes --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + addgroup --system jellyplex_user && \ adduser --system --no-create-home jellyplex_user --ingroup jellyplex_user && \ mkdir -p /app && \ chown -R jellyplex_user:jellyplex_user /app @@ -48,4 +52,5 @@ COPY --chown=jellyplex_user:jellyplex_user . . USER jellyplex_user +ENTRYPOINT ["/bin/tini", "--"] CMD ["python", "-u", "main.py"] From 4e25ae5539bc5d79ef899d0a1c1d83e3f8382d32 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 2 Jan 2024 18:10:56 -0700 Subject: [PATCH 47/62] CI: Validate mark log --- .github/workflows/ci.yml | 1 + test/validate_ci_marklog.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 test/validate_ci_marklog.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dad43c3..b2d0f6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: python main.py cat mark.log + python test/validate_ci_marklog.py docker: runs-on: ubuntu-latest diff --git a/test/validate_ci_marklog.py b/test/validate_ci_marklog.py new file mode 100644 index 0000000..34099f6 --- /dev/null +++ b/test/validate_ci_marklog.py @@ -0,0 +1,35 @@ +# Check the mark.log file that is generated by the CI to make sure it contains the expected values + +import os + +def read_marklog(): + marklog = os.path.join(os.getcwd(), "mark.log") + with open(marklog, "r") as f: + lines = f.readlines() + return lines + +def check_marklog(lines, expected_values): + for line in lines: + # Remove the newline character + line = line.strip() + if line not in expected_values: + print("Line not found in marklog: " + line) + return False + return True + +def main(): + expected_values = [ + "jellyplex_watched/TV Shows/Blender Shorts/Episode 2" + , "JellyUser/Movies/Big Buck Bunny" + , "JellyUser/Shows/Blender Shorts/S01E01" + ] + + lines = read_marklog() + if not check_marklog(lines, expected_values): + print("Marklog did not contain the expected values") + exit(1) + else: + print("Marklog contained the expected values") + +if __name__ == "__main__": + main() From 5014748ee149f75f83aa4e76bbf9f6efcc7a6b22 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 2 Jan 2024 18:15:20 -0700 Subject: [PATCH 48/62] CI: Speedup start containers --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2d0f6e..e0752c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,10 +44,15 @@ jobs: sudo chown -R $PUID:$PGID JellyPlex-Watched-CI + docker pull lscr.io/linuxserver/plex & + docker pull lscr.io/linuxserver/jellyfin & + + wait + docker-compose -f JellyPlex-Watched-CI/plex/docker-compose.yml up -d docker-compose -f JellyPlex-Watched-CI/jellyfin/docker-compose.yml up -d # Wait for containers to start - sleep 15 + sleep 5 docker-compose -f JellyPlex-Watched-CI/plex/docker-compose.yml logs docker-compose -f JellyPlex-Watched-CI/jellyfin/docker-compose.yml logs From ba480d2cb76e928bf1a20c1cb60ca1bf8ac10c11 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 2 Jan 2024 20:59:58 -0700 Subject: [PATCH 49/62] CI: Add workflow dispatch --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0752c1..7df666a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,6 @@ name: CI on: + workflow_dispatch: push: paths-ignore: - .gitignore From de9180a124fa597fa30a802ddd50369bd2f19426 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 2 Jan 2024 23:43:59 -0700 Subject: [PATCH 50/62] Handle episode names are not unique --- .vscode/launch.json | 11 + src/functions.py | 7 + src/library.py | 16 ++ src/watched.py | 61 ++++- test/test_library.py | 10 + test/test_watched.py | 469 +++++++++++++++++++++++++----------- test/validate_ci_marklog.py | 16 +- 7 files changed, 449 insertions(+), 141 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 5092265..60cea81 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,6 +11,17 @@ "program": "main.py", "console": "integratedTerminal", "justMyCode": true + }, + { + "name": "Pytest", + "type": "python", + "request": "launch", + "module": "pytest", + "args": [ + "-vv" + ], + "console": "integratedTerminal", + "justMyCode": true } ] } diff --git a/src/functions.py b/src/functions.py index b7ae693..c1c160b 100644 --- a/src/functions.py +++ b/src/functions.py @@ -64,9 +64,16 @@ def str_to_bool(value: any) -> bool: # Search for nested element in list def contains_nested(element, lst): + if lst is None: + return None + for i, item in enumerate(lst): + if item is None: + continue if element in item: return i + elif element == item: + return i return None diff --git a/src/library.py b/src/library.py index 6112539..c33c962 100644 --- a/src/library.py +++ b/src/library.py @@ -168,12 +168,28 @@ def episode_title_dict(user_list: dict): episode_output_dict["completed"] = [] episode_output_dict["time"] = [] episode_output_dict["locations"] = [] + episode_output_dict["show"] = [] + episode_output_dict["season"] = [] episode_counter = 0 # Initialize a counter for the current episode position # Iterate through the shows, seasons, and episodes in user_list for show in user_list: for season in user_list[show]: for episode in user_list[show][season]: + # Add the show title to the episode_output_dict if it doesn't exist + if "show" not in episode_output_dict: + episode_output_dict["show"] = [None] * episode_counter + + # Add the season number to the episode_output_dict if it doesn't exist + if "season" not in episode_output_dict: + episode_output_dict["season"] = [None] * episode_counter + + # Add the show title to the episode_output_dict + episode_output_dict["show"].append(dict(show)) + + # Add the season number to the episode_output_dict + episode_output_dict["season"].append(season) + # Iterate through the keys and values in each episode for episode_key, episode_value in episode.items(): # If the key is not "status", add the key to episode_output_dict if it doesn't exist diff --git a/src/watched.py b/src/watched.py index 7890ca0..8fa60fa 100644 --- a/src/watched.py +++ b/src/watched.py @@ -97,7 +97,7 @@ def cleanup_watched( continue ( - _, + show_watched_list_2_keys_dict, episode_watched_list_2_keys_dict, movies_watched_list_2_keys_dict, ) = generate_library_guids_dict(watched_list_2[user_2][library_2]) @@ -123,11 +123,18 @@ def cleanup_watched( show_key_dict = dict(show_key_1) for season in watched_list_1[user_1][library_1][show_key_1]: + # Filter the episode_watched_list_2_keys_dict dictionary to handle cases + # where episode location names are not unique such as S01E01.mkv + filtered_episode_watched_list_2_keys_dict = ( + filter_episode_watched_list_2_keys_dict( + episode_watched_list_2_keys_dict, show_key_dict, season + ) + ) for episode in watched_list_1[user_1][library_1][show_key_1][ season ]: episode_index = get_episode_index_in_dict( - episode, episode_watched_list_2_keys_dict + episode, filtered_episode_watched_list_2_keys_dict ) if episode_index is not None: if check_remove_entry( @@ -223,6 +230,56 @@ def get_movie_index_in_dict(movie, movies_watched_list_2_keys_dict): return None +def filter_episode_watched_list_2_keys_dict( + episode_watched_list_2_keys_dict, show_key_dict, season +): + # Filter the episode_watched_list_2_keys_dict dictionary to only include values for the correct show and season + filtered_episode_watched_list_2_keys_dict = {} + show_indecies = [] + season_indecies = [] + + # Iterate through episode_watched_list_2_keys_dict["season"] and find the indecies that match season + for season_index, season_value in enumerate( + episode_watched_list_2_keys_dict["season"] + ): + if season_value == season: + season_indecies.append(season_index) + + # Iterate through episode_watched_list_2_keys_dict["show"] and find the indecies that match show_key_dict + for show_index, show_value in enumerate(episode_watched_list_2_keys_dict["show"]): + # Iterate through the keys and values of the show_value dictionary and check if they match show_key_dict + for show_key, show_key_value in show_value.items(): + if show_key == "locations": + # Iterate through the locations in the show_value dictionary + for location in show_key_value: + # If the location is in the episode_watched_list_2_keys_dict dictionary, return index of the key + if ( + contains_nested(location, show_key_dict["locations"]) + is not None + ): + show_indecies.append(show_index) + break + else: + if show_key in show_key_dict.keys(): + if show_key_value == show_key_dict[show_key]: + show_indecies.append(show_index) + break + + # Find the intersection of the show_indecies and season_indecies lists + indecies = list(set(show_indecies) & set(season_indecies)) + + # Create a copy of the dictionary with indecies that match the show and season and none that don't + filtered_episode_watched_list_2_keys_dict = copy.deepcopy( + episode_watched_list_2_keys_dict + ) + for key, value in filtered_episode_watched_list_2_keys_dict.items(): + for index, item in enumerate(value): + if index not in indecies: + filtered_episode_watched_list_2_keys_dict[key][index] = None + + return filtered_episode_watched_list_2_keys_dict + + def get_episode_index_in_dict(episode, episode_watched_list_2_keys_dict): # Iterate through the keys and values of the episode dictionary for episode_key, episode_value in episode.items(): diff --git a/test/test_library.py b/test/test_library.py index 834ff6e..e506ebb 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -83,6 +83,16 @@ episode_titles = { "tvdb": ["8444132"], "completed": [True], "time": [0], + "season": ["Season 1"], + "show": [ + { + "imdb": "tt3581920", + "locations": ("The Last of Us",), + "title": "The Last of Us", + "tmdb": "100088", + "tvdb": "392256", + } + ], } movie_titles = { "imdb": ["tt2380307"], diff --git a/test/test_watched.py b/test/test_watched.py index 105541a..e66be17 100644 --- a/test/test_watched.py +++ b/test/test_watched.py @@ -18,102 +18,225 @@ from src.watched import cleanup_watched, combine_watched_dicts tv_shows_watched_list_1 = { frozenset( { - ("tvdb", "75710"), - ("title", "Criminal Minds"), - ("imdb", "tt0452046"), - ("locations", ("Criminal Minds",)), - ("tmdb", "4057"), + ("locations", ("Doctor Who (2005) {tvdb-78804} {imdb-tt0436992}",)), + ("imdb", "tt0436992"), + ("tmdb", "57243"), + ("tvdb", "78804"), + ("title", "Doctor Who (2005)"), } ): { - "Season 1": [ + 1: [ { - "imdb": "tt0550489", - "tmdb": "282843", - "tvdb": "176357", - "title": "Extreme Aggressor", - "locations": ( - "Criminal Minds S01E01 Extreme Aggressor WEBDL-720p.mkv", - ), + "imdb": "tt0563001", + "tmdb": "968589", + "tvdb": "295296", + "title": "The Unquiet Dead", + "locations": ("S01E03.mkv",), "status": {"completed": True, "time": 0}, }, { - "imdb": "tt0550487", - "tmdb": "282861", - "tvdb": "300385", - "title": "Compulsion", - "locations": ("Criminal Minds S01E02 Compulsion WEBDL-720p.mkv",), + "imdb": "tt0562985", + "tmdb": "968590", + "tvdb": "295297", + "title": "Aliens of London (1)", + "locations": ("S01E04.mkv",), + "status": {"completed": False, "time": 240000}, + }, + { + "imdb": "tt0563003", + "tmdb": "968592", + "tvdb": "295298", + "title": "World War Three (2)", + "locations": ("S01E05.mkv",), "status": {"completed": True, "time": 0}, }, ] }, - frozenset({("title", "Test"), ("locations", ("Test",))}): { - "Season 1": [ + frozenset( + { + ("title", "Monarch: Legacy of Monsters"), + ("imdb", "tt17220216"), + ("tvdb", "422598"), + ("tmdb", "202411"), + ( + "locations", + ("Monarch - Legacy of Monsters {tvdb-422598} {imdb-tt17220216}",), + ), + } + ): { + 1: [ { - "title": "S01E01", - "locations": ("Test S01E01.mkv",), + "imdb": "tt21255044", + "tmdb": "4661246", + "tvdb": "10009418", + "title": "Secrets and Lies", + "locations": ("S01E03.mkv",), "status": {"completed": True, "time": 0}, }, { - "title": "S01E02", - "locations": ("Test S01E02.mkv",), + "imdb": "tt21255050", + "tmdb": "4712059", + "tvdb": "10009419", + "title": "Parallels and Interiors", + "locations": ("S01E04.mkv",), + "status": {"completed": False, "time": 240000}, + }, + { + "imdb": "tt23787572", + "tmdb": "4712061", + "tvdb": "10009420", + "title": "The Way Out", + "locations": ("S01E05.mkv",), + "status": {"completed": True, "time": 0}, + }, + ] + }, + frozenset( + { + ("tmdb", "125928"), + ("imdb", "tt14681924"), + ("tvdb", "403172"), + ( + "locations", + ("My Adventures with Superman {tvdb-403172} {imdb-tt14681924}",), + ), + ("title", "My Adventures with Superman"), + } + ): { + 1: [ + { + "imdb": "tt15699926", + "tmdb": "3070048", + "tvdb": "8438181", + "title": "Adventures of a Normal Man (1)", + "locations": ("S01E01.mkv",), "status": {"completed": True, "time": 0}, }, { - "title": "S01E04", - "locations": ("Test S01E04.mkv",), - "status": {"completed": False, "time": 5}, + "imdb": "tt20413322", + "tmdb": "4568681", + "tvdb": "9829910", + "title": "Adventures of a Normal Man (2)", + "locations": ("S01E02.mkv",), + "status": {"completed": True, "time": 0}, + }, + { + "imdb": "tt20413328", + "tmdb": "4497012", + "tvdb": "9870382", + "title": "My Interview with Superman", + "locations": ("S01E03.mkv",), + "status": {"completed": True, "time": 0}, }, ] }, } + tv_shows_watched_list_2 = { frozenset( { - ("tvdb", "75710"), - ("title", "Criminal Minds"), - ("imdb", "tt0452046"), - ("locations", ("Criminal Minds",)), - ("tmdb", "4057"), + ("locations", ("Doctor Who (2005) {tvdb-78804} {imdb-tt0436992}",)), + ("imdb", "tt0436992"), + ("tmdb", "57243"), + ("title", "Doctor Who"), + ("tvdb", "78804"), + ("tvrage", "3332"), } ): { - "Season 1": [ + 1: [ { - "imdb": "tt0550487", - "tmdb": "282861", - "tvdb": "300385", - "title": "Compulsion", - "locations": ("Criminal Minds S01E02 Compulsion WEBDL-720p.mkv",), + "tvdb": "295294", + "imdb": "tt0562992", + "title": "Rose", + "locations": ("S01E01.mkv",), "status": {"completed": True, "time": 0}, }, { - "imdb": "tt0550498", - "tmdb": "282865", - "tvdb": "300474", - "title": "Won't Get Fooled Again", - "locations": ( - "Criminal Minds S01E03 Won't Get Fooled Again WEBDL-720p.mkv", - ), + "tvdb": "295295", + "imdb": "tt0562997", + "title": "The End of the World", + "locations": ("S01E02.mkv",), + "status": {"completed": False, "time": 300670}, + }, + { + "tvdb": "295298", + "imdb": "tt0563003", + "title": "World War Three (2)", + "locations": ("S01E05.mkv",), "status": {"completed": True, "time": 0}, }, ] }, - frozenset({("title", "Test"), ("locations", ("Test",))}): { - "Season 1": [ + frozenset( + { + ("title", "Monarch: Legacy of Monsters"), + ("imdb", "tt17220216"), + ("tvdb", "422598"), + ("tmdb", "202411"), + ( + "locations", + ("Monarch - Legacy of Monsters {tvdb-422598} {imdb-tt17220216}",), + ), + } + ): { + 1: [ { - "title": "S01E02", - "locations": ("Test S01E02.mkv",), - "status": {"completed": False, "time": 10}, - }, - { - "title": "S01E03", - "locations": ("Test S01E03.mkv",), + "tvdb": "9959300", + "imdb": "tt20412166", + "title": "Aftermath", + "locations": ("S01E01.mkv",), "status": {"completed": True, "time": 0}, }, { - "title": "S01E04", - "locations": ("Test S01E04.mkv",), - "status": {"completed": False, "time": 10}, + "tvdb": "10009417", + "imdb": "tt22866594", + "title": "Departure", + "locations": ("S01E02.mkv",), + "status": {"completed": False, "time": 300741}, + }, + { + "tvdb": "10009420", + "imdb": "tt23787572", + "title": "The Way Out", + "locations": ("S01E05.mkv",), + "status": {"completed": True, "time": 0}, + }, + ] + }, + frozenset( + { + ("tmdb", "125928"), + ("imdb", "tt14681924"), + ("tvdb", "403172"), + ( + "locations", + ("My Adventures with Superman {tvdb-403172} {imdb-tt14681924}",), + ), + ("title", "My Adventures with Superman"), + } + ): { + 1: [ + { + "tvdb": "8438181", + "imdb": "tt15699926", + "title": "Adventures of a Normal Man (1)", + "locations": ("S01E01.mkv",), + "status": {"completed": True, "time": 0}, + }, + { + "tvdb": "9829910", + "imdb": "tt20413322", + "title": "Adventures of a Normal Man (2)", + "locations": ("S01E02.mkv",), + "status": {"completed": True, "time": 0}, + }, + { + "tvdb": "9870382", + "imdb": "tt20413328", + "title": "My Interview with Superman", + "locations": ("S01E03.mkv",), + "status": {"completed": True, "time": 0}, }, ] }, @@ -122,38 +245,61 @@ tv_shows_watched_list_2 = { expected_tv_show_watched_list_1 = { frozenset( { - ("tvdb", "75710"), - ("title", "Criminal Minds"), - ("imdb", "tt0452046"), - ("locations", ("Criminal Minds",)), - ("tmdb", "4057"), + ("locations", ("Doctor Who (2005) {tvdb-78804} {imdb-tt0436992}",)), + ("imdb", "tt0436992"), + ("tmdb", "57243"), + ("tvdb", "78804"), + ("title", "Doctor Who (2005)"), } ): { - "Season 1": [ + 1: [ { - "imdb": "tt0550489", - "tmdb": "282843", - "tvdb": "176357", - "title": "Extreme Aggressor", - "locations": ( - "Criminal Minds S01E01 Extreme Aggressor WEBDL-720p.mkv", - ), - "status": {"completed": True, "time": 0}, - } - ] - }, - frozenset({("title", "Test"), ("locations", ("Test",))}): { - "Season 1": [ - { - "title": "S01E01", - "locations": ("Test S01E01.mkv",), + "imdb": "tt0563001", + "tmdb": "968589", + "tvdb": "295296", + "title": "The Unquiet Dead", + "locations": ("S01E03.mkv",), "status": {"completed": True, "time": 0}, }, { - "title": "S01E02", - "locations": ("Test S01E02.mkv",), + "imdb": "tt0562985", + "tmdb": "968590", + "tvdb": "295297", + "title": "Aliens of London (1)", + "locations": ("S01E04.mkv",), + "status": {"completed": False, "time": 240000}, + }, + ] + }, + frozenset( + { + ("title", "Monarch: Legacy of Monsters"), + ("imdb", "tt17220216"), + ("tvdb", "422598"), + ("tmdb", "202411"), + ( + "locations", + ("Monarch - Legacy of Monsters {tvdb-422598} {imdb-tt17220216}",), + ), + } + ): { + 1: [ + { + "imdb": "tt21255044", + "tmdb": "4661246", + "tvdb": "10009418", + "title": "Secrets and Lies", + "locations": ("S01E03.mkv",), "status": {"completed": True, "time": 0}, }, + { + "imdb": "tt21255050", + "tmdb": "4712059", + "tvdb": "10009419", + "title": "Parallels and Interiors", + "locations": ("S01E04.mkv",), + "status": {"completed": False, "time": 240000}, + }, ] }, } @@ -161,37 +307,57 @@ expected_tv_show_watched_list_1 = { expected_tv_show_watched_list_2 = { frozenset( { - ("tvdb", "75710"), - ("title", "Criminal Minds"), - ("imdb", "tt0452046"), - ("locations", ("Criminal Minds",)), - ("tmdb", "4057"), + ("locations", ("Doctor Who (2005) {tvdb-78804} {imdb-tt0436992}",)), + ("imdb", "tt0436992"), + ("tmdb", "57243"), + ("title", "Doctor Who"), + ("tvdb", "78804"), + ("tvrage", "3332"), } ): { - "Season 1": [ + 1: [ { - "imdb": "tt0550498", - "tmdb": "282865", - "tvdb": "300474", - "title": "Won't Get Fooled Again", - "locations": ( - "Criminal Minds S01E03 Won't Get Fooled Again WEBDL-720p.mkv", - ), - "status": {"completed": True, "time": 0}, - } - ] - }, - frozenset({("title", "Test"), ("locations", ("Test",))}): { - "Season 1": [ - { - "title": "S01E03", - "locations": ("Test S01E03.mkv",), + "tvdb": "295294", + "imdb": "tt0562992", + "title": "Rose", + "locations": ("S01E01.mkv",), "status": {"completed": True, "time": 0}, }, { - "title": "S01E04", - "locations": ("Test S01E04.mkv",), - "status": {"completed": False, "time": 10}, + "tvdb": "295295", + "imdb": "tt0562997", + "title": "The End of the World", + "locations": ("S01E02.mkv",), + "status": {"completed": False, "time": 300670}, + }, + ] + }, + frozenset( + { + ("title", "Monarch: Legacy of Monsters"), + ("imdb", "tt17220216"), + ("tvdb", "422598"), + ("tmdb", "202411"), + ( + "locations", + ("Monarch - Legacy of Monsters {tvdb-422598} {imdb-tt17220216}",), + ), + } + ): { + 1: [ + { + "tvdb": "9959300", + "imdb": "tt20412166", + "title": "Aftermath", + "locations": ("S01E01.mkv",), + "status": {"completed": True, "time": 0}, + }, + { + "tvdb": "10009417", + "imdb": "tt22866594", + "title": "Departure", + "locations": ("S01E02.mkv",), + "status": {"completed": False, "time": 300741}, }, ] }, @@ -199,61 +365,92 @@ expected_tv_show_watched_list_2 = { movies_watched_list_1 = [ { - "imdb": "tt2380307", - "tmdb": "354912", - "title": "Coco", - "locations": ("Coco (2017) Remux-1080p.mkv",), + "imdb": "tt1254207", + "tmdb": "10378", + "tvdb": "12352", + "title": "Big Buck Bunny", + "locations": ("Big Buck Bunny.mkv",), "status": {"completed": True, "time": 0}, }, { - "tmdbcollection": "448150", - "imdb": "tt1431045", - "tmdb": "293660", - "title": "Deadpool", - "locations": ("Deadpool (2016) Remux-1080p.mkv",), + "imdb": "tt16431870", + "tmdb": "1029575", + "tvdb": "351194", + "title": "The Family Plan", + "locations": ("The Family Plan (2023).mkv",), "status": {"completed": True, "time": 0}, }, + { + "imdb": "tt5537002", + "tmdb": "466420", + "tvdb": "135852", + "title": "Killers of the Flower Moon", + "locations": ("Killers of the Flower Moon (2023).mkv",), + "status": {"completed": False, "time": 240000}, + }, ] movies_watched_list_2 = [ { - "imdb": "tt2380307", - "tmdb": "354912", - "title": "Coco", - "locations": ("Coco (2017) Remux-1080p.mkv",), + "imdb": "tt16431870", + "tmdb": "1029575", + "title": "The Family Plan", + "locations": ("The Family Plan (2023).mkv",), "status": {"completed": True, "time": 0}, }, { - "imdb": "tt0384793", - "tmdb": "9788", - "tvdb": "9103", - "title": "Accepted", - "locations": ("Accepted (2006) Remux-1080p.mkv",), + "imdb": "tt4589218", + "tmdb": "507089", + "title": "Five Nights at Freddy's", + "locations": ("Five Nights at Freddy's (2023).mkv",), "status": {"completed": True, "time": 0}, }, + { + "imdb": "tt10545296", + "tmdb": "695721", + "tmdbcollection": "131635", + "title": "The Hunger Games: The Ballad of Songbirds & Snakes", + "locations": ("The Hunger Games The Ballad of Songbirds & Snakes (2023).mkv",), + "status": {"completed": False, "time": 301215}, + }, ] expected_movie_watched_list_1 = [ { - "tmdbcollection": "448150", - "imdb": "tt1431045", - "tmdb": "293660", - "title": "Deadpool", - "locations": ("Deadpool (2016) Remux-1080p.mkv",), + "imdb": "tt1254207", + "tmdb": "10378", + "tvdb": "12352", + "title": "Big Buck Bunny", + "locations": ("Big Buck Bunny.mkv",), "status": {"completed": True, "time": 0}, - } + }, + { + "imdb": "tt5537002", + "tmdb": "466420", + "tvdb": "135852", + "title": "Killers of the Flower Moon", + "locations": ("Killers of the Flower Moon (2023).mkv",), + "status": {"completed": False, "time": 240000}, + }, ] expected_movie_watched_list_2 = [ { - "imdb": "tt0384793", - "tmdb": "9788", - "tvdb": "9103", - "title": "Accepted", - "locations": ("Accepted (2006) Remux-1080p.mkv",), + "imdb": "tt4589218", + "tmdb": "507089", + "title": "Five Nights at Freddy's", + "locations": ("Five Nights at Freddy's (2023).mkv",), "status": {"completed": True, "time": 0}, - } + }, + { + "imdb": "tt10545296", + "tmdb": "695721", + "tmdbcollection": "131635", + "title": "The Hunger Games: The Ballad of Songbirds & Snakes", + "locations": ("The Hunger Games The Ballad of Songbirds & Snakes (2023).mkv",), + "status": {"completed": False, "time": 301215}, + }, ] # Test to see if objects get deleted all the way up to the root. diff --git a/test/validate_ci_marklog.py b/test/validate_ci_marklog.py index 34099f6..4943bcb 100644 --- a/test/validate_ci_marklog.py +++ b/test/validate_ci_marklog.py @@ -2,12 +2,14 @@ import os + def read_marklog(): marklog = os.path.join(os.getcwd(), "mark.log") with open(marklog, "r") as f: lines = f.readlines() return lines + def check_marklog(lines, expected_values): for line in lines: # Remove the newline character @@ -17,11 +19,18 @@ def check_marklog(lines, expected_values): return False return True + def main(): expected_values = [ - "jellyplex_watched/TV Shows/Blender Shorts/Episode 2" - , "JellyUser/Movies/Big Buck Bunny" - , "JellyUser/Shows/Blender Shorts/S01E01" + "jellyplex_watched/Movies/Five Nights at Freddy's", + "jellyplex_watched/Movies/The Hunger Games: The Ballad of Songbirds & Snakes/301215", + "jellyplex_watched/TV Shows/Doctor Who (2005)/Rose", + "jellyplex_watched/TV Shows/Doctor Who (2005)/The End of the World/300670", + "jellyplex_watched/TV Shows/Monarch: Legacy of Monsters/Aftermath", + "jellyplex_watched/TV Shows/Monarch: Legacy of Monsters/Departure/300741", + "JellyUser/Movies/Big Buck Bunny", + "JellyUser/Shows/Doctor Who/The Unquiet Dead", + "JellyUser/Shows/Monarch: Legacy of Monsters/Secrets and Lies", ] lines = read_marklog() @@ -31,5 +40,6 @@ def main(): else: print("Marklog contained the expected values") + if __name__ == "__main__": main() From 9375d482b05324d68f0115aacbd919df358fce66 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 2 Jan 2024 23:58:05 -0700 Subject: [PATCH 51/62] CI: Improve mark validation --- test/validate_ci_marklog.py | 41 ++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/test/validate_ci_marklog.py b/test/validate_ci_marklog.py index 4943bcb..36ce925 100644 --- a/test/validate_ci_marklog.py +++ b/test/validate_ci_marklog.py @@ -11,15 +11,34 @@ def read_marklog(): def check_marklog(lines, expected_values): - for line in lines: - # Remove the newline character - line = line.strip() - if line not in expected_values: - print("Line not found in marklog: " + line) - return False - return True + try: + # Check to make sure the marklog contains all the expected values and nothing else + found_values = [] + for line in lines: + # Remove the newline character + line = line.strip() + if line not in expected_values: + raise Exception("Line not found in marklog: " + line) + found_values.append(line) + + # Check to make sure the marklog contains the same number of values as the expected values + if len(found_values) != len(expected_values): + raise Exception("Marklog did not contain the same number of values as the expected values, found " + + str(len(found_values)) + " values, expected " + str(len(expected_values)) + " values") + + # Check that the two lists contain the same values + if sorted(found_values) != sorted(expected_values): + raise Exception("Marklog did not contain the same values as the expected values, found:\n" + + "\n".join(sorted(found_values)) + "\n\nExpected:\n" + "\n".join(sorted(expected_values))) + + return True + except Exception as e: + print(e) + return False + + def main(): expected_values = [ "jellyplex_watched/Movies/Five Nights at Freddy's", @@ -35,10 +54,12 @@ def main(): lines = read_marklog() if not check_marklog(lines, expected_values): - print("Marklog did not contain the expected values") + print("Failed to validate marklog") exit(1) - else: - print("Marklog contained the expected values") + + print("Successfully validated marklog") + exit(0) + if __name__ == "__main__": From 26f40110d0dc42f3f62d56c285aed0ffc43e2cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Bani=C4=87?= Date: Fri, 14 Apr 2023 00:28:01 +0200 Subject: [PATCH 52/62] Bump minimum Python version to 3.9 --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index f4779d9..51917e2 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,9 @@ import sys if __name__ == "__main__": - # Check python version 3.6 or higher - if not (3, 6) <= tuple(map(int, sys.version_info[:2])): - print("This script requires Python 3.6 or higher") + # Check python version 3.9 or higher + if not (3, 9) <= tuple(map(int, sys.version_info[:2])): + print("This script requires Python 3.9 or higher") sys.exit(1) from src.main import main From 9498335e222cb5d34c50ad1a8cc8d1968aebd24b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Bani=C4=87?= Date: Fri, 14 Apr 2023 00:28:44 +0200 Subject: [PATCH 53/62] Deduplicate get_movie_guids and get_episode_guids --- src/plex.py | 78 +++++++++++++++++++---------------------------------- 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/src/plex.py b/src/plex.py index ba4fb41..a978724 100644 --- a/src/plex.py +++ b/src/plex.py @@ -1,7 +1,10 @@ -import re, requests, traceback +import re, requests, os, traceback +from typing import Dict, Union + from urllib3.poolmanager import PoolManager from math import floor +from plexapi.video import Episode, Movie from plexapi.server import PlexServer from plexapi.myplex import MyPlexAccount @@ -30,55 +33,28 @@ class HostNameIgnoringAdapter(requests.adapters.HTTPAdapter): ) -def get_movie_guids(video, completed=True): - logger(f"Plex: {video.title} {video.guids} {video.locations}", 3) +def get_guids(item: Union[Movie, Episode], completed=True): + guids: Dict[str, str] = dict( + guid.id.split('://') + for guid + in item.guids + if guid.id is not None and len(guid.id.strip()) > 0 + ) - movie_guids = {} - try: - for guid in video.guids: - # Extract source and id from guid.id - m = re.match(r"(.*)://(.*)", guid.id) - guid_source, guid_id = m.group(1).lower(), m.group(2) - movie_guids[guid_source] = guid_id - except Exception: - logger(f"Plex: Failed to get guids for {video.title}, Using location only", 1) - - movie_guids["title"] = video.title - movie_guids["locations"] = tuple([x.split("/")[-1] for x in video.locations]) - - movie_guids["status"] = { - "completed": completed, - "time": video.viewOffset, - } - - return movie_guids - - -def get_episode_guids(episode, show, completed=True): - episode_guids_temp = {} - try: - for guid in episode.guids: - # Extract after :// from guid.id - m = re.match(r"(.*)://(.*)", guid.id) - guid_source, guid_id = m.group(1).lower(), m.group(2) - episode_guids_temp[guid_source] = guid_id - except Exception: + if len(guids) == 0: logger( - f"Plex: Failed to get guids for {episode.title} in {show.title}, Using location only", + f"Plex: Failed to get any guids for {item.title}, Using location only", 1, ) - episode_guids_temp["title"] = episode.title - episode_guids_temp["locations"] = tuple( - [x.split("/")[-1] for x in episode.locations] - ) - - episode_guids_temp["status"] = { - "completed": completed, - "time": episode.viewOffset, - } - - return episode_guids_temp + return { + 'title': item.title, + 'locations': tuple([location.split("/")[-1] for location in item.locations]), + 'status': { + "completed": completed, + "time": item.viewOffset, + } + } | guids def get_user_library_watched_show(show): @@ -108,15 +84,15 @@ def get_user_library_watched_show(show): if episode.parentIndex not in episode_guids: episode_guids[episode.parentIndex] = [] - episode_guids[episode.parentIndex].append( - get_episode_guids(episode, show, completed=True) + episode_guids[episode.parentTitle].append( + get_guids(episode, completed=True) ) elif episode.viewOffset > 0: if episode.parentIndex not in episode_guids: episode_guids[episode.parentIndex] = [] - episode_guids[episode.parentIndex].append( - get_episode_guids(episode, show, completed=False) + episode_guids[episode.parentTitle].append( + get_guids(episode, completed=False) ) return show_guids, episode_guids @@ -145,7 +121,7 @@ def get_user_library_watched(user, user_plex, library): for video in library_videos.search(unwatched=False): logger(f"Plex: Adding {video.title} to {user_name} watched list", 3) - movie_guids = get_movie_guids(video, completed=True) + movie_guids = get_guids(video, completed=True) user_watched[user_name][library.title].append(movie_guids) @@ -156,7 +132,7 @@ def get_user_library_watched(user, user_plex, library): logger(f"Plex: Adding {video.title} to {user_name} watched list", 3) - movie_guids = get_movie_guids(video, completed=False) + movie_guids = get_guids(video, completed=False) user_watched[user_name][library.title].append(movie_guids) From 2e4c2a68173e7ee08b322b1b0af1bbcbf98ec426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Bani=C4=87?= Date: Fri, 14 Apr 2023 10:14:10 +0200 Subject: [PATCH 54/62] Refactor get_user_library_watched_show --- src/plex.py | 78 ++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/plex.py b/src/plex.py index a978724..e5e39a2 100644 --- a/src/plex.py +++ b/src/plex.py @@ -1,9 +1,13 @@ import re, requests, os, traceback -from typing import Dict, Union +from typing import Dict, Union, FrozenSet +import operator +from itertools import groupby as itertools_groupby from urllib3.poolmanager import PoolManager from math import floor +from requests.adapters import HTTPAdapter as RequestsHTTPAdapter + from plexapi.video import Episode, Movie from plexapi.server import PlexServer from plexapi.myplex import MyPlexAccount @@ -22,7 +26,7 @@ from src.library import ( # Bypass hostname validation for ssl. Taken from https://github.com/pkkid/python-plexapi/issues/143#issuecomment-775485186 -class HostNameIgnoringAdapter(requests.adapters.HTTPAdapter): +class HostNameIgnoringAdapter(RequestsHTTPAdapter): def init_poolmanager(self, connections, maxsize, block=..., **pool_kwargs): self.poolmanager = PoolManager( num_pools=connections, @@ -33,7 +37,7 @@ class HostNameIgnoringAdapter(requests.adapters.HTTPAdapter): ) -def get_guids(item: Union[Movie, Episode], completed=True): +def extract_guids_from_item(item: Union[Movie, Episode]) -> Dict[str, str]: guids: Dict[str, str] = dict( guid.id.split('://') for guid @@ -47,6 +51,10 @@ def get_guids(item: Union[Movie, Episode], completed=True): 1, ) + return guids + + +def get_guids(item: Union[Movie, Episode], completed=True): return { 'title': item.title, 'locations': tuple([location.split("/")[-1] for location in item.locations]), @@ -54,49 +62,41 @@ def get_guids(item: Union[Movie, Episode], completed=True): "completed": completed, "time": item.viewOffset, } - } | guids + } | extract_guids_from_item(item) # Merge the metadata and guid dictionaries def get_user_library_watched_show(show): try: - show_guids = {} - try: - for show_guid in show.guids: - # Extract source and id from guid.id - m = re.match(r"(.*)://(.*)", show_guid.id) - show_guid_source, show_guid_id = m.group(1).lower(), m.group(2) - show_guids[show_guid_source] = show_guid_id - except Exception: - logger( - f"Plex: Failed to get guids for {show.title}, Using location only", 1 + show_guids: FrozenSet = frozenset( + ({ + 'title': show.title, + 'locations': tuple( + [location.split("/")[-1] for location in show.locations]) + } | extract_guids_from_item(show)).items() # Merge the metadata and guid dictionaries + ) + + watched_episodes = show.watched() + episode_guids = { + # Offset group data because the first value will be the key + season: [episode[1] for episode in episodes] + for season, episodes + # Group episodes by first element of tuple (episode.parentIndex) + in itertools_groupby( + [ + ( + episode.parentIndex, + get_guids(episode, completed=episode in watched_episodes) + ) + for episode + in show.episodes() + # Only include watched/partially-watched episodes + if episode in watched_episodes or episode.viewOffset > 0 + ], + operator.itemgetter(0) ) - - show_guids["title"] = show.title - show_guids["locations"] = tuple([x.split("/")[-1] for x in show.locations]) - show_guids = frozenset(show_guids.items()) - - # Get all watched episodes for show - episode_guids = {} - watched = show.watched() - - for episode in show.episodes(): - if episode in watched: - if episode.parentIndex not in episode_guids: - episode_guids[episode.parentIndex] = [] - - episode_guids[episode.parentTitle].append( - get_guids(episode, completed=True) - ) - elif episode.viewOffset > 0: - if episode.parentIndex not in episode_guids: - episode_guids[episode.parentIndex] = [] - - episode_guids[episode.parentTitle].append( - get_guids(episode, completed=False) - ) + } return show_guids, episode_guids - except Exception: return {}, {} From 26f1f80be75c94938087a4d948bef06fa6f10975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Bani=C4=87?= Date: Fri, 14 Apr 2023 11:59:20 +0200 Subject: [PATCH 55/62] Refactor get_user_library_watched --- src/plex.py | 78 ++++++++++++++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/src/plex.py b/src/plex.py index e5e39a2..19f9182 100644 --- a/src/plex.py +++ b/src/plex.py @@ -102,11 +102,8 @@ def get_user_library_watched_show(show): def get_user_library_watched(user, user_plex, library): + user_name: str = user.title.lower() try: - user_name = user.username.lower() if user.username else user.title.lower() - user_watched = {} - user_watched[user_name] = {} - logger( f"Plex: Generating watched for {user_name} in library {library.title}", 0, @@ -115,58 +112,59 @@ def get_user_library_watched(user, user_plex, library): library_videos = user_plex.library.section(library.title) if library.type == "movie": - user_watched[user_name][library.title] = [] + watched = [] - # Get all watched movies - for video in library_videos.search(unwatched=False): - logger(f"Plex: Adding {video.title} to {user_name} watched list", 3) - - movie_guids = get_guids(video, completed=True) - - user_watched[user_name][library.title].append(movie_guids) - - # Get all partially watched movies greater than 1 minute - for video in library_videos.search(inProgress=True): - if video.viewOffset < 60000: - continue - - logger(f"Plex: Adding {video.title} to {user_name} watched list", 3) - - movie_guids = get_guids(video, completed=False) - - user_watched[user_name][library.title].append(movie_guids) + args = [ + [get_guids, video, True] + for video + # Get all watched movies + in library_videos.search(unwatched=False) + ] + [ + [get_guids, video, False] + for video + # Get all partially watched movies + in library_videos.search(inProgress=True) + # Ignore all partially watched movies watched under 1 minute + if video.viewOffset < 60000 + ] + for guid in future_thread_executor( + args, threads=min(os.cpu_count(), 4) + ): + logger(f"Plex: Adding {guid['title']} to {user_name} watched list", 3) + watched.append(guid) elif library.type == "show": - user_watched[user_name][library.title] = {} + watched = {} - # Parallelize show processing - args = [] - - # Get all watched shows - for show in library_videos.search(unwatched=False): - args.append([get_user_library_watched_show, show]) - - # Get all partially watched shows - for show in library_videos.search(inProgress=True): - args.append([get_user_library_watched_show, show]) + # Get all watched shows and partially watched shows + args = [ + (get_user_library_watched_show, show) + for show + in library_videos.search(unwatched=False) + library_videos.search(inProgress=True) + ] for show_guids, episode_guids in future_thread_executor(args, threads=4): if show_guids and episode_guids: # append show, season, episode - if show_guids not in user_watched[user_name][library.title]: - user_watched[user_name][library.title][show_guids] = {} + if show_guids not in watched: + watched[show_guids] = {} - user_watched[user_name][library.title][show_guids] = episode_guids + watched[show_guids] = episode_guids logger( f"Plex: Added {episode_guids} to {user_name} {show_guids} watched list", 3, ) + else: + watched = None logger(f"Plex: Got watched for {user_name} in library {library.title}", 1) - if library.title in user_watched[user_name]: - logger(f"Plex: {user_watched[user_name][library.title]}", 3) + logger(f"Plex: {watched}", 3) - return user_watched + return { + user_name: { + library.title: watched + } if watched is not None else {} + } except Exception as e: logger( f"Plex: Failed to get watched for {user_name} in library {library.title}, Error: {e}", From 64b2197844fa58fdc5f8e60459fc7858be221a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Bani=C4=87?= Date: Fri, 14 Apr 2023 13:27:09 +0200 Subject: [PATCH 56/62] Remove unnecessary check --- src/plex.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/plex.py b/src/plex.py index 19f9182..003c752 100644 --- a/src/plex.py +++ b/src/plex.py @@ -145,10 +145,6 @@ def get_user_library_watched(user, user_plex, library): for show_guids, episode_guids in future_thread_executor(args, threads=4): if show_guids and episode_guids: - # append show, season, episode - if show_guids not in watched: - watched[show_guids] = {} - watched[show_guids] = episode_guids logger( f"Plex: Added {episode_guids} to {user_name} {show_guids} watched list", From 8fa9351ef132c0584916bdea2fc4473d7bfb7db6 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 2 Jan 2024 19:05:36 -0700 Subject: [PATCH 57/62] Plex: Only partially watched more than 1 min --- src/plex.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plex.py b/src/plex.py index 003c752..d4e7b62 100644 --- a/src/plex.py +++ b/src/plex.py @@ -89,8 +89,8 @@ def get_user_library_watched_show(show): ) for episode in show.episodes() - # Only include watched/partially-watched episodes - if episode in watched_episodes or episode.viewOffset > 0 + # Only include watched or partially-watched more than a minute episodes + if episode in watched_episodes or episode.viewOffset >= 60000 ], operator.itemgetter(0) ) @@ -124,8 +124,8 @@ def get_user_library_watched(user, user_plex, library): for video # Get all partially watched movies in library_videos.search(inProgress=True) - # Ignore all partially watched movies watched under 1 minute - if video.viewOffset < 60000 + # Only include partially-watched movies more than a minute + if video.viewOffset >= 60000 ] for guid in future_thread_executor( From 98a824bfdcd311994b977635e21ec631389b3d3e Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Tue, 2 Jan 2024 20:58:26 -0700 Subject: [PATCH 58/62] Plex: Format --- src/plex.py | 76 ++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/src/plex.py b/src/plex.py index d4e7b62..6b14c27 100644 --- a/src/plex.py +++ b/src/plex.py @@ -39,9 +39,8 @@ class HostNameIgnoringAdapter(RequestsHTTPAdapter): def extract_guids_from_item(item: Union[Movie, Episode]) -> Dict[str, str]: guids: Dict[str, str] = dict( - guid.id.split('://') - for guid - in item.guids + guid.id.split("://") + for guid in item.guids if guid.id is not None and len(guid.id.strip()) > 0 ) @@ -56,23 +55,29 @@ def extract_guids_from_item(item: Union[Movie, Episode]) -> Dict[str, str]: def get_guids(item: Union[Movie, Episode], completed=True): return { - 'title': item.title, - 'locations': tuple([location.split("/")[-1] for location in item.locations]), - 'status': { + "title": item.title, + "locations": tuple([location.split("/")[-1] for location in item.locations]), + "status": { "completed": completed, "time": item.viewOffset, - } - } | extract_guids_from_item(item) # Merge the metadata and guid dictionaries + }, + } | extract_guids_from_item( + item + ) # Merge the metadata and guid dictionaries def get_user_library_watched_show(show): try: show_guids: FrozenSet = frozenset( - ({ - 'title': show.title, - 'locations': tuple( - [location.split("/")[-1] for location in show.locations]) - } | extract_guids_from_item(show)).items() # Merge the metadata and guid dictionaries + ( + { + "title": show.title, + "locations": tuple( + [location.split("/")[-1] for location in show.locations] + ), + } + | extract_guids_from_item(show) + ).items() # Merge the metadata and guid dictionaries ) watched_episodes = show.watched() @@ -85,14 +90,13 @@ def get_user_library_watched_show(show): [ ( episode.parentIndex, - get_guids(episode, completed=episode in watched_episodes) + get_guids(episode, completed=episode in watched_episodes), ) - for episode - in show.episodes() + for episode in show.episodes() # Only include watched or partially-watched more than a minute episodes if episode in watched_episodes or episode.viewOffset >= 60000 ], - operator.itemgetter(0) + operator.itemgetter(0), ) } @@ -115,22 +119,20 @@ def get_user_library_watched(user, user_plex, library): watched = [] args = [ - [get_guids, video, True] - for video - # Get all watched movies - in library_videos.search(unwatched=False) - ] + [ - [get_guids, video, False] - for video - # Get all partially watched movies - in library_videos.search(inProgress=True) - # Only include partially-watched movies more than a minute - if video.viewOffset >= 60000 - ] + [get_guids, video, True] + for video + # Get all watched movies + in library_videos.search(unwatched=False) + ] + [ + [get_guids, video, False] + for video + # Get all partially watched movies + in library_videos.search(inProgress=True) + # Only include partially-watched movies more than a minute + if video.viewOffset >= 60000 + ] - for guid in future_thread_executor( - args, threads=min(os.cpu_count(), 4) - ): + for guid in future_thread_executor(args, threads=min(os.cpu_count(), 4)): logger(f"Plex: Adding {guid['title']} to {user_name} watched list", 3) watched.append(guid) elif library.type == "show": @@ -139,8 +141,8 @@ def get_user_library_watched(user, user_plex, library): # Get all watched shows and partially watched shows args = [ (get_user_library_watched_show, show) - for show - in library_videos.search(unwatched=False) + library_videos.search(inProgress=True) + for show in library_videos.search(unwatched=False) + + library_videos.search(inProgress=True) ] for show_guids, episode_guids in future_thread_executor(args, threads=4): @@ -156,11 +158,7 @@ def get_user_library_watched(user, user_plex, library): logger(f"Plex: Got watched for {user_name} in library {library.title}", 1) logger(f"Plex: {watched}", 3) - return { - user_name: { - library.title: watched - } if watched is not None else {} - } + return {user_name: {library.title: watched} if watched is not None else {}} except Exception as e: logger( f"Plex: Failed to get watched for {user_name} in library {library.title}, Error: {e}", From 1edfecae42ed89e55af80c64e20cd2aff2864725 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Fri, 5 Jan 2024 22:44:56 -0700 Subject: [PATCH 59/62] Cleanup --- src/watched.py | 4 ++-- test/validate_ci_marklog.py | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/watched.py b/src/watched.py index 8fa60fa..8f76670 100644 --- a/src/watched.py +++ b/src/watched.py @@ -97,7 +97,7 @@ def cleanup_watched( continue ( - show_watched_list_2_keys_dict, + _, episode_watched_list_2_keys_dict, movies_watched_list_2_keys_dict, ) = generate_library_guids_dict(watched_list_2[user_2][library_2]) @@ -273,7 +273,7 @@ def filter_episode_watched_list_2_keys_dict( episode_watched_list_2_keys_dict ) for key, value in filtered_episode_watched_list_2_keys_dict.items(): - for index, item in enumerate(value): + for index in range(len(value)): if index not in indecies: filtered_episode_watched_list_2_keys_dict[key][index] = None diff --git a/test/validate_ci_marklog.py b/test/validate_ci_marklog.py index 36ce925..45f69bf 100644 --- a/test/validate_ci_marklog.py +++ b/test/validate_ci_marklog.py @@ -20,24 +20,32 @@ def check_marklog(lines, expected_values): if line not in expected_values: raise Exception("Line not found in marklog: " + line) - found_values.append(line) # Check to make sure the marklog contains the same number of values as the expected values if len(found_values) != len(expected_values): - raise Exception("Marklog did not contain the same number of values as the expected values, found " + - str(len(found_values)) + " values, expected " + str(len(expected_values)) + " values") - + raise Exception( + "Marklog did not contain the same number of values as the expected values, found " + + str(len(found_values)) + + " values, expected " + + str(len(expected_values)) + + " values" + ) + # Check that the two lists contain the same values if sorted(found_values) != sorted(expected_values): - raise Exception("Marklog did not contain the same values as the expected values, found:\n" + - "\n".join(sorted(found_values)) + "\n\nExpected:\n" + "\n".join(sorted(expected_values))) + raise Exception( + "Marklog did not contain the same values as the expected values, found:\n" + + "\n".join(sorted(found_values)) + + "\n\nExpected:\n" + + "\n".join(sorted(expected_values)) + ) return True except Exception as e: print(e) return False - + def main(): expected_values = [ @@ -61,6 +69,5 @@ def main(): exit(0) - if __name__ == "__main__": main() From f80c20d70c0d1f5f3548435b7be4b1ddc13a4daf Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Fri, 5 Jan 2024 23:46:15 -0700 Subject: [PATCH 60/62] Watched: Remove deepcopy due to performance --- src/plex.py | 1 - src/watched.py | 15 +++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/plex.py b/src/plex.py index 28a7650..6b14c27 100644 --- a/src/plex.py +++ b/src/plex.py @@ -100,7 +100,6 @@ def get_user_library_watched_show(show): ) } - return show_guids, episode_guids except Exception: return {}, {} diff --git a/src/watched.py b/src/watched.py index 8f76670..23fa2d9 100644 --- a/src/watched.py +++ b/src/watched.py @@ -268,14 +268,17 @@ def filter_episode_watched_list_2_keys_dict( # Find the intersection of the show_indecies and season_indecies lists indecies = list(set(show_indecies) & set(season_indecies)) + filtered_episode_watched_list_2_keys_dict = {} # Create a copy of the dictionary with indecies that match the show and season and none that don't - filtered_episode_watched_list_2_keys_dict = copy.deepcopy( - episode_watched_list_2_keys_dict - ) - for key, value in filtered_episode_watched_list_2_keys_dict.items(): + for key, value in episode_watched_list_2_keys_dict.items(): + if key not in filtered_episode_watched_list_2_keys_dict: + filtered_episode_watched_list_2_keys_dict[key] = [] + for index in range(len(value)): - if index not in indecies: - filtered_episode_watched_list_2_keys_dict[key][index] = None + if index in indecies: + filtered_episode_watched_list_2_keys_dict[key].append(value[index]) + else: + filtered_episode_watched_list_2_keys_dict[key].append(None) return filtered_episode_watched_list_2_keys_dict From 7317e8533d5ecd2b2921af47463261d4033b8cd7 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sat, 6 Jan 2024 00:16:13 -0700 Subject: [PATCH 61/62] Watched: Use enumerate --- src/watched.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/watched.py b/src/watched.py index 23fa2d9..d1325da 100644 --- a/src/watched.py +++ b/src/watched.py @@ -274,7 +274,7 @@ def filter_episode_watched_list_2_keys_dict( if key not in filtered_episode_watched_list_2_keys_dict: filtered_episode_watched_list_2_keys_dict[key] = [] - for index in range(len(value)): + for index, _ in enumerate(value): if index in indecies: filtered_episode_watched_list_2_keys_dict[key].append(value[index]) else: From 95f2a9ad308684d456f4b5451faed7e40fdfcea9 Mon Sep 17 00:00:00 2001 From: Luigi311 Date: Sat, 6 Jan 2024 01:13:15 -0700 Subject: [PATCH 62/62] If only one worker, run in main thread to avoid overhead --- src/functions.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/functions.py b/src/functions.py index c1c160b..2d3d306 100644 --- a/src/functions.py +++ b/src/functions.py @@ -99,6 +99,13 @@ def future_thread_executor(args: list, threads: int = 32): workers = min(int(os.getenv("MAX_THREADS", 32)), os.cpu_count() * 2, threads) + # If only one worker, run in main thread to avoid overhead + if workers == 1: + results = [] + for arg in args: + results.append(arg[0](*arg[1:])) + return results + with ThreadPoolExecutor(max_workers=workers) as executor: for arg in args: # * arg unpacks the list into actual arguments