Merge pull request #44 from luigi311/dev

Fix issues with certain libraries failing

.env.sample

@@ -16,15 +16,15 @@ SLEEP_DURATION = "3600"
 LOGFILE = "log.log"
 
 ## Map usernames between servers in the event that they are different, order does not matter
-## Comma seperated for multiple options
+## Comma separated for multiple options
 #USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" }
 
 ## Map libraries between servers in the even that they are different, order does not matter
-## Comma seperated for multiple options
+## Comma separated for multiple options
 #LIBRARY_MAPPING = { "Shows": "TV Shows", "Movie": "Movies" }
 
 ## Blacklisting/Whitelisting libraries, library types such as Movies/TV Shows, and users. Mappings apply so if the mapping for the user or library exist then both will be excluded.
-## Comma seperated for multiple options
+## Comma separated for multiple options
 #BLACKLIST_LIBRARY = ""
 #WHITELIST_LIBRARY = ""
 #BLACKLIST_LIBRARY_TYPE = "" 
@@ -38,15 +38,15 @@ WHITELIST_USERS = "testuser1,testuser2"
 
 ## Recommended to use token as it is faster to connect as it is direct to the server instead of going through the plex servers
 ## URL of the plex server, use hostname or IP address if the hostname is not resolving correctly
-## Comma seperated list for multiple servers
+## Comma separated list for multiple servers
 PLEX_BASEURL = "http://localhost:32400, https://nas:32400"
 
 ## Plex token https://support.plex.tv/articles/204059436-finding-an-authentication-token-x-plex-token/
-## Comma seperated list for multiple servers
+## Comma separated list for multiple servers
 PLEX_TOKEN = "SuperSecretToken, SuperSecretToken2"
 
 ## If not using plex token then use username and password of the server admin along with the servername
-## Comma seperated for multiple options
+## Comma separated for multiple options
 #PLEX_USERNAME = "PlexUser, PlexUser2"
 #PLEX_PASSWORD = "SuperSecret, SuperSecret2"
 #PLEX_SERVERNAME = "Plex Server1, Plex Server2"
@@ -60,9 +60,9 @@ SSL_BYPASS = "False"
 # Jellyfin
 
 ## Jellyfin server URL, use hostname or IP address if the hostname is not resolving correctly
-## Comma seperated list for multiple servers
+## Comma separated list for multiple servers
 JELLYFIN_BASEURL = "http://localhost:8096, http://nas:8096"
 
 ## Jellyfin api token, created manually by logging in to the jellyfin server admin dashboard and creating an api key
-## Comma seperated list for multiple servers
+## Comma separated list for multiple servers
 JELLYFIN_TOKEN = "SuperSecretToken, SuperSecretToken2"

README.md

@@ -6,7 +6,7 @@ Sync watched between jellyfin and plex locally
 
 ## Description
 
-Keep in sync all your users watched history between jellyfin and plex servers locally. This uses file names and provider ids to find the correct episode/movie between the two. This is not perfect but it works for most cases. You can use this for as many servers as you want by enterying multiple options in the .env plex/jellyfin section seperated by commas.
+Keep in sync all your users watched history between jellyfin and plex servers locally. This uses file names and provider ids to find the correct episode/movie between the two. This is not perfect but it works for most cases. You can use this for as many servers as you want by entering multiple options in the .env plex/jellyfin section separated by commas.
 
 ## Configuration
 
@@ -29,15 +29,15 @@ SLEEP_DURATION = "3600"
 LOGFILE = "log.log"
 
 ## Map usernames between servers in the event that they are different, order does not matter
-## Comma seperated for multiple options
+## Comma separated for multiple options
 USER_MAPPING = { "testuser2": "testuser3", "testuser1":"testuser4" }
 
 ## Map libraries between servers in the even that they are different, order does not matter
-## Comma seperated for multiple options
+## Comma separated for multiple options
 LIBRARY_MAPPING = { "Shows": "TV Shows", "Movie": "Movies" }
 
 ## Blacklisting/Whitelisting libraries, library types such as Movies/TV Shows, and users. Mappings apply so if the mapping for the user or library exist then both will be excluded.
-## Comma seperated for multiple options
+## Comma separated for multiple options
 BLACKLIST_LIBRARY = ""
 WHITELIST_LIBRARY = ""
 BLACKLIST_LIBRARY_TYPE = "" 
@@ -51,15 +51,15 @@ WHITELIST_USERS = "testuser1,testuser2"
 
 ## Recommended to use token as it is faster to connect as it is direct to the server instead of going through the plex servers
 ## URL of the plex server, use hostname or IP address if the hostname is not resolving correctly
-## Comma seperated list for multiple servers
+## Comma separated list for multiple servers
 PLEX_BASEURL = "http://localhost:32400, https://nas:32400"
 
 ## Plex token https://support.plex.tv/articles/204059436-finding-an-authentication-token-x-plex-token/
-## Comma seperated list for multiple servers
+## Comma separated list for multiple servers
 PLEX_TOKEN = "SuperSecretToken, SuperSecretToken2"
 
 ## If not using plex token then use username and password of the server admin along with the servername
-## Comma seperated for multiple options
+## Comma separated for multiple options
 #PLEX_USERNAME = "PlexUser, PlexUser2"
 #PLEX_PASSWORD = "SuperSecret, SuperSecret2"
 #PLEX_SERVERNAME = "Plex Server1, Plex Server2"
@@ -73,11 +73,11 @@ SSL_BYPASS = "False"
 # Jellyfin
 
 ## Jellyfin server URL, use hostname or IP address if the hostname is not resolving correctly
-## Comma seperated list for multiple servers
+## Comma separated list for multiple servers
 JELLYFIN_BASEURL = "http://localhost:8096, http://nas:8096"
 
 ## Jellyfin api token, created manually by logging in to the jellyfin server admin dashboard and creating an api key
-## Comma seperated list for multiple servers
+## Comma separated list for multiple servers
 JELLYFIN_TOKEN = "SuperSecretToken, SuperSecretToken2"
 ```
 
@@ -136,7 +136,7 @@ JELLYFIN_TOKEN = "SuperSecretToken, SuperSecretToken2"
 
 ## Contributing
 
-I am open to recieving pull requests. If you are submitting a pull request, please make sure run it locally for a day or two to make sure it is working as expected and stable. Make all pull requests against the dev branch and nothing will be merged into the main without going through the lower branches.
+I am open to receiving pull requests. If you are submitting a pull request, please make sure run it locally for a day or two to make sure it is working as expected and stable. Make all pull requests against the dev branch and nothing will be merged into the main without going through the lower branches.
 
 ## License
 

src/jellyfin.py

@@ -1,4 +1,4 @@
-import asyncio, aiohttp
+import asyncio, aiohttp, traceback
 from src.functions import (
     logger,
     search_mapping,
@@ -46,9 +46,14 @@ class Jellyfin:
                 ) as response:
                     results = await response.json()
 
+            if type(results) is str:
+                logger(f"Jellyfin: Query {query_type} {query} {results}", 2)
+                raise Exception(results)
+
             # append identifiers to results
             if identifiers:
                 results["Identifiers"] = identifiers
+
             return results
 
         except Exception as e:
@@ -63,7 +68,7 @@ class Jellyfin:
             async with aiohttp.ClientSession() as session:
                 response = await self.query(query_string, "get", session)
 
-            # If reponse is not empty
+            # If response is not empty
             if response:
                 for user in response:
                     users[user["Name"]] = user["Id"]
@@ -148,7 +153,7 @@ class Jellyfin:
                             )
 
                 # TV Shows
-                if library_type == "Series":
+                if library_type in ["Series", "Episode"]:
                     # Initialize an empty dictionary for the given user and library
                     user_watched[user_name][library_title] = {}
 
@@ -184,6 +189,7 @@ class Jellyfin:
                             "show_guids": show_guids,
                             "show_id": show["Id"],
                         }
+
                         season_task = asyncio.ensure_future(
                             self.query(
                                 f"/Shows/{show['Id']}/Seasons"
@@ -309,7 +315,9 @@ class Jellyfin:
                 f"Jellyfin: Failed to get watched for {user_name} in library {library_title}, Error: {e}",
                 2,
             )
-            raise Exception(e)
+
+            logger(traceback.format_exc(), 2)
+            return {}
 
     async def get_users_watched(
         self,
@@ -362,7 +370,7 @@ class Jellyfin:
                         [
                             x["Type"]
                             for x in watched["Items"]
-                            if x["Type"] in ["Movie", "Series"]
+                            if x["Type"] in ["Movie", "Series", "Episode"]
                         ]
                     )
 
@@ -385,8 +393,14 @@ class Jellyfin:
 
                     # If there are multiple types in library raise error
                     if types is None or len(types) < 1:
+                        all_types = set(
+                            [
+                                x["Type"]
+                                for x in watched["Items"]
+                            ]
+                        )
                         logger(
-                            f"Jellyfin: Skipping Library {library_title} not a single type: {types}",
+                            f"Jellyfin: Skipping Library {library_title} found types: {types}, all types: {all_types}",
                             1,
                         )
                         continue

src/plex.py

@@ -1,4 +1,4 @@
-import re, requests
+import re, requests, os, traceback
 from urllib3.poolmanager import PoolManager
 
 from plexapi.server import PlexServer
@@ -24,6 +24,52 @@ class HostNameIgnoringAdapter(requests.adapters.HTTPAdapter):
         )
 
 
+def get_user_library_watched_show(show):
+    try:
+        show_guids = {}
+        for show_guid in show.guids:
+            # Extract source and id from guid.id
+            m = re.match(r"(.*)://(.*)", show_guid.id)
+            show_guid_source, show_guid_id = m.group(1).lower(), m.group(2)
+            show_guids[show_guid_source] = show_guid_id
+
+        show_guids["title"] = show.title
+        show_guids["locations"] = tuple([x.split("/")[-1] for x in show.locations])
+        show_guids = frozenset(show_guids.items())
+
+        # Get all watched episodes for show
+        episode_guids = {}
+        watched_episodes = show.watched()
+        for episode in watched_episodes:
+            episode_guids_temp = {}
+            try:
+                if len(episode.guids) > 0:
+                    for guid in episode.guids:
+                        # Extract after :// from guid.id
+                        m = re.match(r"(.*)://(.*)", guid.id)
+                        guid_source, guid_id = m.group(1).lower(), m.group(2)
+                        episode_guids_temp[guid_source] = guid_id
+            except:
+                logger(
+                    f"Plex: Failed to get guids for {episode.title} in {show.title}, Using location only",
+                    1,
+                )
+
+            episode_guids_temp["locations"] = tuple(
+                [x.split("/")[-1] for x in episode.locations]
+            )
+
+            if episode.parentTitle not in episode_guids:
+                episode_guids[episode.parentTitle] = []
+
+            episode_guids[episode.parentTitle].append(episode_guids_temp)
+
+        return show_guids, episode_guids
+
+    except Exception as e:
+        return {}, {}
+
+
 def get_user_library_watched(user, user_plex, library):
     try:
         user_name = user.title.lower()
@@ -61,41 +107,17 @@ def get_user_library_watched(user, user_plex, library):
 
         elif library.type == "show":
             user_watched[user_name][library.title] = {}
+            shows = library_videos.search(unwatched=False)
 
-            for show in library_videos.search(unwatched=False):
+            # Parallelize show processing
-                logger(f"Plex: Adding {show.title} to {user_name} watched list", 3)
+            args = []
-                show_guids = {}
+            for show in shows:
-                for show_guid in show.guids:
+                args.append([get_user_library_watched_show, show])
-                    # Extract source and id from guid.id
-                    m = re.match(r"(.*)://(.*)", show_guid.id)
-                    show_guid_source, show_guid_id = m.group(1).lower(), m.group(2)
-                    show_guids[show_guid_source] = show_guid_id
 
-                show_guids["title"] = show.title
+            for show_guids, episode_guids in future_thread_executor(
-                show_guids["locations"] = tuple(
+                args, workers=min(os.cpu_count(), 4)
-                    [x.split("/")[-1] for x in show.locations]
+            ):
-                )
+                if show_guids and episode_guids:
-                show_guids = frozenset(show_guids.items())
-
-                # Get all watched episodes for show
-                episode_guids = {}
-                for episode in show.watched():
-                    if episode.viewCount > 0:
-                        episode_guids_temp = {}
-                        for guid in episode.guids:
-                            # Extract after :// from guid.id
-                            m = re.match(r"(.*)://(.*)", guid.id)
-                            guid_source, guid_id = m.group(1).lower(), m.group(2)
-                            episode_guids_temp[guid_source] = guid_id
-
-                        episode_guids_temp["locations"] = tuple(
-                            [x.split("/")[-1] for x in episode.locations]
-                        )
-                        if episode.parentTitle not in episode_guids:
-                            episode_guids[episode.parentTitle] = []
-                        episode_guids[episode.parentTitle].append(episode_guids_temp)
-
-                if episode_guids:
                     # append show, season, episode
                     if show_guids not in user_watched[user_name][library.title]:
                         user_watched[user_name][library.title][show_guids] = {}
@@ -116,7 +138,7 @@ def get_user_library_watched(user, user_plex, library):
             f"Plex: Failed to get watched for {user_name} in library {library.title}, Error: {e}",
             2,
         )
-        raise Exception(e)
+        return {}
 
 
 def update_user_watched(user, user_plex, library, videos, dryrun):
@@ -201,24 +223,30 @@ def update_user_watched(user, user_plex, library, videos, dryrun):
                                 break
 
                         if not episode_found:
-                            for episode_guid in episode_search.guids:
+                            try:
-                                episode_guid_source = (
+                                for episode_guid in episode_search.guids:
-                                    re.search(r"(.*)://", episode_guid.id)
+                                    episode_guid_source = (
-                                    .group(1)
+                                        re.search(r"(.*)://", episode_guid.id)
-                                    .lower()
+                                        .group(1)
-                                )
+                                        .lower()
-                                episode_guid_id = re.search(
+                                    )
-                                    r"://(.*)", episode_guid.id
+                                    episode_guid_id = re.search(
-                                ).group(1)
+                                        r"://(.*)", episode_guid.id
+                                    ).group(1)
 
-                                # If episode provider source and episode provider id are in videos_episodes_ids exactly, then the episode is in the list
+                                    # If episode provider source and episode provider id are in videos_episodes_ids exactly, then the episode is in the list
-                                if episode_guid_source in videos_episodes_ids.keys():
+                                    if episode_guid_source in videos_episodes_ids.keys():
-                                    if (
+                                        if (
-                                        episode_guid_id
+                                            episode_guid_id
-                                        in videos_episodes_ids[episode_guid_source]
+                                            in videos_episodes_ids[episode_guid_source]
-                                    ):
+                                        ):
-                                        episode_found = True
+                                            episode_found = True
-                                        break
+                                            break
+                            except Exception as e:
+                                logger(
+                                    f"Plex: Failed to get episode guid for {episode_search.title}, Error: {e}",
+                                    1,
+                                )
 
                         if episode_found:
                             msg = f"{show_search.title} {episode_search.title} as watched for {user.title} in {library} for Plex"
@@ -249,7 +277,7 @@ def update_user_watched(user, user_plex, library, videos, dryrun):
             f"Plex: Failed to update watched for {user.title} in library {library}, Error: {e}",
             2,
         )
-        raise Exception(e)
+        logger(traceback.format_exc(), 2)
 
 
 # class plex accept base url and token and username and password but default with none