bel
/
JellyPlex-Watched
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix server 2 always running async runner. Speedup plex get watched

pull/33/head
Luigi311 2022-12-18 22:27:42 -07:00
parent 251937431b
commit d0746cec5a
5 changed files with 185 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/functions.py
View File

@ -178,28 +178,6 @@ def combine_watched_dicts(dicts: list):
return combined_dict return combined_dict
def future_thread_executor(args: list, workers: int = -1):
futures_list = []
results = []
if workers == -1:
workers = min(32, os.cpu_count() * 1.25)
with ThreadPoolExecutor(max_workers=workers) as executor:
for arg in args:
# * arg unpacks the list into actual arguments
futures_list.append(executor.submit(*arg))
for future in futures_list:
try:
result = future.result()
results.append(result)
except Exception as e:
raise Exception(e)
return results
def cleanup_watched( def cleanup_watched(
watched_list_1, watched_list_2, user_mapping=None, library_mapping=None watched_list_1, watched_list_2, user_mapping=None, library_mapping=None
): ):
@ -326,4 +304,25 @@ def is_episode_in_dict(episode, episode_watched_list_2_keys_dict):
return True return True
# If the loop completes without finding a match, return False # If the loop completes without finding a match, return False
return False return False
def future_thread_executor(args: list, workers: int = -1):
futures_list = []
results = []
if workers == -1:
workers = min(32, os.cpu_count() * 2)
with ThreadPoolExecutor(max_workers=workers) as executor:
for arg in args:
# * arg unpacks the list into actual arguments
futures_list.append(executor.submit(*arg))
for future in futures_list:
try:
result = future.result()
results.append(result)
except Exception as e:
raise Exception(e)
return results

14
src/jellyfin.py
View File

@ -239,7 +239,8 @@ class Jellyfin:
][episode_identifiers["season_name"]].append( ][episode_identifiers["season_name"]].append(
episode_guids episode_guids
) )
logger(f"Jellyfin: Got watched for {user_name} in library {library_title}")
return user_watched return user_watched
except Exception as e: except Exception as e:
logger( logger(
@ -342,7 +343,7 @@ class Jellyfin:
for user_name, user_id in users.items(): for user_name, user_id in users.items():
watched.append( watched.append(
await self.get_users_watched( self.get_users_watched(
user_name, user_name,
user_id, user_id,
blacklist_library, blacklist_library,
@ -353,6 +354,7 @@ class Jellyfin:
) )
) )
watched = await asyncio.gather(*watched, return_exceptions=True)
for user_watched in watched: for user_watched in watched:
user_watched_temp = combine_watched_dicts(user_watched) user_watched_temp = combine_watched_dicts(user_watched)
for user, user_watched_temp in user_watched_temp.items(): for user, user_watched_temp in user_watched_temp.items():
@ -534,12 +536,12 @@ class Jellyfin:
else: else:
logger( logger(
f"Jellyfin: Skipping episode {jellyfin_episode['Name']} as it is not in mark list for {user_name}", f"Jellyfin: Skipping episode {jellyfin_episode['Name']} as it is not in mark list for {user_name}",
1, 3,
) )
else: else:
logger( logger(
f"Jellyfin: Skipping show {jellyfin_show['Name']} as it is not in mark list for {user_name}", f"Jellyfin: Skipping show {jellyfin_show['Name']} as it is not in mark list for {user_name}",
1, 3,
) )
if ( if (
@ -618,13 +620,13 @@ class Jellyfin:
else: else:
logger( logger(
f"Jellyfin: Library {library} or {library_other} not found in library list", f"Jellyfin: Library {library} or {library_other} not found in library list",
2, 1,
) )
continue continue
else: else:
logger( logger(
f"Jellyfin: Library {library} not found in library list", f"Jellyfin: Library {library} not found in library list",
2, 1,
) )
continue continue

58
src/main.py
View File

@ -310,6 +310,32 @@ def generate_server_connections():
return servers return servers
def get_server_watched(server_connection: list, users: dict, blacklist_library: list, whitelist_library: list, blacklist_library_type: list, whitelist_library_type: list, library_mapping: dict):
if server_connection[0] == "plex":
return server_connection[1].get_watched(
users,
blacklist_library,
whitelist_library,
blacklist_library_type,
whitelist_library_type,
library_mapping,
)
elif server_connection[0] == "jellyfin":
return asyncio.run(server_connection[1].get_watched(
users,
blacklist_library,
whitelist_library,
blacklist_library_type,
whitelist_library_type,
library_mapping,
))
def update_server_watched(server_connection: list, server_watched_filtered: dict, user_mapping: dict, library_mapping: dict, dryrun: bool):
if server_connection[0] == "plex":
server_connection[1].update_watched(server_watched_filtered, user_mapping, library_mapping, dryrun)
elif server_connection[0] == "jellyfin":
asyncio.run(server_connection[1].update_watched(server_watched_filtered, user_mapping, library_mapping, dryrun))
def main_loop(): def main_loop():
logfile = os.getenv("LOGFILE", "log.log") logfile = os.getenv("LOGFILE", "log.log")
@ -379,7 +405,8 @@ def main_loop():
) )
logger("Creating watched lists", 1) logger("Creating watched lists", 1)
server_1_watched = server_1_connection.get_watched( server_1_watched = get_server_watched(
server_1,
server_1_users, server_1_users,
blacklist_library, blacklist_library,
whitelist_library, whitelist_library,
@ -388,15 +415,14 @@ def main_loop():
library_mapping, library_mapping,
) )
logger("Finished creating watched list server 1", 1) logger("Finished creating watched list server 1", 1)
server_2_watched = asyncio.run( server_2_watched = get_server_watched(
server_2_connection.get_watched( server_2,
server_2_users, server_2_users,
blacklist_library, blacklist_library,
whitelist_library, whitelist_library,
blacklist_library_type, blacklist_library_type,
whitelist_library_type, whitelist_library_type,
library_mapping, library_mapping,
)
) )
logger("Finished creating watched list server 2", 1) logger("Finished creating watched list server 2", 1)
logger(f"Server 1 watched: {server_1_watched}", 3) logger(f"Server 1 watched: {server_1_watched}", 3)
@ -421,13 +447,12 @@ def main_loop():
1, 1,
) )
server_1_connection.update_watched( update_server_watched(
server_2_watched_filtered, user_mapping, library_mapping, dryrun server_1, server_2_watched_filtered, user_mapping, library_mapping, dryrun
) )
asyncio.run(
server_2_connection.update_watched( update_server_watched(
server_1_watched_filtered, user_mapping, library_mapping, dryrun server_2, server_1_watched_filtered, user_mapping, library_mapping, dryrun
)
) )
@ -455,6 +480,7 @@ def main():
logger(error, log_type=2) logger(error, log_type=2)
logger(traceback.format_exc(), 2) logger(traceback.format_exc(), 2)
logger(f"Retrying in {sleep_duration}", log_type=0) logger(f"Retrying in {sleep_duration}", log_type=0)
sleep(sleep_duration) sleep(sleep_duration)

77
src/plex.py
View File

@ -54,14 +54,13 @@ def get_user_watched(user, user_plex, library):
user_watched[user_name][library.title] = {} user_watched[user_name][library.title] = {}
library_videos = user_plex.library.section(library.title) library_videos = user_plex.library.section(library.title)
for show in library_videos.search(unwatched=False): shows = library_videos.search(unwatched=False)
for show in shows:
show_guids = {} show_guids = {}
for show_guid in show.guids: for show_guid in show.guids:
# Extract after :// from guid.id # Extract source and id from guid.id
show_guid_source = ( m = re.match(r"(.*)://(.*)", show_guid.id)
re.search(r"(.*)://", show_guid.id).group(1).lower() show_guid_source, show_guid_id = m.group(1).lower(), m.group(2)
)
show_guid_id = re.search(r"://(.*)", show_guid.id).group(1)
show_guids[show_guid_source] = show_guid_id show_guids[show_guid_source] = show_guid_id
show_guids["title"] = show.title show_guids["title"] = show.title
@ -69,40 +68,33 @@ def get_user_watched(user, user_plex, library):
[x.split("/")[-1] for x in show.locations] [x.split("/")[-1] for x in show.locations]
) )
show_guids = frozenset(show_guids.items()) show_guids = frozenset(show_guids.items())
# Get all watched episodes for show
episode_guids = {}
for episode in show.watched():
if episode.viewCount > 0:
episode_guids_temp = {}
for guid in episode.guids:
# Extract after :// from guid.id
m = re.match(r"(.*)://(.*)", guid.id)
guid_source, guid_id = m.group(1).lower(), m.group(2)
episode_guids_temp[guid_source] = guid_id
for season in show.seasons(): episode_guids_temp["locations"] = tuple(
episode_guids = [] [x.split("/")[-1] for x in episode.locations]
for episode in season.episodes(): )
if episode.viewCount > 0: if episode.parentTitle not in episode_guids:
episode_guids_temp = {} episode_guids[episode.parentTitle] = []
for guid in episode.guids: episode_guids[episode.parentTitle].append(episode_guids_temp)
# Extract after :// from guid.id
guid_source = ( if episode_guids:
re.search(r"(.*)://", guid.id).group(1).lower() # append show, season, episode
) if show_guids not in user_watched[user_name][library.title]:
guid_id = re.search(r"://(.*)", guid.id).group(1) user_watched[user_name][library.title][show_guids] = {}
episode_guids_temp[guid_source] = guid_id
user_watched[user_name][library.title][show_guids] = episode_guids
episode_guids_temp["locations"] = tuple(
[x.split("/")[-1] for x in episode.locations]
)
episode_guids.append(episode_guids_temp)
if episode_guids:
# append show, season, episode
if show_guids not in user_watched[user_name][library.title]:
user_watched[user_name][library.title][show_guids] = {}
if (
season.title
not in user_watched[user_name][library.title][show_guids]
):
user_watched[user_name][library.title][show_guids][
season.title
] = {}
user_watched[user_name][library.title][show_guids][
season.title
] = episode_guids
logger(f"Plex: Got watched for {user_name} in library {library.title}", 0)
return user_watched return user_watched
except Exception as e: except Exception as e:
logger( logger(
@ -223,12 +215,12 @@ def update_user_watched(user, user_plex, library, videos, dryrun):
else: else:
logger( logger(
f"Plex: Skipping episode {episode_search.title} as it is not in mark list for {user.title}", f"Plex: Skipping episode {episode_search.title} as it is not in mark list for {user.title}",
1, 3,
) )
else: else:
logger( logger(
f"Plex: Skipping show {show_search.title} as it is not in mark list for {user.title}", f"Plex: Skipping show {show_search.title} as it is not in mark list for {user.title}",
1, 3,
) )
if not videos_movies_ids and not videos_shows_ids and not videos_episodes_ids: if not videos_movies_ids and not videos_shows_ids and not videos_episodes_ids:
@ -415,12 +407,13 @@ class Plex:
else: else:
logger( logger(
f"Plex: Library {library} or {library_other} not found in library list", f"Plex: Library {library} or {library_other} not found in library list",
2, 1,
) )
continue continue
else: else:
logger( logger(
f"Plex: Library {library} not found in library list", 2 f"Plex: Library {library} not found in library list",
1,
) )
continue continue

156
test/test_main.py
View File

@ -1,78 +1,78 @@
import sys import sys
import os import os
# getting the name of the directory # getting the name of the directory
# where the this file is present. # where the this file is present.
current = os.path.dirname(os.path.realpath(__file__)) current = os.path.dirname(os.path.realpath(__file__))
# Getting the parent directory name # Getting the parent directory name
# where the current directory is present. # where the current directory is present.
parent = os.path.dirname(current) parent = os.path.dirname(current)
# adding the parent directory to # adding the parent directory to
# the sys.path. # the sys.path.
sys.path.append(parent) sys.path.append(parent)
from src.main import setup_black_white_lists from src.main import setup_black_white_lists
def test_setup_black_white_lists(): def test_setup_black_white_lists():
# Simple # Simple
blacklist_library = "library1, library2" blacklist_library = "library1, library2"
whitelist_library = "library1, library2" whitelist_library = "library1, library2"
blacklist_library_type = "library_type1, library_type2" blacklist_library_type = "library_type1, library_type2"
whitelist_library_type = "library_type1, library_type2" whitelist_library_type = "library_type1, library_type2"
blacklist_users = "user1, user2" blacklist_users = "user1, user2"
whitelist_users = "user1, user2" whitelist_users = "user1, user2"
( (
results_blacklist_library, results_blacklist_library,
return_whitelist_library, return_whitelist_library,
return_blacklist_library_type, return_blacklist_library_type,
return_whitelist_library_type, return_whitelist_library_type,
return_blacklist_users, return_blacklist_users,
return_whitelist_users, return_whitelist_users,
) = setup_black_white_lists( ) = setup_black_white_lists(
blacklist_library, blacklist_library,
whitelist_library, whitelist_library,
blacklist_library_type, blacklist_library_type,
whitelist_library_type, whitelist_library_type,
blacklist_users, blacklist_users,
whitelist_users, whitelist_users,
) )
assert results_blacklist_library == ["library1", "library2"] assert results_blacklist_library == ["library1", "library2"]
assert return_whitelist_library == ["library1", "library2"] assert return_whitelist_library == ["library1", "library2"]
assert return_blacklist_library_type == ["library_type1", "library_type2"] assert return_blacklist_library_type == ["library_type1", "library_type2"]
assert return_whitelist_library_type == ["library_type1", "library_type2"] assert return_whitelist_library_type == ["library_type1", "library_type2"]
assert return_blacklist_users == ["user1", "user2"] assert return_blacklist_users == ["user1", "user2"]
assert return_whitelist_users == ["user1", "user2"] assert return_whitelist_users == ["user1", "user2"]
# Library Mapping and user mapping # Library Mapping and user mapping
library_mapping = {"library1": "library3"} library_mapping = {"library1": "library3"}
user_mapping = {"user1": "user3"} user_mapping = {"user1": "user3"}
( (
results_blacklist_library, results_blacklist_library,
return_whitelist_library, return_whitelist_library,
return_blacklist_library_type, return_blacklist_library_type,
return_whitelist_library_type, return_whitelist_library_type,
return_blacklist_users, return_blacklist_users,
return_whitelist_users, return_whitelist_users,
) = setup_black_white_lists( ) = setup_black_white_lists(
blacklist_library, blacklist_library,
whitelist_library, whitelist_library,
blacklist_library_type, blacklist_library_type,
whitelist_library_type, whitelist_library_type,
blacklist_users, blacklist_users,
whitelist_users, whitelist_users,
library_mapping, library_mapping,
user_mapping, user_mapping,
) )
assert results_blacklist_library == ["library1", "library2", "library3"] assert results_blacklist_library == ["library1", "library2", "library3"]
assert return_whitelist_library == ["library1", "library2", "library3"] assert return_whitelist_library == ["library1", "library2", "library3"]
assert return_blacklist_library_type == ["library_type1", "library_type2"] assert return_blacklist_library_type == ["library_type1", "library_type2"]
assert return_whitelist_library_type == ["library_type1", "library_type2"] assert return_whitelist_library_type == ["library_type1", "library_type2"]
assert return_blacklist_users == ["user1", "user2", "user3"] assert return_blacklist_users == ["user1", "user2", "user3"]
assert return_whitelist_users == ["user1", "user2", "user3"] assert return_whitelist_users == ["user1", "user2", "user3"]