feat(api): plex Sync (Movies)

Also adds winston logging

server/api/plexapi.ts

@@ -1,8 +1,49 @@
 import NodePlexAPI from 'plex-api';
 import { getSettings } from '../lib/settings';
 
+export interface PlexLibraryItem {
+  ratingKey: string;
+  title: string;
+  guid: string;
+  type: 'movie' | 'show';
+}
+
+interface PlexLibraryResponse {
+  MediaContainer: {
+    Metadata: PlexLibraryItem[];
+  };
+}
+
+export interface PlexLibrary {
+  type: 'show' | 'movie';
+  key: string;
+  title: string;
+}
+
+interface PlexLibrariesResponse {
+  MediaContainer: {
+    Directory: PlexLibrary[];
+  };
+}
+
+export interface PlexMetadata {
+  ratingKey: string;
+  guid: string;
+  type: 'movie' | 'show';
+  title: string;
+  Guid: {
+    id: string;
+  }[];
+}
+
+interface PlexMetadataResponse {
+  MediaContainer: {
+    Metadata: PlexMetadata[];
+  };
+}
+
 class PlexAPI {
-  private plexClient: typeof NodePlexAPI;
+  private plexClient: NodePlexAPI;
 
   constructor({ plexToken }: { plexToken?: string }) {
     const settings = getSettings();
@@ -13,7 +54,7 @@ class PlexAPI {
       token: plexToken,
       authenticator: {
         authenticate: (
-          _plexApi: typeof PlexAPI,
+          _plexApi,
           cb: (err?: string, token?: string) => void
         ) => {
           if (!plexToken) {
@@ -34,6 +75,36 @@ class PlexAPI {
   public async getStatus() {
     return await this.plexClient.query('/');
   }
+
+  public async getLibraries(): Promise<PlexLibrary[]> {
+    const response = await this.plexClient.query<PlexLibrariesResponse>(
+      '/library/sections'
+    );
+
+    return response.MediaContainer.Directory;
+  }
+
+  public async getLibraryContents(id: string): Promise<PlexLibraryItem[]> {
+    const response = await this.plexClient.query<PlexLibraryResponse>(
+      `/library/sections/${id}/all`
+    );
+
+    return response.MediaContainer.Metadata;
+  }
+
+  public async getMetadata(key: string): Promise<PlexMetadata> {
+    const response = await this.plexClient.query<PlexMetadataResponse>(
+      `/library/metadata/${key}`
+    );
+
+    return response.MediaContainer.Metadata[0];
+  }
+
+  public async getRecentlyAdded() {
+    const response = await this.plexClient.query('/library/recentlyAdded');
+
+    return response;
+  }
 }
 
 export default PlexAPI;

server/api/plextv.ts

@@ -1,6 +1,7 @@
 import axios, { AxiosInstance } from 'axios';
 import xml2js from 'xml2js';
 import { getSettings } from '../lib/settings';
+import logger from '../logger';
 
 interface PlexAccountResponse {
   user: PlexUser;
@@ -79,9 +80,9 @@ class PlexTvAPI {
 
       return account.data.user;
     } catch (e) {
-      console.error(
-        'Something broke when getting account from plex.tv',
-        e.message
+      logger.error(
+        `Something went wrong getting the account from plex.tv: ${e.message}`,
+        { label: 'Plex.tv API' }
       );
       throw new Error('Invalid auth token');
     }
@@ -124,7 +125,7 @@ class PlexTvAPI {
         (server) => server.$.machineIdentifier === settings.plex.machineId
       );
     } catch (e) {
-      console.log(`Error checking user access: ${e.message}`);
+      logger.error(`Error checking user access: ${e.message}`);
       return false;
     }
   }

server/api/themoviedb.ts

@@ -100,6 +100,11 @@ interface TmdbSearchTvResponse extends TmdbPaginatedResponse {
   results: TmdbTvResult[];
 }
 
+interface TmdbExternalIdResponse {
+  movie_results: TmdbMovieResult[];
+  tv_results: TmdbTvResult[];
+}
+
 export interface TmdbCreditCast {
   cast_id: number;
   character: string;
@@ -549,6 +554,70 @@ class TheMovieDb {
       throw new Error(`[TMDB] Failed to fetch all trending: ${e.message}`);
     }
   };
+
+  public async getByExternalId({
+    externalId,
+    type,
+    language = 'en-US',
+  }:
+    | {
+        externalId: string;
+        type: 'imdb';
+        language?: string;
+      }
+    | {
+        externalId: number;
+        type: 'tvdb';
+        language?: string;
+      }): Promise<TmdbExternalIdResponse> {
+    try {
+      const response = await this.axios.get<TmdbExternalIdResponse>(
+        `/find/${externalId}`,
+        {
+          params: {
+            external_source: type === 'imdb' ? 'imdb_id' : 'tvdb_id',
+            language,
+          },
+        }
+      );
+
+      return response.data;
+    } catch (e) {
+      throw new Error(`[TMDB] Failed to find by external ID: ${e.message}`);
+    }
+  }
+
+  public async getMovieByImdbId({
+    imdbId,
+    language = 'en-US',
+  }: {
+    imdbId: string;
+    language?: string;
+  }): Promise<TmdbMovieDetails> {
+    try {
+      const extResponse = await this.getByExternalId({
+        externalId: imdbId,
+        type: 'imdb',
+      });
+
+      if (extResponse.movie_results[0]) {
+        const movie = await this.getMovie({
+          movieId: extResponse.movie_results[0].id,
+          language,
+        });
+
+        return movie;
+      }
+
+      throw new Error(
+        '[TMDB] Failed to find a title with the provided IMDB id'
+      );
+    } catch (e) {
+      throw new Error(
+        `[TMDB] Failed to get movie by external imdb ID: ${e.message}`
+      );
+    }
+  }
 }
 
 export default TheMovieDb;

server/entity/Media.ts

@@ -11,6 +11,7 @@ import {
 } from 'typeorm';
 import { MediaRequest } from './MediaRequest';
 import { MediaStatus, MediaType } from '../constants/media';
+import logger from '../logger';
 
 @Entity()
 class Media {
@@ -33,7 +34,7 @@ class Media {
 
       return media;
     } catch (e) {
-      console.error(e.messaage);
+      logger.error(e.message);
       return [];
     }
   }
@@ -48,7 +49,7 @@ class Media {
 
       return media;
     } catch (e) {
-      console.error(e.messaage);
+      logger.error(e.messaage);
       return undefined;
     }
   }
@@ -65,7 +66,11 @@ class Media {
 
   @Column({ unique: true, nullable: true })
   @Index()
-  public tvdbId: number;
+  public tvdbId?: number;
+
+  @Column({ unique: true, nullable: true })
+  @Index()
+  public imdbId?: string;
 
   @Column({ type: 'int', default: MediaStatus.UNKNOWN })
   public status: MediaStatus;

server/entity/MovieRequest.ts

@@ -4,6 +4,7 @@ import TheMovieDb from '../api/themoviedb';
 import RadarrAPI from '../api/radarr';
 import { getSettings } from '../lib/settings';
 import { MediaType, MediaRequestStatus } from '../constants/media';
+import logger from '../logger';
 
 @ChildEntity(MediaType.MOVIE)
 class MovieRequest extends MediaRequest {
@@ -18,8 +19,9 @@ class MovieRequest extends MediaRequest {
       try {
         const settings = getSettings();
         if (settings.radarr.length === 0 && !settings.radarr[0]) {
-          console.log(
-            '[MediaRequest] Skipped radarr request as there is no radarr configured'
+          logger.info(
+            'Skipped radarr request as there is no radarr configured',
+            { label: 'Media Request' }
           );
           return;
         }
@@ -44,7 +46,7 @@ class MovieRequest extends MediaRequest {
           monitored: true,
           searchNow: true,
         });
-        console.log('[MediaRequest] Sent request to Radarr');
+        logger.info('Sent request to Radarr', { label: 'Media Request' });
       } catch (e) {
         throw new Error(
           `[MediaRequest] Request failed to send to radarr: ${e.message}`

server/index.ts

@@ -12,6 +12,7 @@ import swaggerUi from 'swagger-ui-express';
 import { OpenApiValidator } from 'express-openapi-validator';
 import { Session } from './entity/Session';
 import { getSettings } from './lib/settings';
+import logger from './logger';
 
 const API_SPEC_PATH = path.join(__dirname, 'overseerr-api.yml');
 
@@ -40,9 +41,12 @@ app
         secret: 'verysecret',
         resave: false,
         saveUninitialized: false,
+        cookie: {
+          maxAge: 1000 * 60 * 60 * 24 * 30,
+        },
         store: new TypeormStore({
           cleanupLimit: 2,
-          ttl: 86400,
+          ttl: 1000 * 60 * 60 * 24 * 30,
         }).connect(sessionRespository),
       })
     );
@@ -87,10 +91,12 @@ app
       if (err) {
         throw err;
       }
-      console.log(`Ready to do stuff http://localhost:${port}`);
+      logger.info(`Server ready on port ${port}`, {
+        label: 'SERVER',
+      });
     });
   })
   .catch((err) => {
-    console.error(err.stack);
+    logger.error(err.stack);
     process.exit(1);
   });

server/job/plexsync.ts

@@ -0,0 +1,182 @@
+import { getRepository } from 'typeorm';
+import { User } from '../entity/User';
+import PlexAPI, { PlexLibraryItem } from '../api/plexapi';
+import TheMovieDb from '../api/themoviedb';
+import Media from '../entity/Media';
+import { MediaStatus, MediaType } from '../constants/media';
+import logger from '../logger';
+import { getSettings, Library } from '../lib/settings';
+import { resolve } from 'dns';
+
+const BUNDLE_SIZE = 10;
+
+const imdbRegex = new RegExp(/imdb:\/\/(tt[0-9]+)/);
+const tmdbRegex = new RegExp(/tmdb:\/\/([0-9]+)/);
+const plexRegex = new RegExp(/plex:\/\//);
+
+class JobPlexSync {
+  private tmdb: TheMovieDb;
+  private plexClient: PlexAPI;
+  private items: PlexLibraryItem[] = [];
+  private progress = 0;
+  private libraries: Library[];
+  private currentLibrary: Library;
+  private running = false;
+
+  constructor() {
+    this.tmdb = new TheMovieDb();
+  }
+
+  private async getExisting(tmdbId: number) {
+    const mediaRepository = getRepository(Media);
+
+    const existing = await mediaRepository.findOne({
+      where: { tmdbId: tmdbId },
+    });
+
+    return existing;
+  }
+
+  private async processMovie(plexitem: PlexLibraryItem) {
+    const mediaRepository = getRepository(Media);
+    if (plexitem.guid.match(plexRegex)) {
+      const metadata = await this.plexClient.getMetadata(plexitem.ratingKey);
+      const newMedia = new Media();
+
+      metadata.Guid.forEach((ref) => {
+        if (ref.id.match(imdbRegex)) {
+          newMedia.imdbId = ref.id.match(imdbRegex)?.[1] ?? undefined;
+        } else if (ref.id.match(tmdbRegex)) {
+          const tmdbMatch = ref.id.match(tmdbRegex)?.[1];
+          newMedia.tmdbId = Number(tmdbMatch);
+        }
+      });
+
+      const existing = await this.getExisting(newMedia.tmdbId);
+
+      if (existing && existing.status === MediaStatus.AVAILABLE) {
+        this.log(`Title exists and is already available ${metadata.title}`);
+      } else if (existing && existing.status !== MediaStatus.AVAILABLE) {
+        existing.status = MediaStatus.AVAILABLE;
+        mediaRepository.save(existing);
+        this.log(
+          `Request for ${metadata.title} exists. Setting status AVAILABLE`
+        );
+      } else {
+        newMedia.status = MediaStatus.AVAILABLE;
+        newMedia.mediaType = MediaType.MOVIE;
+        await mediaRepository.save(newMedia);
+        this.log(`Saved ${plexitem.title}`);
+      }
+    } else {
+      const matchedid = plexitem.guid.match(/imdb:\/\/(tt[0-9]+)/);
+
+      if (matchedid?.[1]) {
+        const tmdbMovie = await this.tmdb.getMovieByImdbId({
+          imdbId: matchedid[1],
+        });
+
+        const existing = await this.getExisting(tmdbMovie.id);
+        if (existing && existing.status === MediaStatus.AVAILABLE) {
+          this.log(`Title exists and is already available ${plexitem.title}`);
+        } else if (existing && existing.status !== MediaStatus.AVAILABLE) {
+          existing.status = MediaStatus.AVAILABLE;
+          await mediaRepository.save(existing);
+          this.log(
+            `Request for ${plexitem.title} exists. Setting status AVAILABLE`
+          );
+        } else if (tmdbMovie) {
+          const newMedia = new Media();
+          newMedia.imdbId = tmdbMovie.external_ids.imdb_id;
+          newMedia.tmdbId = tmdbMovie.id;
+          newMedia.status = MediaStatus.AVAILABLE;
+          newMedia.mediaType = MediaType.MOVIE;
+          await mediaRepository.save(newMedia);
+          this.log(`Saved ${tmdbMovie.title}`);
+        }
+      }
+    }
+  }
+
+  private async processItems(slicedItems: PlexLibraryItem[]) {
+    await Promise.all(
+      slicedItems.map(async (plexitem) => {
+        if (plexitem.type === 'movie') {
+          await this.processMovie(plexitem);
+        }
+      })
+    );
+  }
+
+  private async loop({
+    start = 0,
+    end = BUNDLE_SIZE,
+  }: {
+    start?: number;
+    end?: number;
+  } = {}) {
+    const slicedItems = this.items.slice(start, end);
+    if (start < this.items.length && this.running) {
+      this.progress = start;
+      await this.processItems(slicedItems);
+
+      await new Promise((resolve) =>
+        setTimeout(async () => {
+          await this.loop({
+            start: start + BUNDLE_SIZE,
+            end: end + BUNDLE_SIZE,
+          });
+          resolve();
+        }, 5000)
+      );
+    }
+  }
+
+  private log(message: string): void {
+    logger.info(message, { label: 'Plex Sync' });
+  }
+
+  public async run(): Promise<void> {
+    const settings = getSettings();
+    if (!this.running) {
+      this.running = true;
+      const userRepository = getRepository(User);
+      const admin = await userRepository.findOneOrFail({
+        select: ['id', 'plexToken'],
+        order: { id: 'ASC' },
+      });
+
+      this.plexClient = new PlexAPI({ plexToken: admin.plexToken });
+      this.libraries = settings.plex.libraries.filter(
+        (library) => library.enabled
+      );
+
+      for (const library of this.libraries) {
+        this.currentLibrary = library;
+        this.log(`Beginning to process library: ${library.name}`);
+        this.items = await this.plexClient.getLibraryContents(library.id);
+        await this.loop();
+      }
+      this.running = false;
+      this.log('complete');
+    }
+  }
+
+  public status() {
+    return {
+      running: this.running,
+      progress: this.progress,
+      total: this.items.length,
+      currentLibrary: this.currentLibrary,
+      libraries: this.libraries,
+    };
+  }
+
+  public cancel(): void {
+    this.running = false;
+  }
+}
+
+const jobPlexSync = new JobPlexSync();
+
+export default jobPlexSync;

server/lib/settings.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import { v4 as uuidv4 } from 'uuid';
 
-interface Library {
+export interface Library {
   id: string;
   name: string;
   enabled: boolean;

server/logger.ts

@@ -0,0 +1,32 @@
+import * as winston from 'winston';
+import path from 'path';
+
+const hformat = winston.format.printf(
+  ({ level, label, message, timestamp, ...metadata }) => {
+    let msg = `${timestamp} [${level}]${
+      label ? `[${label}]` : ''
+    }: ${message} `;
+    if (Object.keys(metadata).length > 0) {
+      msg += JSON.stringify(metadata);
+    }
+    return msg;
+  }
+);
+
+const logger = winston.createLogger({
+  level: process.env.LOG_LEVEL || 'debug',
+  format: winston.format.combine(
+    winston.format.colorize(),
+    winston.format.splat(),
+    winston.format.timestamp(),
+    hformat
+  ),
+  transports: [
+    new winston.transports.Console(),
+    new winston.transports.File({
+      filename: path.join(__dirname, '../config/logs/overseerr.log'),
+    }),
+  ],
+});
+
+export default logger;

server/overseerr-api.yml

@@ -770,6 +770,69 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/PlexSettings'
+  /settings/plex/library:
+    get:
+      summary: Get a list of current plex libraries
+      description: Returns a list of plex libraries in a JSON array
+      tags:
+        - settings
+      parameters:
+        - in: query
+          name: sync
+          description: Syncs the current libraries with the current plex server
+          schema:
+            type: string
+            nullable: true
+        - in: query
+          name: enable
+          description: Comma separated list of libraries to enable. Any libraries not passed will be disabled!
+          schema:
+            type: string
+            nullable: true
+      responses:
+        '200':
+          description: 'Plex libraries returned'
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/PlexLibrary'
+  /settings/plex/sync:
+    get:
+      summary: Start a full Plex Library sync
+      description: Runs a full plex library sync and returns the progress in a JSON array
+      tags:
+        - settings
+      parameters:
+        - in: query
+          name: cancel
+          schema:
+            type: boolean
+            example: false
+      responses:
+        '200':
+          description: Status of Plex Sync
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  running:
+                    type: boolean
+                    example: false
+                  progress:
+                    type: number
+                    example: 0
+                  total:
+                    type: number
+                    example: 100
+                  currentLibrary:
+                    $ref: '#/components/schemas/PlexLibrary'
+                  libraries:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/PlexLibrary'
   /settings/radarr:
     get:
       summary: Get all radarr settings

server/routes/auth.ts

@@ -4,6 +4,7 @@ import { User } from '../entity/User';
 import PlexTvAPI from '../api/plextv';
 import { isAuthenticated } from '../middleware/auth';
 import { Permission } from '../lib/permissions';
+import logger from '../logger';
 
 const authRoutes = Router();
 
@@ -95,7 +96,7 @@ authRoutes.post('/login', async (req, res) => {
 
     return res.status(200).json(user?.filter() ?? {});
   } catch (e) {
-    console.error(e);
+    logger.error(e.message, { label: 'Auth' });
     res
       .status(500)
       .json({ error: 'Something went wrong. Is your auth token valid?' });

server/routes/settings.ts

@@ -1,8 +1,14 @@
 import { Router } from 'express';
-import { getSettings, RadarrSettings, SonarrSettings } from '../lib/settings';
+import {
+  getSettings,
+  RadarrSettings,
+  SonarrSettings,
+  Library,
+} from '../lib/settings';
 import { getRepository } from 'typeorm';
 import { User } from '../entity/User';
-import PlexAPI from '../api/plexapi';
+import PlexAPI, { PlexLibrary } from '../api/plexapi';
+import jobPlexSync from '../job/plexsync';
 
 const settingsRoutes = Router();
 
@@ -58,6 +64,55 @@ settingsRoutes.post('/plex', async (req, res, next) => {
   return res.status(200).json(settings.plex);
 });
 
+settingsRoutes.get('/plex/library', async (req, res) => {
+  const settings = getSettings();
+
+  if (req.query.sync) {
+    const userRepository = getRepository(User);
+    const admin = await userRepository.findOneOrFail({
+      select: ['id', 'plexToken'],
+      order: { id: 'ASC' },
+    });
+    const plexapi = new PlexAPI({ plexToken: admin.plexToken });
+
+    const libraries = await plexapi.getLibraries();
+
+    const newLibraries: Library[] = libraries.map((library) => {
+      const existing = settings.plex.libraries.find(
+        (l) => l.id === library.key
+      );
+
+      return {
+        id: library.key,
+        name: library.title,
+        enabled: existing?.enabled ?? false,
+      };
+    });
+
+    settings.plex.libraries = newLibraries;
+  }
+
+  const enabledLibraries = req.query.enable
+    ? (req.query.enable as string).split(',')
+    : [];
+  settings.plex.libraries = settings.plex.libraries.map((library) => ({
+    ...library,
+    enabled: enabledLibraries.includes(library.id),
+  }));
+  settings.save();
+  return res.status(200).json(settings.plex.libraries);
+});
+
+settingsRoutes.get('/plex/sync', (req, res) => {
+  if (req.query.cancel) {
+    jobPlexSync.cancel();
+  } else {
+    jobPlexSync.run();
+  }
+
+  return res.status(200).json(jobPlexSync.status());
+});
+
 settingsRoutes.get('/radarr', (req, res) => {
   const settings = getSettings();
 

server/types/plex-api.d.ts

@@ -1 +1,23 @@
-declare module 'plex-api';
+declare module 'plex-api' {
+  export default class PlexAPI {
+    constructor(intiialOptions: {
+      hostname: string;
+      post: number;
+      token?: string;
+      authenticator: {
+        authenticate: (
+          _plexApi: PlexAPI,
+          cb: (err?: string, token?: string) => void
+        ) => void;
+      };
+      options: {
+        identifier: string;
+        product: string;
+        deviceName: string;
+        platform: string;
+      };
+    });
+
+    query: <T extends Record<string, any>>(endpoint: string) => Promise<T>;
+  }
+}