From 7a0c865bb2bfa39e818f33cd1a588856058ff176 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Wed, 11 Jun 2025 14:51:55 -0500 Subject: [PATCH] Receivers: Multiple MediaCache fixes --- receivers/common/web/MediaCache.ts | 298 ++++++++++++++++--------- receivers/common/web/UtilityBackend.ts | 20 +- receivers/electron/src/Main.ts | 21 +- receivers/electron/src/Updater.ts | 2 +- 4 files changed, 208 insertions(+), 133 deletions(-) diff --git a/receivers/common/web/MediaCache.ts b/receivers/common/web/MediaCache.ts index 428d21c..aa92b64 100644 --- a/receivers/common/web/MediaCache.ts +++ b/receivers/common/web/MediaCache.ts @@ -23,17 +23,28 @@ class CacheObject { export class MediaCache { private static instance: MediaCache = null; - private cache = new Map(); - private cacheUrlMap = new Map(); + private cache: Map; + private cacheUrlMap: Map; private playlist: PlaylistContent; + private playlistIndex: number; private quota: number; - private cacheSize: number = 0; - private cacheWindowStart: number = 0; - private cacheWindowEnd: number = 0; + private cacheSize: number; + private cacheWindowStart: number; + private cacheWindowEnd: number; + private pendingDownloads: Set; + private isDownloading: boolean; constructor(playlist: PlaylistContent) { MediaCache.instance = this; this.playlist = playlist; + this.playlistIndex = playlist.offset ? playlist.offset : 0; + this.cache = new Map(); + this.cacheUrlMap = new Map(); + this.cacheSize = 0; + this.cacheWindowStart = 0; + this.cacheWindowEnd = 0; + this.pendingDownloads = new Set(); + this.isDownloading = false; if (!fs.existsSync('/cache')) { fs.mkdirSync('/cache'); @@ -41,7 +52,8 @@ export class MediaCache { // @ts-ignore if (TARGET === 'electron') { - this.quota = Math.min(Math.floor(os.freemem() / 4), 4 * 1024 * 1024 * 1024); // 4GB + // this.quota = Math.min(Math.floor(os.freemem() / 4), 4 * 1024 * 1024 * 1024); // 4GB + this.quota = Math.min(Math.floor(os.freemem() / 4), 35 * 1024 * 1024); // 4GB // @ts-ignore } else if (TARGET === 'webOS' || TARGET === 'tizenOS') { @@ -55,6 +67,8 @@ export class MediaCache { } public destroy() { + this.cache.forEach((item) => { fs.unlinkSync(item.path); }); + MediaCache.instance = null; this.cache.clear(); this.cache = null; @@ -65,6 +79,8 @@ export class MediaCache { this.cacheSize = 0; this.cacheWindowStart = 0; this.cacheWindowEnd = 0; + this.pendingDownloads.clear(); + this.isDownloading = false; } public static getInstance() { @@ -89,137 +105,207 @@ export class MediaCache { return this.cache.get(this.cacheUrlMap.get(url)).size; } - public cacheForwardItems(cacheIndex: number, cacheAmount: number, playlistIndex: number) { - if (cacheAmount > 0) { - for (let i = cacheIndex; i < this.playlist.items.length; i++) { - const item = this.playlist.items[i]; - if (item.cache) { - if (this.cache.has(i)) { - this.cacheForwardItems(i + 1, cacheAmount - 1, playlistIndex); - break; - } - const tempCacheObject = new CacheObject(); + public cacheItems(playlistIndex: number) { + this.playlistIndex = playlistIndex; - downloadFile(item.url, tempCacheObject.path, - (downloadedBytes: number) => { - let underQuota = true; - if (this.cacheSize + downloadedBytes > this.quota) { - underQuota = this.purgeCacheItems(i, downloadedBytes, playlistIndex); - } + if (this.playlist.forwardCache && this.playlist.forwardCache > 0) { + let cacheAmount = this.playlist.forwardCache; - return underQuota; - }, null, - (downloadedBytes: number) => { - this.finalizeCacheItem(tempCacheObject, i, downloadedBytes, playlistIndex); - this.cacheForwardItems(i + 1, cacheAmount - 1, playlistIndex); - }, true) - .catch((error) => { - logger.error(error); - }); + for (let i = playlistIndex + 1; i < this.playlist.items.length; i++) { + if (cacheAmount === 0) { break; } + + if (this.playlist.items[i].cache) { + cacheAmount--; + + if (!this.cache.has(i)) { + this.pendingDownloads.add(i); + } + } } } - } - public cacheBackwardItems(cacheIndex: number, cacheAmount: number, playlistIndex: number) { - if (cacheAmount > 0) { - for (let i = cacheIndex; i >= 0; i--) { - const item = this.playlist.items[i]; - if (item.cache) { - if (this.cache.has(i)) { - this.cacheBackwardItems(i - 1, cacheAmount - 1, playlistIndex); - break; - } - const tempCacheObject = new CacheObject(); + if (this.playlist.backwardCache && this.playlist.backwardCache > 0) { + let cacheAmount = this.playlist.backwardCache; - downloadFile(item.url, tempCacheObject.path, - (downloadedBytes: number) => { - let underQuota = true; - if (this.cacheSize + downloadedBytes > this.quota) { - underQuota = this.purgeCacheItems(i, downloadedBytes, playlistIndex); - } - - return underQuota; - }, null, - (downloadedBytes: number) => { - this.finalizeCacheItem(tempCacheObject, i, downloadedBytes, playlistIndex); - this.cacheBackwardItems(i - 1, cacheAmount - 1, playlistIndex); - }, true) - .catch((error) => { - logger.error(error); - }); + for (let i = playlistIndex - 1; i >= 0; i--) { + if (cacheAmount === 0) { break; } + + if (this.playlist.items[i].cache) { + cacheAmount--; + + if (!this.cache.has(i)) { + this.pendingDownloads.add(i); + } + } } } + + this.updateCacheWindow(); + + if (!this.isDownloading) { + this.isDownloading = true; + this.downloadItems(); + } } - private purgeCacheItems(downloadItem: number, downloadedBytes: number, playlistIndex: number): boolean { - this.updateCacheWindow(playlistIndex); - let underQuota = true; - let purgeIndex = playlistIndex; - let purgeDistance = 0; - logger.debug(`Downloading item ${downloadItem} with playlist index ${playlistIndex} and cache window: [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`); + private downloadItems() { + if (this.pendingDownloads.size > 0) { + let itemIndex = 0; + let minDistance = this.playlist.items.length; + for (let i of this.pendingDownloads.values()) { + if (Math.abs(this.playlistIndex - i) < minDistance) { + minDistance = Math.abs(this.playlistIndex - i); + itemIndex = i; + } + else if (Math.abs(this.playlistIndex - i) === minDistance && i > this.playlistIndex) { + itemIndex = i; + } + } + this.pendingDownloads.delete(itemIndex); - // Priority: - // 1. Purge first encountered item outside cache window - // 2. Purge item furthest from view index inside window (except next item from view index) - for (let index of this.cache.keys()) { - if (index === downloadItem || index === playlistIndex || index === playlistIndex + 1) { - continue; + // Due to downloads being async, pending downloads can become out-of-sync with the current playlist index/target cache window + if (!this.shouldDownloadItem(itemIndex)) { + logger.debug(`Discarding download index ${itemIndex} since its outside cache window [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`); + this.downloadItems(); + return; } - if (index < this.cacheWindowStart) { - purgeIndex = index; - break; - } - else if (index > this.cacheWindowEnd) { - purgeIndex = index; - break; - } - else if (Math.abs(playlistIndex - index) > purgeDistance) { - purgeDistance = Math.abs(playlistIndex - index); - purgeIndex = index; - } - } + const tempCacheObject = new CacheObject(); + downloadFile(this.playlist.items[itemIndex].url, tempCacheObject.path, true, this.playlist.items[itemIndex].headers, + (downloadedBytes: number) => { + let underQuota = true; + if (this.cacheSize + downloadedBytes > this.quota) { + underQuota = this.purgeCacheItems(itemIndex, downloadedBytes); + } - if (purgeIndex !== playlistIndex) { - const deleteItem = this.cache.get(purgeIndex); - this.cacheSize -= deleteItem.size; - this.cacheUrlMap.delete(deleteItem.url); - this.cache.delete(purgeIndex); - this.updateCacheWindow(playlistIndex); - logger.info(`Item ${downloadItem} pending download (${downloadedBytes} bytes) cannot fit in cache, purging ${purgeIndex} from cache. Remaining quota ${this.quota - this.cacheSize} bytes`); - - if (this.cacheSize + downloadedBytes > this.quota) { - underQuota = this.purgeCacheItems(downloadItem, downloadedBytes, playlistIndex); - } + return underQuota; + }, null) + .then(() => { + this.finalizeCacheItem(tempCacheObject, itemIndex); + this.downloadItems(); + }, (error) => { + logger.warn(error); + this.downloadItems(); + }); } else { - // Cannot purge current item since we may already be streaming it - logger.warn(`Aborting item caching, cannot fit item ${downloadItem} (${downloadedBytes} bytes) within remaining space quota (${this.quota - this.cacheSize} bytes)`); - underQuota = false; + this.isDownloading = false; + } + } + + private shouldDownloadItem(index: number): boolean { + let download = false; + + if (index > this.playlistIndex) { + if (this.playlist.forwardCache && this.playlist.forwardCache > 0) { + const indexList = [...this.cache.keys(), index].sort((a, b) => a - b); + let forwardCacheItems = this.playlist.forwardCache; + + for (let i of indexList) { + if (i > this.playlistIndex) { + forwardCacheItems--; + + if (i === index) { + download = true; + } + else if (forwardCacheItems === 0) { + break; + } + } + } + } + } + else if (index < this.playlistIndex) { + if (this.playlist.backwardCache && this.playlist.backwardCache > 0) { + const indexList = [...this.cache.keys(), index].sort((a, b) => b - a); + let backwardCacheItems = this.playlist.backwardCache; + + for (let i of indexList) { + if (i < this.playlistIndex) { + backwardCacheItems--; + + if (i === index) { + download = true; + } + else if (backwardCacheItems === 0) { + break; + } + } + } + } + } + + return download; + } + + private purgeCacheItems(downloadItem: number, downloadedBytes: number): boolean { + let underQuota = true; + + while (this.cacheSize + downloadedBytes > this.quota) { + let purgeIndex = this.playlistIndex; + let purgeDistance = 0; + logger.debug(`Downloading item ${downloadItem} with playlist index ${this.playlistIndex} and cache window: [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`); + + // Priority: + // 1. Purge first encountered item outside cache window + // 2. Purge item furthest from view index inside window (except next item from view index) + for (let index of this.cache.keys()) { + if (index === downloadItem || index === this.playlistIndex || index === this.playlistIndex + 1) { + continue; + } + + if (index < this.cacheWindowStart || index > this.cacheWindowEnd) { + purgeIndex = index; + break; + } + else if (Math.abs(this.playlistIndex - index) > purgeDistance) { + purgeDistance = Math.abs(this.playlistIndex - index); + purgeIndex = index; + } + } + + if (purgeIndex !== this.playlistIndex) { + const deleteItem = this.cache.get(purgeIndex); + fs.unlinkSync(deleteItem.path); + this.cacheSize -= deleteItem.size; + this.cacheUrlMap.delete(deleteItem.url); + this.cache.delete(purgeIndex); + this.updateCacheWindow(); + logger.info(`Item ${downloadItem} pending download (${downloadedBytes} bytes) cannot fit in cache, purging ${purgeIndex} from cache. Remaining quota ${this.quota - this.cacheSize} bytes`); + } + else { + // Cannot purge current item since we may already be streaming it + logger.warn(`Aborting item caching, cannot fit item ${downloadItem} (${downloadedBytes} bytes) within remaining space quota (${this.quota - this.cacheSize} bytes)`); + underQuota = false; + break; + } } return underQuota; } - private finalizeCacheItem(cacheObject: CacheObject, index: number, size: number, playlistIndex: number) { + private finalizeCacheItem(cacheObject: CacheObject, index: number) { + const size = fs.statSync(cacheObject.path).size; cacheObject.size = size; this.cacheSize += size; logger.info(`Cached item ${index} (${cacheObject.size} bytes) with remaining quota ${this.quota - this.cacheSize} bytes: ${cacheObject.url}`); this.cache.set(index, cacheObject); this.cacheUrlMap.set(cacheObject.url, index); - this.updateCacheWindow(playlistIndex); + this.updateCacheWindow(); } - private updateCacheWindow(playlistIndex: number) { + private updateCacheWindow() { + const indexList = [...this.cache.keys()].sort((a, b) => a - b); + if (this.playlist.forwardCache && this.playlist.forwardCache > 0) { let forwardCacheItems = this.playlist.forwardCache; - for (let index of this.cache.keys()) { - if (index > playlistIndex) { + for (let index of indexList) { + if (index > this.playlistIndex) { forwardCacheItems--; if (forwardCacheItems === 0) { @@ -230,13 +316,13 @@ export class MediaCache { } } else { - this.cacheWindowEnd = playlistIndex; + this.cacheWindowEnd = this.playlistIndex; } if (this.playlist.backwardCache && this.playlist.backwardCache > 0) { let backwardCacheItems = this.playlist.backwardCache; - for (let index of this.cache.keys()) { - if (index < playlistIndex) { + for (let index of indexList) { + if (index < this.playlistIndex) { backwardCacheItems--; if (backwardCacheItems === 0) { @@ -247,7 +333,7 @@ export class MediaCache { } } else { - this.cacheWindowStart = playlistIndex + this.cacheWindowStart = this.playlistIndex } } } diff --git a/receivers/common/web/UtilityBackend.ts b/receivers/common/web/UtilityBackend.ts index 95608ee..69c7e58 100644 --- a/receivers/common/web/UtilityBackend.ts +++ b/receivers/common/web/UtilityBackend.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import * as url from 'url'; import { http, https } from 'modules/follow-redirects'; import * as memfs from 'modules/memfs'; import { Logger, LoggerType } from 'common/Logger'; @@ -35,14 +36,22 @@ export async function fetchJSON(url: string): Promise { }); } -export async function downloadFile(url: string, destination: string, startCb: (downloadSize: number) => boolean = null, progressCb: (downloadedBytes: number, downloadSize: number) => void = null, finishCb: (downloadedBytes: number) => void = null, inMemory: boolean = false): Promise { +export async function downloadFile(downloadUrl: string, destination: string, inMemory: boolean = false, requestHeaders: { [key: string]: string } = null, + startCb: (downloadSize: number) => boolean = null, + progressCb: (downloadedBytes: number, downloadSize: number) => void = null): Promise { return new Promise((resolve, reject) => { const file = inMemory ? memfs.fs.createWriteStream(destination) : fs.createWriteStream(destination); - const protocol = url.startsWith('https') ? https : http; + const protocol = downloadUrl.startsWith('https') ? https : http; - protocol.get(url, (response) => { + const parsedUrl = url.parse(downloadUrl); + const options = protocol.RequestOptions = { + ...parsedUrl, + headers: requestHeaders + }; + + protocol.get(options, (response) => { const downloadSize = Number(response.headers['content-length']); - logger.info(`Downloading file ${url} to ${destination} with size: ${downloadSize} bytes`); + logger.info(`Downloading file ${downloadUrl} to ${destination} with size: ${downloadSize} bytes`); if (startCb) { if (!startCb(downloadSize)) { file.close(); @@ -61,9 +70,6 @@ export async function downloadFile(url: string, destination: string, startCb: (d }); file.on('finish', () => { file.close(); - if (finishCb) { - finishCb(downloadedBytes); - } resolve(); }); }).on('error', (err) => { diff --git a/receivers/electron/src/Main.ts b/receivers/electron/src/Main.ts index 5b9bf4a..635b680 100644 --- a/receivers/electron/src/Main.ts +++ b/receivers/electron/src/Main.ts @@ -24,7 +24,6 @@ class AppCache { public appVersion: string = null; public playMessage: PlayMessage = null; public playerVolume: number = null; - public playlist: PlaylistContent = null; public subscribedKeys = new Set(); } @@ -181,7 +180,6 @@ export class Main { case ContentType.Playlist: { rendererMessage = json as PlaylistContent; rendererEvent = 'play-playlist'; - Main.cache.playlist = rendererMessage; if ((rendererMessage.forwardCache && rendererMessage.forwardCache > 0) || (rendererMessage.backwardCache && rendererMessage.backwardCache > 0)) { Main.mediaCache?.destroy(); @@ -322,23 +320,8 @@ export class Main { ipcMain.on('play-request', (event: IpcMainEvent, value: PlayMessage, playlistIndex: number) => { logger.debug(`Received play request for index ${playlistIndex}:`, value); - - if (Main.cache.playlist.forwardCache && Main.cache.playlist.forwardCache > 0) { - if (Main.mediaCache.has(playlistIndex)) { - value.url = Main.mediaCache.getUrl(playlistIndex); - } - - Main.mediaCache.cacheForwardItems(playlistIndex + 1, Main.cache.playlist.forwardCache, playlistIndex); - } - - if (Main.cache.playlist.backwardCache && Main.cache.playlist.backwardCache > 0) { - if (Main.mediaCache.has(playlistIndex)) { - value.url = Main.mediaCache.getUrl(playlistIndex); - } - - Main.mediaCache.cacheBackwardItems(playlistIndex - 1, Main.cache.playlist.backwardCache, playlistIndex); - } - + value.url = Main.mediaCache.has(playlistIndex) ? Main.mediaCache.getUrl(playlistIndex) : value.url; + Main.mediaCache.cacheItems(playlistIndex); Main.play(value); }); ipcMain.on('send-download-request', async () => { diff --git a/receivers/electron/src/Updater.ts b/receivers/electron/src/Updater.ts index ba73d71..b635383 100644 --- a/receivers/electron/src/Updater.ts +++ b/receivers/electron/src/Updater.ts @@ -365,7 +365,7 @@ export class Updater { const destination = path.join(Updater.updateDataPath, file); logger.info(`Downloading '${fileInfo.url}' to '${destination}'.`); Updater.isDownloading = true; - await downloadFile(fileInfo.url.toString(), destination, null, (downloadedBytes: number, downloadSize: number) => { + await downloadFile(fileInfo.url.toString(), destination, false, null, null, (downloadedBytes: number, downloadSize: number) => { Updater.updateProgress = downloadedBytes / downloadSize; });