Receivers: Multiple MediaCache fixes

receivers/common/web/MediaCache.ts

@@ -23,17 +23,28 @@ class CacheObject {
 
 export class MediaCache {
     private static instance: MediaCache = null;
-    private cache = new Map<number, CacheObject>();
+    private cache: Map<number, CacheObject>;
-    private cacheUrlMap = new Map<string,number>();
+    private cacheUrlMap: Map<string,number>;
     private playlist: PlaylistContent;
+    private playlistIndex: number;
     private quota: number;
-    private cacheSize: number = 0;
+    private cacheSize: number;
-    private cacheWindowStart: number = 0;
+    private cacheWindowStart: number;
-    private cacheWindowEnd: number = 0;
+    private cacheWindowEnd: number;
+    private pendingDownloads: Set<number>;
+    private isDownloading: boolean;
 
     constructor(playlist: PlaylistContent) {
         MediaCache.instance = this;
         this.playlist = playlist;
+        this.playlistIndex = playlist.offset ? playlist.offset : 0;
+        this.cache = new Map<number, CacheObject>();
+        this.cacheUrlMap = new Map<string,number>();
+        this.cacheSize = 0;
+        this.cacheWindowStart = 0;
+        this.cacheWindowEnd = 0;
+        this.pendingDownloads = new Set();
+        this.isDownloading = false;
 
         if (!fs.existsSync('/cache')) {
             fs.mkdirSync('/cache');
@@ -41,7 +52,8 @@ export class MediaCache {
 
         // @ts-ignore
         if (TARGET === 'electron') {
-            this.quota = Math.min(Math.floor(os.freemem() / 4), 4 * 1024 * 1024 * 1024); // 4GB
+            // this.quota = Math.min(Math.floor(os.freemem() / 4), 4 * 1024 * 1024 * 1024); // 4GB
+            this.quota = Math.min(Math.floor(os.freemem() / 4), 35 * 1024 * 1024); // 4GB
 
         // @ts-ignore
         } else if (TARGET === 'webOS' || TARGET === 'tizenOS') {
@@ -55,6 +67,8 @@ export class MediaCache {
     }
 
     public destroy() {
+        this.cache.forEach((item) => { fs.unlinkSync(item.path); });
+
         MediaCache.instance = null;
         this.cache.clear();
         this.cache = null;
@@ -65,6 +79,8 @@ export class MediaCache {
         this.cacheSize = 0;
         this.cacheWindowStart = 0;
         this.cacheWindowEnd = 0;
+        this.pendingDownloads.clear();
+        this.isDownloading = false;
     }
 
     public static getInstance() {
@@ -89,137 +105,207 @@ export class MediaCache {
         return this.cache.get(this.cacheUrlMap.get(url)).size;
     }
 
-    public cacheForwardItems(cacheIndex: number, cacheAmount: number, playlistIndex: number) {
+    public cacheItems(playlistIndex: number) {
-        if (cacheAmount > 0) {
+        this.playlistIndex = playlistIndex;
-            for (let i = cacheIndex; i < this.playlist.items.length; i++) {
-                const item = this.playlist.items[i];
-                if (item.cache) {
-                    if (this.cache.has(i)) {
-                        this.cacheForwardItems(i + 1, cacheAmount - 1, playlistIndex);
-                        break;
-                    }
-                    const tempCacheObject = new CacheObject();
 
-                    downloadFile(item.url, tempCacheObject.path,
+        if (this.playlist.forwardCache && this.playlist.forwardCache > 0) {
-                    (downloadedBytes: number) => {
+            let cacheAmount = this.playlist.forwardCache;
-                        let underQuota = true;
-                        if (this.cacheSize + downloadedBytes > this.quota) {
-                            underQuota = this.purgeCacheItems(i, downloadedBytes, playlistIndex);
-                        }
 
-                        return underQuota;
+            for (let i = playlistIndex + 1; i < this.playlist.items.length; i++) {
-                    }, null,
+                if (cacheAmount === 0) {
-                    (downloadedBytes: number) => {
-                        this.finalizeCacheItem(tempCacheObject, i, downloadedBytes, playlistIndex);
-                        this.cacheForwardItems(i + 1, cacheAmount - 1, playlistIndex);
-                    }, true)
-                    .catch((error) => {
-                        logger.error(error);
-                    });
                     break;
                 }
+
+                if (this.playlist.items[i].cache) {
+                    cacheAmount--;
+
+                    if (!this.cache.has(i)) {
+                        this.pendingDownloads.add(i);
+                    }
+                }
             }
         }
-    }
 
-    public cacheBackwardItems(cacheIndex: number, cacheAmount: number, playlistIndex: number) {
+        if (this.playlist.backwardCache && this.playlist.backwardCache > 0) {
-        if (cacheAmount > 0) {
+            let cacheAmount = this.playlist.backwardCache;
-            for (let i = cacheIndex; i >= 0; i--) {
-                const item = this.playlist.items[i];
-                if (item.cache) {
-                    if (this.cache.has(i)) {
-                        this.cacheBackwardItems(i - 1, cacheAmount - 1, playlistIndex);
-                        break;
-                    }
-                    const tempCacheObject = new CacheObject();
 
-                    downloadFile(item.url, tempCacheObject.path,
+            for (let i = playlistIndex - 1; i >= 0; i--) {
-                    (downloadedBytes: number) => {
+                if (cacheAmount === 0) {
-                        let underQuota = true;
-                        if (this.cacheSize + downloadedBytes > this.quota) {
-                            underQuota = this.purgeCacheItems(i, downloadedBytes, playlistIndex);
-                        }
-
-                        return underQuota;
-                    }, null,
-                    (downloadedBytes: number) => {
-                        this.finalizeCacheItem(tempCacheObject, i, downloadedBytes, playlistIndex);
-                        this.cacheBackwardItems(i - 1, cacheAmount - 1, playlistIndex);
-                    }, true)
-                    .catch((error) => {
-                        logger.error(error);
-                    });
                     break;
                 }
+
+                if (this.playlist.items[i].cache) {
+                    cacheAmount--;
+
+                    if (!this.cache.has(i)) {
+                        this.pendingDownloads.add(i);
+                    }
+                }
             }
         }
+
+        this.updateCacheWindow();
+
+        if (!this.isDownloading) {
+            this.isDownloading = true;
+            this.downloadItems();
+        }
     }
 
-    private purgeCacheItems(downloadItem: number, downloadedBytes: number, playlistIndex: number): boolean {
+    private downloadItems() {
-        this.updateCacheWindow(playlistIndex);
+        if (this.pendingDownloads.size > 0) {
-        let underQuota = true;
+            let itemIndex = 0;
-        let purgeIndex = playlistIndex;
+            let minDistance = this.playlist.items.length;
-        let purgeDistance = 0;
+            for (let i of this.pendingDownloads.values()) {
-        logger.debug(`Downloading item ${downloadItem} with playlist index ${playlistIndex} and cache window: [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`);
+                if (Math.abs(this.playlistIndex - i) < minDistance) {
+                    minDistance = Math.abs(this.playlistIndex - i);
+                    itemIndex = i;
+                }
+                else if (Math.abs(this.playlistIndex - i) === minDistance && i > this.playlistIndex) {
+                    itemIndex = i;
+                }
+            }
+            this.pendingDownloads.delete(itemIndex);
 
-        // Priority:
+            // Due to downloads being async, pending downloads can become out-of-sync with the current playlist index/target cache window
-        // 1. Purge first encountered item outside cache window
+            if (!this.shouldDownloadItem(itemIndex)) {
-        // 2. Purge item furthest from view index inside window (except next item from view index)
+                logger.debug(`Discarding download index ${itemIndex} since its outside cache window [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`);
-        for (let index of this.cache.keys()) {
+                this.downloadItems();
-            if (index === downloadItem || index === playlistIndex || index === playlistIndex + 1) {
+                return;
-                continue;
             }
 
-            if (index < this.cacheWindowStart) {
+            const tempCacheObject = new CacheObject();
-                purgeIndex = index;
+            downloadFile(this.playlist.items[itemIndex].url, tempCacheObject.path, true, this.playlist.items[itemIndex].headers,
-                break;
+            (downloadedBytes: number) => {
-            }
+                let underQuota = true;
-            else if (index > this.cacheWindowEnd) {
+                if (this.cacheSize + downloadedBytes > this.quota) {
-                purgeIndex = index;
+                    underQuota = this.purgeCacheItems(itemIndex, downloadedBytes);
-                break;
+                }
-            }
-            else if (Math.abs(playlistIndex - index) > purgeDistance) {
-                purgeDistance = Math.abs(playlistIndex - index);
-                purgeIndex = index;
-            }
-        }
 
-        if (purgeIndex !== playlistIndex) {
+                return underQuota;
-            const deleteItem = this.cache.get(purgeIndex);
+            }, null)
-            this.cacheSize -= deleteItem.size;
+            .then(() => {
-            this.cacheUrlMap.delete(deleteItem.url);
+                this.finalizeCacheItem(tempCacheObject, itemIndex);
-            this.cache.delete(purgeIndex);
+                this.downloadItems();
-            this.updateCacheWindow(playlistIndex);
+            }, (error) => {
-            logger.info(`Item ${downloadItem} pending download (${downloadedBytes} bytes) cannot fit in cache, purging ${purgeIndex} from cache. Remaining quota ${this.quota - this.cacheSize} bytes`);
+                logger.warn(error);
-
+                this.downloadItems();
-            if (this.cacheSize + downloadedBytes > this.quota) {
+            });
-                underQuota = this.purgeCacheItems(downloadItem, downloadedBytes, playlistIndex);
-            }
         }
         else {
-            // Cannot purge current item since we may already be streaming it
+            this.isDownloading = false;
-            logger.warn(`Aborting item caching, cannot fit item ${downloadItem} (${downloadedBytes} bytes) within remaining space quota (${this.quota - this.cacheSize} bytes)`);
+        }
-            underQuota = false;
+    }
+
+    private shouldDownloadItem(index: number): boolean {
+        let download = false;
+
+        if (index > this.playlistIndex) {
+            if (this.playlist.forwardCache && this.playlist.forwardCache > 0) {
+                const indexList = [...this.cache.keys(), index].sort((a, b) => a - b);
+                let forwardCacheItems = this.playlist.forwardCache;
+
+                for (let i of indexList) {
+                    if (i > this.playlistIndex) {
+                        forwardCacheItems--;
+
+                        if (i === index) {
+                            download = true;
+                        }
+                        else if (forwardCacheItems === 0) {
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        else if (index < this.playlistIndex) {
+            if (this.playlist.backwardCache && this.playlist.backwardCache > 0) {
+                const indexList = [...this.cache.keys(), index].sort((a, b) => b - a);
+                let backwardCacheItems = this.playlist.backwardCache;
+
+                for (let i of indexList) {
+                    if (i < this.playlistIndex) {
+                        backwardCacheItems--;
+
+                        if (i === index) {
+                            download = true;
+                        }
+                        else if (backwardCacheItems === 0) {
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+
+        return download;
+    }
+
+    private purgeCacheItems(downloadItem: number, downloadedBytes: number): boolean {
+        let underQuota = true;
+
+        while (this.cacheSize + downloadedBytes > this.quota) {
+            let purgeIndex = this.playlistIndex;
+            let purgeDistance = 0;
+            logger.debug(`Downloading item ${downloadItem} with playlist index ${this.playlistIndex} and cache window: [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`);
+
+            // Priority:
+            // 1. Purge first encountered item outside cache window
+            // 2. Purge item furthest from view index inside window (except next item from view index)
+            for (let index of this.cache.keys()) {
+                if (index === downloadItem || index === this.playlistIndex || index === this.playlistIndex + 1) {
+                    continue;
+                }
+
+                if (index < this.cacheWindowStart || index > this.cacheWindowEnd) {
+                    purgeIndex = index;
+                    break;
+                }
+                else if (Math.abs(this.playlistIndex - index) > purgeDistance) {
+                    purgeDistance = Math.abs(this.playlistIndex - index);
+                    purgeIndex = index;
+                }
+            }
+
+            if (purgeIndex !== this.playlistIndex) {
+                const deleteItem = this.cache.get(purgeIndex);
+                fs.unlinkSync(deleteItem.path);
+                this.cacheSize -= deleteItem.size;
+                this.cacheUrlMap.delete(deleteItem.url);
+                this.cache.delete(purgeIndex);
+                this.updateCacheWindow();
+                logger.info(`Item ${downloadItem} pending download (${downloadedBytes} bytes) cannot fit in cache, purging ${purgeIndex} from cache. Remaining quota ${this.quota - this.cacheSize} bytes`);
+            }
+            else {
+                // Cannot purge current item since we may already be streaming it
+                logger.warn(`Aborting item caching, cannot fit item ${downloadItem} (${downloadedBytes} bytes) within remaining space quota (${this.quota - this.cacheSize} bytes)`);
+                underQuota = false;
+                break;
+            }
         }
 
         return underQuota;
     }
 
-    private finalizeCacheItem(cacheObject: CacheObject, index: number, size: number, playlistIndex: number) {
+    private finalizeCacheItem(cacheObject: CacheObject, index: number) {
+        const size = fs.statSync(cacheObject.path).size;
         cacheObject.size = size;
         this.cacheSize += size;
         logger.info(`Cached item ${index} (${cacheObject.size} bytes) with remaining quota ${this.quota - this.cacheSize} bytes: ${cacheObject.url}`);
 
         this.cache.set(index, cacheObject);
         this.cacheUrlMap.set(cacheObject.url, index);
-        this.updateCacheWindow(playlistIndex);
+        this.updateCacheWindow();
     }
 
-    private updateCacheWindow(playlistIndex: number) {
+    private updateCacheWindow() {
+        const indexList = [...this.cache.keys()].sort((a, b) => a - b);
+
         if (this.playlist.forwardCache && this.playlist.forwardCache > 0) {
             let forwardCacheItems = this.playlist.forwardCache;
-            for (let index of this.cache.keys()) {
+            for (let index of indexList) {
-                if (index > playlistIndex) {
+                if (index > this.playlistIndex) {
                     forwardCacheItems--;
 
                     if (forwardCacheItems === 0) {
@@ -230,13 +316,13 @@ export class MediaCache {
             }
         }
         else {
-            this.cacheWindowEnd = playlistIndex;
+            this.cacheWindowEnd = this.playlistIndex;
         }
 
         if (this.playlist.backwardCache && this.playlist.backwardCache > 0) {
             let backwardCacheItems = this.playlist.backwardCache;
-            for (let index of this.cache.keys()) {
+            for (let index of indexList) {
-                if (index < playlistIndex) {
+                if (index < this.playlistIndex) {
                     backwardCacheItems--;
 
                     if (backwardCacheItems === 0) {
@@ -247,7 +333,7 @@ export class MediaCache {
             }
         }
         else {
-            this.cacheWindowStart = playlistIndex
+            this.cacheWindowStart = this.playlistIndex
         }
     }
 }

receivers/common/web/UtilityBackend.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs';
+import * as url from 'url';
 import { http, https } from 'modules/follow-redirects';
 import * as memfs from 'modules/memfs';
 import { Logger, LoggerType } from 'common/Logger';
@@ -35,14 +36,22 @@ export async function fetchJSON(url: string): Promise<any> {
     });
 }
 
-export async function downloadFile(url: string, destination: string, startCb: (downloadSize: number) => boolean = null, progressCb: (downloadedBytes: number, downloadSize: number) => void = null, finishCb: (downloadedBytes: number) => void = null, inMemory: boolean = false): Promise<void> {
+export async function downloadFile(downloadUrl: string, destination: string, inMemory: boolean = false, requestHeaders: { [key: string]: string } = null,
+                                   startCb: (downloadSize: number) => boolean = null,
+                                   progressCb: (downloadedBytes: number, downloadSize: number) => void = null): Promise<void> {
     return new Promise((resolve, reject) => {
         const file = inMemory ? memfs.fs.createWriteStream(destination) : fs.createWriteStream(destination);
-        const protocol = url.startsWith('https') ? https : http;
+        const protocol = downloadUrl.startsWith('https') ? https : http;
 
-        protocol.get(url, (response) => {
+        const parsedUrl = url.parse(downloadUrl);
+        const options = protocol.RequestOptions = {
+            ...parsedUrl,
+            headers: requestHeaders
+        };
+
+        protocol.get(options, (response) => {
             const downloadSize = Number(response.headers['content-length']);
-            logger.info(`Downloading file ${url} to ${destination} with size: ${downloadSize} bytes`);
+            logger.info(`Downloading file ${downloadUrl} to ${destination} with size: ${downloadSize} bytes`);
             if (startCb) {
                 if (!startCb(downloadSize)) {
                     file.close();
@@ -61,9 +70,6 @@ export async function downloadFile(url: string, destination: string, startCb: (d
             });
             file.on('finish', () => {
                 file.close();
-                if (finishCb) {
-                    finishCb(downloadedBytes);
-                }
                 resolve();
             });
         }).on('error', (err) => {

receivers/electron/src/Main.ts

@@ -24,7 +24,6 @@ class AppCache {
     public appVersion: string = null;
     public playMessage: PlayMessage = null;
     public playerVolume: number = null;
-    public playlist: PlaylistContent = null;
     public subscribedKeys = new Set<string>();
 }
 
@@ -181,7 +180,6 @@ export class Main {
                     case ContentType.Playlist: {
                         rendererMessage = json as PlaylistContent;
                         rendererEvent = 'play-playlist';
-                        Main.cache.playlist = rendererMessage;
 
                         if ((rendererMessage.forwardCache && rendererMessage.forwardCache > 0) || (rendererMessage.backwardCache && rendererMessage.backwardCache > 0)) {
                             Main.mediaCache?.destroy();
@@ -322,23 +320,8 @@ export class Main {
 
         ipcMain.on('play-request', (event: IpcMainEvent, value: PlayMessage, playlistIndex: number) => {
             logger.debug(`Received play request for index ${playlistIndex}:`, value);
-
+            value.url = Main.mediaCache.has(playlistIndex) ? Main.mediaCache.getUrl(playlistIndex) : value.url;
-            if (Main.cache.playlist.forwardCache && Main.cache.playlist.forwardCache > 0) {
+            Main.mediaCache.cacheItems(playlistIndex);
-                if (Main.mediaCache.has(playlistIndex)) {
-                    value.url = Main.mediaCache.getUrl(playlistIndex);
-                }
-
-                Main.mediaCache.cacheForwardItems(playlistIndex + 1, Main.cache.playlist.forwardCache, playlistIndex);
-            }
-
-            if (Main.cache.playlist.backwardCache && Main.cache.playlist.backwardCache > 0) {
-                if (Main.mediaCache.has(playlistIndex)) {
-                    value.url = Main.mediaCache.getUrl(playlistIndex);
-                }
-
-                Main.mediaCache.cacheBackwardItems(playlistIndex - 1, Main.cache.playlist.backwardCache, playlistIndex);
-            }
-
             Main.play(value);
         });
         ipcMain.on('send-download-request', async () => {

receivers/electron/src/Updater.ts

@@ -365,7 +365,7 @@ export class Updater {
             const destination = path.join(Updater.updateDataPath, file);
             logger.info(`Downloading '${fileInfo.url}' to '${destination}'.`);
             Updater.isDownloading = true;
-            await downloadFile(fileInfo.url.toString(), destination, null, (downloadedBytes: number, downloadSize: number) => {
+            await downloadFile(fileInfo.url.toString(), destination, false, null, null, (downloadedBytes: number, downloadSize: number) => {
                 Updater.updateProgress = downloadedBytes / downloadSize;
             });