mirrors/fcast
1
0
Fork 0
mirror of https://gitlab.com/futo-org/fcast.git synced 2025-06-24 21:25:23 +00:00
Code Activity

Receivers: Multiple MediaCache fixes

Browse source
This commit is contained in:
Michael Hollister 2025-06-11 14:51:55 -05:00
parent c95f254b3b
commit 7a0c865bb2
4 changed files with 208 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

298
receivers/common/web/MediaCache.ts
View file

@ -23,17 +23,28 @@ class CacheObject {
export class MediaCache {
private static instance: MediaCache = null;
private cache = new Map<number, CacheObject>();
private cacheUrlMap = new Map<string,number>();
private cache: Map<number, CacheObject>;
private cacheUrlMap: Map<string,number>;
private playlist: PlaylistContent;
private playlistIndex: number;
private quota: number;
private cacheSize: number = 0;
private cacheWindowStart: number = 0;
private cacheWindowEnd: number = 0;
private cacheSize: number;
private cacheWindowStart: number;
private cacheWindowEnd: number;
private pendingDownloads: Set<number>;
private isDownloading: boolean;
constructor(playlist: PlaylistContent) {
MediaCache.instance = this;
this.playlist = playlist;
this.playlistIndex = playlist.offset ? playlist.offset : 0;
this.cache = new Map<number, CacheObject>();
this.cacheUrlMap = new Map<string,number>();
this.cacheSize = 0;
this.cacheWindowStart = 0;
this.cacheWindowEnd = 0;
this.pendingDownloads = new Set();
this.isDownloading = false;
if (!fs.existsSync('/cache')) {
fs.mkdirSync('/cache');
@ -41,7 +52,8 @@ export class MediaCache {
// @ts-ignore
if (TARGET === 'electron') {
this.quota = Math.min(Math.floor(os.freemem() / 4), 4 * 1024 * 1024 * 1024); // 4GB
// this.quota = Math.min(Math.floor(os.freemem() / 4), 4 * 1024 * 1024 * 1024); // 4GB
this.quota = Math.min(Math.floor(os.freemem() / 4), 35 * 1024 * 1024); // 4GB
// @ts-ignore
} else if (TARGET === 'webOS' || TARGET === 'tizenOS') {
@ -55,6 +67,8 @@ export class MediaCache {
}
public destroy() {
this.cache.forEach((item) => { fs.unlinkSync(item.path); });
MediaCache.instance = null;
this.cache.clear();
this.cache = null;
@ -65,6 +79,8 @@ export class MediaCache {
this.cacheSize = 0;
this.cacheWindowStart = 0;
this.cacheWindowEnd = 0;
this.pendingDownloads.clear();
this.isDownloading = false;
}
public static getInstance() {
@ -89,137 +105,207 @@ export class MediaCache {
return this.cache.get(this.cacheUrlMap.get(url)).size;
}
public cacheForwardItems(cacheIndex: number, cacheAmount: number, playlistIndex: number) {
if (cacheAmount > 0) {
for (let i = cacheIndex; i < this.playlist.items.length; i++) {
const item = this.playlist.items[i];
if (item.cache) {
if (this.cache.has(i)) {
this.cacheForwardItems(i + 1, cacheAmount - 1, playlistIndex);
break;
}
const tempCacheObject = new CacheObject();
public cacheItems(playlistIndex: number) {
this.playlistIndex = playlistIndex;
downloadFile(item.url, tempCacheObject.path,
(downloadedBytes: number) => {
let underQuota = true;
if (this.cacheSize + downloadedBytes > this.quota) {
underQuota = this.purgeCacheItems(i, downloadedBytes, playlistIndex);
}
if (this.playlist.forwardCache && this.playlist.forwardCache > 0) {
let cacheAmount = this.playlist.forwardCache;
return underQuota;
}, null,
(downloadedBytes: number) => {
this.finalizeCacheItem(tempCacheObject, i, downloadedBytes, playlistIndex);
this.cacheForwardItems(i + 1, cacheAmount - 1, playlistIndex);
}, true)
.catch((error) => {
logger.error(error);
});
for (let i = playlistIndex + 1; i < this.playlist.items.length; i++) {
if (cacheAmount === 0) {
break;
}
if (this.playlist.items[i].cache) {
cacheAmount--;
if (!this.cache.has(i)) {
this.pendingDownloads.add(i);
}
}
}
}
}
public cacheBackwardItems(cacheIndex: number, cacheAmount: number, playlistIndex: number) {
if (cacheAmount > 0) {
for (let i = cacheIndex; i >= 0; i--) {
const item = this.playlist.items[i];
if (item.cache) {
if (this.cache.has(i)) {
this.cacheBackwardItems(i - 1, cacheAmount - 1, playlistIndex);
break;
}
const tempCacheObject = new CacheObject();
if (this.playlist.backwardCache && this.playlist.backwardCache > 0) {
let cacheAmount = this.playlist.backwardCache;
downloadFile(item.url, tempCacheObject.path,
(downloadedBytes: number) => {
let underQuota = true;
if (this.cacheSize + downloadedBytes > this.quota) {
underQuota = this.purgeCacheItems(i, downloadedBytes, playlistIndex);
}
return underQuota;
}, null,
(downloadedBytes: number) => {
this.finalizeCacheItem(tempCacheObject, i, downloadedBytes, playlistIndex);
this.cacheBackwardItems(i - 1, cacheAmount - 1, playlistIndex);
}, true)
.catch((error) => {
logger.error(error);
});
for (let i = playlistIndex - 1; i >= 0; i--) {
if (cacheAmount === 0) {
break;
}
if (this.playlist.items[i].cache) {
cacheAmount--;
if (!this.cache.has(i)) {
this.pendingDownloads.add(i);
}
}
}
}
this.updateCacheWindow();
if (!this.isDownloading) {
this.isDownloading = true;
this.downloadItems();
}
}
private purgeCacheItems(downloadItem: number, downloadedBytes: number, playlistIndex: number): boolean {
this.updateCacheWindow(playlistIndex);
let underQuota = true;
let purgeIndex = playlistIndex;
let purgeDistance = 0;
logger.debug(`Downloading item ${downloadItem} with playlist index ${playlistIndex} and cache window: [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`);
private downloadItems() {
if (this.pendingDownloads.size > 0) {
let itemIndex = 0;
let minDistance = this.playlist.items.length;
for (let i of this.pendingDownloads.values()) {
if (Math.abs(this.playlistIndex - i) < minDistance) {
minDistance = Math.abs(this.playlistIndex - i);
itemIndex = i;
}
else if (Math.abs(this.playlistIndex - i) === minDistance && i > this.playlistIndex) {
itemIndex = i;
}
}
this.pendingDownloads.delete(itemIndex);
// Priority:
// 1. Purge first encountered item outside cache window
// 2. Purge item furthest from view index inside window (except next item from view index)
for (let index of this.cache.keys()) {
if (index === downloadItem || index === playlistIndex || index === playlistIndex + 1) {
continue;
// Due to downloads being async, pending downloads can become out-of-sync with the current playlist index/target cache window
if (!this.shouldDownloadItem(itemIndex)) {
logger.debug(`Discarding download index ${itemIndex} since its outside cache window [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`);
this.downloadItems();
return;
}
if (index < this.cacheWindowStart) {
purgeIndex = index;
break;
}
else if (index > this.cacheWindowEnd) {
purgeIndex = index;
break;
}
else if (Math.abs(playlistIndex - index) > purgeDistance) {
purgeDistance = Math.abs(playlistIndex - index);
purgeIndex = index;
}
}
const tempCacheObject = new CacheObject();
downloadFile(this.playlist.items[itemIndex].url, tempCacheObject.path, true, this.playlist.items[itemIndex].headers,
(downloadedBytes: number) => {
let underQuota = true;
if (this.cacheSize + downloadedBytes > this.quota) {
underQuota = this.purgeCacheItems(itemIndex, downloadedBytes);
}
if (purgeIndex !== playlistIndex) {
const deleteItem = this.cache.get(purgeIndex);
this.cacheSize -= deleteItem.size;
this.cacheUrlMap.delete(deleteItem.url);
this.cache.delete(purgeIndex);
this.updateCacheWindow(playlistIndex);
logger.info(`Item ${downloadItem} pending download (${downloadedBytes} bytes) cannot fit in cache, purging ${purgeIndex} from cache. Remaining quota ${this.quota - this.cacheSize} bytes`);
if (this.cacheSize + downloadedBytes > this.quota) {
underQuota = this.purgeCacheItems(downloadItem, downloadedBytes, playlistIndex);
}
return underQuota;
}, null)
.then(() => {
this.finalizeCacheItem(tempCacheObject, itemIndex);
this.downloadItems();
}, (error) => {
logger.warn(error);
this.downloadItems();
});
}
else {
// Cannot purge current item since we may already be streaming it
logger.warn(`Aborting item caching, cannot fit item ${downloadItem} (${downloadedBytes} bytes) within remaining space quota (${this.quota - this.cacheSize} bytes)`);
underQuota = false;
this.isDownloading = false;
}
}
private shouldDownloadItem(index: number): boolean {
let download = false;
if (index > this.playlistIndex) {
if (this.playlist.forwardCache && this.playlist.forwardCache > 0) {
const indexList = [...this.cache.keys(), index].sort((a, b) => a - b);
let forwardCacheItems = this.playlist.forwardCache;
for (let i of indexList) {
if (i > this.playlistIndex) {
forwardCacheItems--;
if (i === index) {
download = true;
}
else if (forwardCacheItems === 0) {
break;
}
}
}
}
}
else if (index < this.playlistIndex) {
if (this.playlist.backwardCache && this.playlist.backwardCache > 0) {
const indexList = [...this.cache.keys(), index].sort((a, b) => b - a);
let backwardCacheItems = this.playlist.backwardCache;
for (let i of indexList) {
if (i < this.playlistIndex) {
backwardCacheItems--;
if (i === index) {
download = true;
}
else if (backwardCacheItems === 0) {
break;
}
}
}
}
}
return download;
}
private purgeCacheItems(downloadItem: number, downloadedBytes: number): boolean {
let underQuota = true;
while (this.cacheSize + downloadedBytes > this.quota) {
let purgeIndex = this.playlistIndex;
let purgeDistance = 0;
logger.debug(`Downloading item ${downloadItem} with playlist index ${this.playlistIndex} and cache window: [${this.cacheWindowStart} - ${this.cacheWindowEnd}]`);
// Priority:
// 1. Purge first encountered item outside cache window
// 2. Purge item furthest from view index inside window (except next item from view index)
for (let index of this.cache.keys()) {
if (index === downloadItem || index === this.playlistIndex || index === this.playlistIndex + 1) {
continue;
}
if (index < this.cacheWindowStart || index > this.cacheWindowEnd) {
purgeIndex = index;
break;
}
else if (Math.abs(this.playlistIndex - index) > purgeDistance) {
purgeDistance = Math.abs(this.playlistIndex - index);
purgeIndex = index;
}
}
if (purgeIndex !== this.playlistIndex) {
const deleteItem = this.cache.get(purgeIndex);
fs.unlinkSync(deleteItem.path);
this.cacheSize -= deleteItem.size;
this.cacheUrlMap.delete(deleteItem.url);
this.cache.delete(purgeIndex);
this.updateCacheWindow();
logger.info(`Item ${downloadItem} pending download (${downloadedBytes} bytes) cannot fit in cache, purging ${purgeIndex} from cache. Remaining quota ${this.quota - this.cacheSize} bytes`);
}
else {
// Cannot purge current item since we may already be streaming it
logger.warn(`Aborting item caching, cannot fit item ${downloadItem} (${downloadedBytes} bytes) within remaining space quota (${this.quota - this.cacheSize} bytes)`);
underQuota = false;
break;
}
}
return underQuota;
}
private finalizeCacheItem(cacheObject: CacheObject, index: number, size: number, playlistIndex: number) {
private finalizeCacheItem(cacheObject: CacheObject, index: number) {
const size = fs.statSync(cacheObject.path).size;
cacheObject.size = size;
this.cacheSize += size;
logger.info(`Cached item ${index} (${cacheObject.size} bytes) with remaining quota ${this.quota - this.cacheSize} bytes: ${cacheObject.url}`);
this.cache.set(index, cacheObject);
this.cacheUrlMap.set(cacheObject.url, index);
this.updateCacheWindow(playlistIndex);
this.updateCacheWindow();
}
private updateCacheWindow(playlistIndex: number) {
private updateCacheWindow() {
const indexList = [...this.cache.keys()].sort((a, b) => a - b);
if (this.playlist.forwardCache && this.playlist.forwardCache > 0) {
let forwardCacheItems = this.playlist.forwardCache;
for (let index of this.cache.keys()) {
if (index > playlistIndex) {
for (let index of indexList) {
if (index > this.playlistIndex) {
forwardCacheItems--;
if (forwardCacheItems === 0) {
@ -230,13 +316,13 @@ export class MediaCache {
}
}
else {
this.cacheWindowEnd = playlistIndex;
this.cacheWindowEnd = this.playlistIndex;
}
if (this.playlist.backwardCache && this.playlist.backwardCache > 0) {
let backwardCacheItems = this.playlist.backwardCache;
for (let index of this.cache.keys()) {
if (index < playlistIndex) {
for (let index of indexList) {
if (index < this.playlistIndex) {
backwardCacheItems--;
if (backwardCacheItems === 0) {
@ -247,7 +333,7 @@ export class MediaCache {
}
}
else {
this.cacheWindowStart = playlistIndex
this.cacheWindowStart = this.playlistIndex
}
}
}

20
receivers/common/web/UtilityBackend.ts
View file

@ -1,4 +1,5 @@
import * as fs from 'fs';
import * as url from 'url';
import { http, https } from 'modules/follow-redirects';
import * as memfs from 'modules/memfs';
import { Logger, LoggerType } from 'common/Logger';
@ -35,14 +36,22 @@ export async function fetchJSON(url: string): Promise<any> {
});
}
export async function downloadFile(url: string, destination: string, startCb: (downloadSize: number) => boolean = null, progressCb: (downloadedBytes: number, downloadSize: number) => void = null, finishCb: (downloadedBytes: number) => void = null, inMemory: boolean = false): Promise<void> {
export async function downloadFile(downloadUrl: string, destination: string, inMemory: boolean = false, requestHeaders: { [key: string]: string } = null,
startCb: (downloadSize: number) => boolean = null,
progressCb: (downloadedBytes: number, downloadSize: number) => void = null): Promise<void> {
return new Promise((resolve, reject) => {
const file = inMemory ? memfs.fs.createWriteStream(destination) : fs.createWriteStream(destination);
const protocol = url.startsWith('https') ? https : http;
const protocol = downloadUrl.startsWith('https') ? https : http;
protocol.get(url, (response) => {
const parsedUrl = url.parse(downloadUrl);
const options = protocol.RequestOptions = {
...parsedUrl,
headers: requestHeaders
};
protocol.get(options, (response) => {
const downloadSize = Number(response.headers['content-length']);
logger.info(`Downloading file ${url} to ${destination} with size: ${downloadSize} bytes`);
logger.info(`Downloading file ${downloadUrl} to ${destination} with size: ${downloadSize} bytes`);
if (startCb) {
if (!startCb(downloadSize)) {
file.close();
@ -61,9 +70,6 @@ export async function downloadFile(url: string, destination: string, startCb: (d
});
file.on('finish', () => {
file.close();
if (finishCb) {
finishCb(downloadedBytes);
}
resolve();
});
}).on('error', (err) => {

21
receivers/electron/src/Main.ts
View file

@ -24,7 +24,6 @@ class AppCache {
public appVersion: string = null;
public playMessage: PlayMessage = null;
public playerVolume: number = null;
public playlist: PlaylistContent = null;
public subscribedKeys = new Set<string>();
}
@ -181,7 +180,6 @@ export class Main {
case ContentType.Playlist: {
rendererMessage = json as PlaylistContent;
rendererEvent = 'play-playlist';
Main.cache.playlist = rendererMessage;
if ((rendererMessage.forwardCache && rendererMessage.forwardCache > 0) || (rendererMessage.backwardCache && rendererMessage.backwardCache > 0)) {
Main.mediaCache?.destroy();
@ -322,23 +320,8 @@ export class Main {
ipcMain.on('play-request', (event: IpcMainEvent, value: PlayMessage, playlistIndex: number) => {
logger.debug(`Received play request for index ${playlistIndex}:`, value);
if (Main.cache.playlist.forwardCache && Main.cache.playlist.forwardCache > 0) {
if (Main.mediaCache.has(playlistIndex)) {
value.url = Main.mediaCache.getUrl(playlistIndex);
}
Main.mediaCache.cacheForwardItems(playlistIndex + 1, Main.cache.playlist.forwardCache, playlistIndex);
}
if (Main.cache.playlist.backwardCache && Main.cache.playlist.backwardCache > 0) {
if (Main.mediaCache.has(playlistIndex)) {
value.url = Main.mediaCache.getUrl(playlistIndex);
}
Main.mediaCache.cacheBackwardItems(playlistIndex - 1, Main.cache.playlist.backwardCache, playlistIndex);
}
value.url = Main.mediaCache.has(playlistIndex) ? Main.mediaCache.getUrl(playlistIndex) : value.url;
Main.mediaCache.cacheItems(playlistIndex);
Main.play(value);
});
ipcMain.on('send-download-request', async () => {

2
receivers/electron/src/Updater.ts
View file

@ -365,7 +365,7 @@ export class Updater {
const destination = path.join(Updater.updateDataPath, file);
logger.info(`Downloading '${fileInfo.url}' to '${destination}'.`);
Updater.isDownloading = true;
await downloadFile(fileInfo.url.toString(), destination, null, (downloadedBytes: number, downloadSize: number) => {
await downloadFile(fileInfo.url.toString(), destination, false, null, null, (downloadedBytes: number, downloadSize: number) => {
Updater.updateProgress = downloadedBytes / downloadSize;
});