Add fMP4 playback support (HEVC, AV1) in HLS.js

Tested codecs:
video: h264, hevc, av1 (av1 in firefox require 1.5.0)
audio: mp3, aac, ac3, eac3 (flac and opus require 1.5.0)

Tested browsers:
Chrome, Firefox, Edge Chromium, Safari and their mobile versions

Signed-off-by: nyanmisaka <nst799610810@gmail.com>

src/components/apphost.js

@@ -16,10 +16,13 @@ function getBaseProfileOptions(item) {
         if (browser.edge) {
             disableHlsVideoAudioCodecs.push('mp3');
         }
-
+        if (!browser.edgeChromium) {
-        disableHlsVideoAudioCodecs.push('ac3');
+            disableHlsVideoAudioCodecs.push('ac3');
-        disableHlsVideoAudioCodecs.push('eac3');
+            disableHlsVideoAudioCodecs.push('eac3');
-        disableHlsVideoAudioCodecs.push('opus');
+        }
+        if (!(browser.chrome || browser.edgeChromium || browser.firefox)) {
+            disableHlsVideoAudioCodecs.push('opus');
+        }
     }
 
     return {

src/components/htmlMediaHelper.js

@@ -43,8 +43,8 @@ export function enableHlsJsPlayer(runTimeTicks, mediaType) {
     }
 
     if (canPlayNativeHls()) {
-        // Having trouble with chrome's native support and transcoded music
+        // Android Webview's native HLS has performance and compatiblity issues
-        if (browser.android && mediaType === 'Audio') {
+        if (browser.android && (mediaType === 'Audio' || mediaType === 'Video')) {
             return true;
         }
 

src/controllers/dashboard/encodingsettings.html

@@ -124,6 +124,12 @@
                             <span>${AllowHevcEncoding}</span>
                         </label>
                     </div>
+                    <div class="checkboxList">
+                        <label>
+                            <input type="checkbox" is="emby-checkbox" id="chkAllowAv1Encoding" />
+                            <span>${AllowAv1Encoding}</span>
+                        </label>
+                    </div>
                 </div>
 
                 <div class="vppTonemappingOptions hide">

src/controllers/dashboard/encodingsettings.js

@@ -18,6 +18,7 @@ function loadPage(page, config, systemInfo) {
     page.querySelector('#chkIntelLpHevcHwEncoder').checked = config.EnableIntelLowPowerHevcHwEncoder;
     page.querySelector('#chkHardwareEncoding').checked = config.EnableHardwareEncoding;
     page.querySelector('#chkAllowHevcEncoding').checked = config.AllowHevcEncoding;
+    page.querySelector('#chkAllowAv1Encoding').checked = config.AllowAv1Encoding;
     $('#selectVideoDecoder', page).val(config.HardwareAccelerationType);
     $('#selectThreadCount', page).val(config.EncodingThreadCount);
     page.querySelector('#chkEnableAudioVbr').checked = config.EnableAudioVbr;
@@ -123,6 +124,7 @@ function onSubmit() {
             config.EnableIntelLowPowerHevcHwEncoder = form.querySelector('#chkIntelLpHevcHwEncoder').checked;
             config.EnableHardwareEncoding = form.querySelector('#chkHardwareEncoding').checked;
             config.AllowHevcEncoding = form.querySelector('#chkAllowHevcEncoding').checked;
+            config.AllowAv1Encoding = form.querySelector('#chkAllowAv1Encoding').checked;
             ApiClient.updateNamedConfiguration('encoding', config).then(function () {
                 updateEncoder(form);
             }, function () {

src/scripts/browserDeviceProfile.js

@@ -23,6 +23,17 @@ function canPlayHevc(videoTestElement, options) {
         || videoTestElement.canPlayType('video/mp4; codecs="hev1.1.0.L120"').replace(/no/, ''));
 }
 
+function canPlayAv1(videoTestElement) {
+    if (browser.tizenVersion >= 5.5 || browser.web0sVersion >= 5) {
+        return true;
+    }
+
+    // av1 main level 5.3
+    return !!videoTestElement.canPlayType
+        && (videoTestElement.canPlayType('video/mp4; codecs="av01.0.15M.08"').replace(/no/, '')
+        && videoTestElement.canPlayType('video/mp4; codecs="av01.0.15M.10"').replace(/no/, ''));
+}
+
 let _supportsTextTracks;
 function supportsTextTracks() {
     if (browser.tizen) {
@@ -56,6 +67,14 @@ function canPlayNativeHls() {
             || media.canPlayType('application/vnd.apple.mpegURL').replace(/no/, ''));
 }
 
+function canPlayNativeHlsInFmp4() {
+    if (browser.tizenVersion >= 3 || browser.web0sVersion >= 3.5) {
+        return true;
+    }
+
+    return (browser.iOS && browser.iOSVersion >= 11) || browser.osx;
+}
+
 function canPlayHlsWithMSE() {
     // text tracks don’t work with this in firefox
     return window.MediaSource != null; /* eslint-disable-line compat/compat */
@@ -157,14 +176,6 @@ function testCanPlayMkv(videoTestElement) {
     return !!browser.edgeUwp;
 }
 
-function testCanPlayAv1(videoTestElement) {
-    if (browser.tizenVersion >= 5.5 || browser.web0sVersion >= 5) {
-        return true;
-    }
-
-    return videoTestElement.canPlayType('video/webm; codecs="av01.0.15M.10"').replace(/no/, '');
-}
-
 function testCanPlayTs() {
     return browser.tizen || browser.web0s || browser.edgeUwp;
 }
@@ -437,8 +448,15 @@ export default function (options) {
     // Do not use AC3 for audio transcoding unless AAC and MP3 are not supported.
     if (canPlayAc3VideoAudio) {
         videoAudioCodecs.push('ac3');
+        if (browser.edgeChromium) {
+            hlsInFmp4VideoAudioCodecs.push('ac3');
+        }
+
         if (canPlayEac3VideoAudio) {
             videoAudioCodecs.push('eac3');
+            if (browser.edgeChromium) {
+                hlsInFmp4VideoAudioCodecs.push('eac3');
+            }
         }
 
         if (canPlayAc3VideoAudioInHls) {
@@ -492,6 +510,9 @@ export default function (options) {
         if (browser.tizen) {
             hlsInTsVideoAudioCodecs.push('opus');
         }
+        if (!browser.safari) {
+            hlsInFmp4VideoAudioCodecs.push('opus');
+        }
     }
 
     if (canPlayAudioFormat('flac')) {
@@ -521,17 +542,22 @@ export default function (options) {
     const hlsInTsVideoCodecs = [];
     const hlsInFmp4VideoCodecs = [];
 
-    if ((browser.safari || browser.tizen || browser.web0s) && canPlayHevc(videoTestElement, options)) {
+    if (canPlayAv1(videoTestElement)
+        && !browser.mobile && (browser.edgeChromium || browser.firefox || browser.chrome)) {
+        // disable av1 on mobile since it can be very slow software decoding
+        hlsInFmp4VideoCodecs.push('av1');
+    }
+
+    if (canPlayHevc(videoTestElement, options)
+        && (browser.edgeChromium || browser.safari || browser.tizen || browser.web0s || (browser.chrome && (!browser.android || browser.chrome.versionMajor >= 105)))) {
+        // Chromium used to support HEVC on Android but not via MSE
         hlsInFmp4VideoCodecs.push('hevc');
     }
 
     if (canPlayH264(videoTestElement)) {
         mp4VideoCodecs.push('h264');
         hlsInTsVideoCodecs.push('h264');
-
+        hlsInFmp4VideoCodecs.push('h264');
-        if (browser.safari || browser.tizen || browser.web0s) {
-            hlsInFmp4VideoCodecs.push('h264');
-        }
     }
 
     if (canPlayHevc(videoTestElement, options)) {
@@ -566,7 +592,7 @@ export default function (options) {
         webmVideoCodecs.push('vp9');
     }
 
-    if (testCanPlayAv1(videoTestElement)) {
+    if (canPlayAv1(videoTestElement)) {
         mp4VideoCodecs.push('av1');
         webmVideoCodecs.push('av1');
     }
@@ -687,7 +713,11 @@ export default function (options) {
     });
 
     if (canPlayHls() && options.enableHls !== false) {
-        if (hlsInFmp4VideoCodecs.length && hlsInFmp4VideoAudioCodecs.length && userSettings.preferFmp4HlsContainer() && (browser.safari || browser.tizen || browser.web0s)) {
+        let enableFmp4Hls = userSettings.preferFmp4HlsContainer();
+        if ((browser.safari || browser.tizen || browser.web0s) && !canPlayNativeHlsInFmp4()) {
+            enableFmp4Hls = false;
+        }
+        if (hlsInFmp4VideoCodecs.length && hlsInFmp4VideoAudioCodecs.length && enableFmp4Hls) {
             profile.TranscodingProfiles.push({
                 Container: 'mp4',
                 Type: 'Video',
@@ -817,6 +847,33 @@ export default function (options) {
         hevcProfiles = 'main|main 10';
     }
 
+    let maxAv1Level = 15; // level 5.3
+    const av1Profiles = 'main'; // av1 main covers 4:2:0 8 & 10 bits
+
+    // av1 main level 6.0
+    if (videoTestElement.canPlayType('video/mp4; codecs="av01.0.16M.08"').replace(/no/, '')
+            && videoTestElement.canPlayType('video/mp4; codecs="av01.0.16M.10"').replace(/no/, '')) {
+        maxAv1Level = 16;
+    }
+
+    // av1 main level 6.1
+    if (videoTestElement.canPlayType('video/mp4; codecs="av01.0.17M.08"').replace(/no/, '')
+            && videoTestElement.canPlayType('video/mp4; codecs="av01.0.17M.10"').replace(/no/, '')) {
+        maxAv1Level = 17;
+    }
+
+    // av1 main level 6.2
+    if (videoTestElement.canPlayType('video/mp4; codecs="av01.0.18M.08"').replace(/no/, '')
+            && videoTestElement.canPlayType('video/mp4; codecs="av01.0.18M.10"').replace(/no/, '')) {
+        maxAv1Level = 18;
+    }
+
+    // av1 main level 6.3
+    if (videoTestElement.canPlayType('video/mp4; codecs="av01.0.19M.08"').replace(/no/, '')
+            && videoTestElement.canPlayType('video/mp4; codecs="av01.0.19M.10"').replace(/no/, '')) {
+        maxAv1Level = 19;
+    }
+
     const h264VideoRangeTypes = 'SDR';
     let hevcVideoRangeTypes = 'SDR';
     let vp9VideoRangeTypes = 'SDR';
@@ -830,12 +887,15 @@ export default function (options) {
     }
 
     if (browser.tizen || browser.web0s) {
-        hevcVideoRangeTypes += '|HDR10|HLG|DOVI';
+        hevcVideoRangeTypes += '|HDR10|HLG';
         vp9VideoRangeTypes += '|HDR10|HLG';
         av1VideoRangeTypes += '|HDR10|HLG';
     }
 
-    if (browser.edgeChromium || browser.chrome || browser.firefox) {
+    // Chrome mobile and Firefox have no client side tone-mapping
+    // Edge Chromium on Nvidia is known to have color issues on 10-bit video
+    if (browser.chrome && !browser.mobile) {
+        hevcVideoRangeTypes += '|HDR10|HLG';
         vp9VideoRangeTypes += '|HDR10|HLG';
         av1VideoRangeTypes += '|HDR10|HLG';
     }
@@ -904,11 +964,29 @@ export default function (options) {
     ];
 
     const av1CodecProfileConditions = [
+        {
+            Condition: 'NotEquals',
+            Property: 'IsAnamorphic',
+            Value: 'true',
+            IsRequired: false
+        },
+        {
+            Condition: 'EqualsAny',
+            Property: 'VideoProfile',
+            Value: av1Profiles,
+            IsRequired: false
+        },
         {
             Condition: 'EqualsAny',
             Property: 'VideoRangeType',
             Value: av1VideoRangeTypes,
             IsRequired: false
+        },
+        {
+            Condition: 'LessThanEqual',
+            Property: 'VideoLevel',
+            Value: maxAv1Level.toString(),
+            IsRequired: false
         }
     ];
 
@@ -942,6 +1020,13 @@ export default function (options) {
             Value: maxVideoWidth.toString(),
             IsRequired: false
         });
+
+        av1CodecProfileConditions.push({
+            Condition: 'LessThanEqual',
+            Property: 'Width',
+            Value: maxVideoWidth.toString(),
+            IsRequired: false
+        });
     }
 
     const globalMaxVideoBitrate = (getGlobalMaxVideoBitrate() || '').toString();
@@ -950,6 +1035,8 @@ export default function (options) {
 
     const hevcMaxVideoBitrate = globalMaxVideoBitrate;
 
+    const av1MaxVideoBitrate = globalMaxVideoBitrate;
+
     if (h264MaxVideoBitrate) {
         h264CodecProfileConditions.push({
             Condition: 'LessThanEqual',
@@ -968,6 +1055,15 @@ export default function (options) {
         });
     }
 
+    if (av1MaxVideoBitrate) {
+        av1CodecProfileConditions.push({
+            Condition: 'LessThanEqual',
+            Property: 'VideoBitrate',
+            Value: av1MaxVideoBitrate,
+            IsRequired: true
+        });
+    }
+
     // On iOS 12.x, for TS container max h264 level is 4.2
     if (browser.iOS && browser.iOSVersion < 13) {
         const codecProfile = {

src/strings/en-us.json

@@ -1715,5 +1715,6 @@
     "Unreleased": "Not yet released",
     "LabelTonemappingMode": "Tone mapping mode",
     "TonemappingModeHelp": "Select the tone mapping mode. If you experience blown out highlights try switching to the RGB mode.",
-    "Unknown": "Unknown"
+    "Unknown": "Unknown",
+    "AllowAv1Encoding": "Allow encoding in AV1 format"
 }