diff --git a/SABR_CLOUD_POLICY.md b/SABR_CLOUD_POLICY.md deleted file mode 100644 index b07d20ac7..000000000 --- a/SABR_CLOUD_POLICY.md +++ /dev/null @@ -1,51 +0,0 @@ -# SABR cloud policy delivery - -Release builds use the PipePipe policy repository by default. Deployments can override it with two -build environment variables: - -- `SABR_POLICY_PUBLIC_KEY_BASE64`: raw 32-byte Ed25519 public key encoded as Base64. -- `SABR_POLICY_URL`: HTTPS endpoint serving the UTF-8 JSON document accepted by - `SabrPolicyRuntime.installDocument`. - -The production defaults are: - -- Repository: `https://github.com/InfinityLoop1308/PipePipeSabrPolicies` -- Policy URL: - `https://raw.githubusercontent.com/InfinityLoop1308/PipePipeSabrPolicies/main/policy.json` -- Raw Ed25519 public key: `Yyi5q4s0ikpgAitzw+62H8+ggPt+dTILJ0Of4vSIcrU=` - -When both values are present, the client restores the last verified policy at startup, requests an -update immediately, and schedules a connected-network refresh every six hours. The endpoint must -return the signed policy document as the response body with HTTP 200. HTTP 204 is treated as no -update. Policies are signature checked, time bounded, and revision monotonic before activation. - -The document is deliberately human-readable and suitable for publication in a public source -repository: - -```json -{ - "format": 1, - "revision": 1001, - "validFromMs": 1784390400000, - "validUntilMs": 1792166400000, - "source": "function createSabrPolicy(sabr) { /* ... */ }", - "signature": "base64-encoded Ed25519 signature" -} -``` - -The Ed25519 signature covers the canonical `SabrScriptPolicy` payload reconstructed from revision, -validity bounds, and source. JSON whitespace and key order are not signed and may be changed without -invalidating the document. Numeric metadata must use exact JSON integers; fractions, scientific -notation, strings, and values outside the signed 64-bit range are rejected rather than coerced. The -client keeps its verified cache in a private binary format; that cache is never downloaded or -executed as a binary. - -If a policy throws while building requests, interpreting responses, routing demanded segments, or -decoding media headers, the client removes that cached policy, retains its highest revision to -prevent rollback, and uses the builtin policy until a higher signed revision is delivered. - -Policies that declare `demand: true` can route reader-demand retries and decide how to handle exact -target omissions using bounded, payload-free returned segment identities. Policies without that -declaration retain the builtin demand behavior, so an older valid cloud policy remains compatible -with a client that supports the expanded contract. See the Extractor's -`SABR_JAVASCRIPT_POLICY.md` for the complete event and decision schema. diff --git a/app/build.gradle b/app/build.gradle index 92c69aee3..86560e3ab 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -10,8 +10,6 @@ def keyPath = System.getenv("KEY_PATH") def keyStorePassword = System.getenv("KEY_STORE_PASSWORD") def signingKeyAlias = System.getenv("KEY_ALIAS") def signingKeyPassword = System.getenv("KEY_PASSWORD") -def sabrPolicyPublicKey = "" -def sabrPolicyUrl = "" def baseVersionCode = 1105 def appVersionName = "5.2.4" def abiCodes = ['armeabi-v7a': 1, 'x86': 2, 'x86_64': 3, 'arm64-v8a': 4] @@ -32,10 +30,6 @@ android { testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" vectorDrawables.useSupportLibrary = true - buildConfigField "String", "SABR_POLICY_PUBLIC_KEY_BASE64", - "\"${sabrPolicyPublicKey.replace('\\', '\\\\').replace('"', '\\"')}\"" - buildConfigField "String", "SABR_POLICY_URL", - "\"${sabrPolicyUrl.replace('\\', '\\\\').replace('"', '\\"')}\"" javaCompileOptions { annotationProcessorOptions { arguments = ["room.schemaLocation": "$projectDir/schemas".toString()] @@ -190,7 +184,6 @@ dependencies { /** NewPipe libraries **/ implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' implementation group: 'com.github.InfinityLoop1308.PipePipeExtractor', name: 'extractor' - implementation 'io.github.dokar3:quickjs-kt:1.0.5' /** Kotlin **/ implementation "org.jetbrains.kotlin:kotlin-stdlib" @@ -307,6 +300,7 @@ dependencies { implementation 'com.arthenica:smart-exception-java:0.2.1' /** Instrumented integration tests **/ + testImplementation 'junit:junit:4.13.2' androidTestImplementation 'androidx.test:runner:1.7.0' androidTestImplementation 'androidx.test.ext:junit:1.3.0' androidTestImplementation 'junit:junit:4.13.2' diff --git a/app/src/androidTest/assets/equivalent-builtin-sabr-policy.js b/app/src/androidTest/assets/equivalent-builtin-sabr-policy.js deleted file mode 100644 index c2703e902..000000000 --- a/app/src/androidTest/assets/equivalent-builtin-sabr-policy.js +++ /dev/null @@ -1,159 +0,0 @@ -function createSabrPolicy(sabr) { - function text(bytes) { - var result = '', index = 0, first, second, third, code; - while (index < bytes.length) { - first = bytes[index++]; - if (first < 128) { - result += String.fromCharCode(first); - } else if (first < 224) { - second = bytes[index++]; - result += String.fromCharCode(((first & 31) << 6) | (second & 63)); - } else if (first < 240) { - second = bytes[index++]; - third = bytes[index++]; - result += String.fromCharCode(((first & 15) << 12) - | ((second & 63) << 6) | (third & 63)); - } else { - second = bytes[index++]; - third = bytes[index++]; - code = ((first & 7) << 18) | ((second & 63) << 12) - | ((third & 63) << 6) | (bytes[index++] & 63); - code -= 65536; - result += String.fromCharCode(55296 + (code >> 10), 56320 + (code & 1023)); - } - } - return result; - } - - function mediaHeader(event) { - var fields = sabr.proto.decode(sabr.base64.decode(event.data)); - var result = {}; - var timeRange = null; - var index, field, nested, nestedIndex, nestedField; - for (index = 0; index < fields.length; index++) { - field = fields[index]; - if (field.n === 1) result.headerId = field.v; - else if (field.n === 2) result.videoId = text(field.b); - else if (field.n === 3) result.itag = field.v; - else if (field.n === 4) result.lastModified = field.v; - else if (field.n === 5) result.xtags = text(field.b); - else if (field.n === 6) result.startRange = field.v; - else if (field.n === 7) result.compressionAlgorithm = field.v; - else if (field.n === 8) result.initSegment = field.v !== 0; - else if (field.n === 9) result.sequenceNumber = field.v; - else if (field.n === 10) result.bitrateBps = field.v; - else if (field.n === 11) result.startMs = field.v; - else if (field.n === 12) result.durationMs = field.v; - else if (field.n === 13) { - nested = sabr.proto.decode(field.b); - for (nestedIndex = 0; nestedIndex < nested.length; nestedIndex++) { - nestedField = nested[nestedIndex]; - if (nestedField.n === 1 && result.itag === undefined) result.itag = nestedField.v; - else if (nestedField.n === 2 && result.lastModified === undefined) { - result.lastModified = nestedField.v; - } else if (nestedField.n === 3 && result.xtags === undefined) { - result.xtags = text(nestedField.b); - } - } - } else if (field.n === 14) result.contentLength = field.v; - else if (field.n === 15) { - timeRange = {}; - nested = sabr.proto.decode(field.b); - for (nestedIndex = 0; nestedIndex < nested.length; nestedIndex++) { - nestedField = nested[nestedIndex]; - if (nestedField.n === 1) timeRange.start = nestedField.v; - else if (nestedField.n === 2) timeRange.duration = nestedField.v; - else if (nestedField.n === 3) timeRange.timescale = nestedField.v; - } - } else if (field.n === 16) result.sequenceLastModified = field.v; - } - if (timeRange) { - result.timeRangeStartTicks = timeRange.start; - result.timeRangeDurationTicks = timeRange.duration; - result.timeRangeTimescale = timeRange.timescale; - if (timeRange.timescale > 0) { - if (result.startMs === undefined && timeRange.start >= 0) { - result.startMs = Math.floor(timeRange.start * 1000 / timeRange.timescale); - } - if (result.durationMs === undefined && timeRange.duration >= 0) { - result.durationMs = Math.floor(timeRange.duration * 1000 / timeRange.timescale); - } - } - } - return result; - } - - function response(event) { - var actions = ['APPLY_BUILTIN_RESPONSE_STATE']; - var state = { - redirectCount: event.redirectCount, - poTokenRefreshes: event.poTokenRefreshes - }; - var redirect = event.builtin.redirectUrl; - var backoff = Math.max(0, event.builtin.backoffMs || 0); - if (redirect) { - actions.push('APPLY_REDIRECT'); - state.redirectCount++; - } - if (event.builtin.error) { - actions.push('FAIL_SABR_ERROR'); - return {actions: actions, state: state, redirectUrl: redirect, - errorDetails: 'SABR error'}; - } - if (event.builtin.reload) { - actions.push('TRY_RELOAD'); - return {actions: actions, state: state, redirectUrl: redirect}; - } - if (event.builtin.protection) { - actions.push(event.mode === 'FETCH_SEGMENT' ? 'REQUIRE_PO_TOKEN' : 'REFRESH_PO_TOKEN'); - } - if (event.mode === 'PUMP' && event.segmentCount > 0) { - state.redirectCount = 0; - state.poTokenRefreshes = 0; - actions.push('RESET_RECOVERY_BUDGETS'); - } - if (backoff > 0) { - actions.push(event.honorBackoff ? 'SLEEP_BACKOFF' : 'DEFER_BACKOFF'); - } else if (!event.honorBackoff) { - actions.push('CLEAR_DEMAND_BACKOFF'); - } - actions.push(event.mode === 'FETCH_SEGMENT' && event.builtin.protection - ? 'RETRY' : 'CONTINUE'); - return {actions: actions, state: state, backoffMs: backoff, redirectUrl: redirect}; - } - - function demandRoute(event) { - var recover = event.responsesWithoutDemandedSegment > event.recoveryCount; - if (event.targetStartMs < event.bufferedEdgeMs) { - return {route: recover ? 'RECOVER_REWIND' : 'REWIND'}; - } - if (event.targetStartMs > event.bufferedEdgeMs + 30000) { - return {route: recover ? 'RECOVER_FORWARD' : 'FORWARD'}; - } - return {route: recover ? 'RECOVER_MISSING' : 'STREAM'}; - } - - function demandResponse(event) { - if (event.responsesWithoutDemandedSegment >= 3) { - return {outcome: 'FAIL_REPEATED_TARGET_OMISSION', retryDelayMs: 0}; - } - if (event.elapsedMs >= 15000) { - return {outcome: event.targetTrackSegmentCount > 0 - ? 'FAIL_REPEATED_TARGET_OMISSION' : 'FAIL_NO_TARGET_MEDIA', retryDelayMs: 0}; - } - return {outcome: 'CONTINUE', retryDelayMs: 0}; - } - - return { - describe: function () { - return {demand: true, media: {headerType: 20, mediaType: 21, endType: 22, - headerDecoder: 'builtin'}}; - }, - initialRequest: function (event) { return {body: event.fallbackBody}; }, - followUpRequest: function (event) { return {body: event.fallbackBody}; }, - response: response, - demandRoute: demandRoute, - demandResponse: demandResponse, - mediaHeader: mediaHeader - }; -} diff --git a/app/src/androidTest/java/org/schabi/newpipe/YoutubeSessionPoTokenSettingTest.java b/app/src/androidTest/java/org/schabi/newpipe/YoutubeSessionPoTokenSettingTest.java deleted file mode 100644 index a585347a5..000000000 --- a/app/src/androidTest/java/org/schabi/newpipe/YoutubeSessionPoTokenSettingTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.schabi.newpipe; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import androidx.test.ext.junit.runners.AndroidJUnit4; - -import org.junit.Test; -import org.junit.runner.RunWith; - -@RunWith(AndroidJUnit4.class) -public final class YoutubeSessionPoTokenSettingTest { - @Test - public void androidVrAlwaysGetsSessionPoTokens() { - assertTrue(App.shouldProvideYoutubeSessionPoToken("ANDROID_VR", false)); - assertTrue(App.shouldProvideYoutubeSessionPoToken("ANDROID_VR", true)); - } - - @Test - public void otherClientsFollowVisitorDataSetting() { - assertFalse(App.shouldProvideYoutubeSessionPoToken("WEB", false)); - assertFalse(App.shouldProvideYoutubeSessionPoToken("MWEB", false)); - assertTrue(App.shouldProvideYoutubeSessionPoToken("WEB", true)); - assertTrue(App.shouldProvideYoutubeSessionPoToken("MWEB", true)); - } -} diff --git a/app/src/androidTest/java/org/schabi/newpipe/player/SabrPlaybackSmokeTest.java b/app/src/androidTest/java/org/schabi/newpipe/player/SabrPlaybackSmokeTest.java index 3fe28d5f9..267d3dd39 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/player/SabrPlaybackSmokeTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/player/SabrPlaybackSmokeTest.java @@ -36,6 +36,7 @@ import org.junit.runner.RunWith; import org.schabi.newpipe.App; import org.schabi.newpipe.DownloaderImpl; +import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.downloader.CancellableCall; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Request; @@ -46,6 +47,7 @@ import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.playlist.PlaylistInfo; import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment; import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRequestDumper; import org.schabi.newpipe.extractor.services.youtube.sabr.SabrResponseDecoder; @@ -57,12 +59,14 @@ import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.StreamInfo; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.player.datasource.SabrDashMediaSource; import org.schabi.newpipe.player.datasource.SabrSegmentDataSource; import org.schabi.newpipe.player.helper.LegacySubtitleRenderersFactory; import org.schabi.newpipe.player.helper.LoadController; import org.schabi.newpipe.player.helper.PlayerDataSource; +import org.schabi.newpipe.player.resolver.AudioPlaybackResolver; import org.schabi.newpipe.player.resolver.QualityResolver; import org.schabi.newpipe.player.resolver.VideoPlaybackResolver; import org.schabi.newpipe.player.datasource.SabrSessionStore; @@ -131,6 +135,42 @@ public void extractorToMedia3PlaysAndSeeks() throws Exception { runSmokeCase(SmokeCase.playback()); } + @Test + public void anonymousSequentialAudioCrossesSabrProtectionBoundaries() throws Exception { + final Bundle arguments = InstrumentationRegistry.getArguments(); + final String playlistUrl = arguments.getString("anonymousPlaylistUrl", ""); + assumeTrue("Set anonymousPlaylistUrl to run the sequential anonymous SABR probe", + !playlistUrl.isEmpty()); + + final Context context = InstrumentationRegistry.getInstrumentation() + .getTargetContext().getApplicationContext(); + assertTrue("The target process must use PipePipe's App initialization", + context instanceof App); + final int videoCount = Integer.parseInt(arguments.getString( + "anonymousVideoCount", "10")); + final long playbackMs = Long.parseLong(arguments.getString( + "anonymousPlaybackMs", "130000")); + assertTrue("The probe must cross the 60s SABR protection boundary", + playbackMs > 60_000); + + ServiceList.YouTube.setTokens(""); + NewPipe.setYoutubePlayerClient("mweb"); + + final PlaylistInfo playlist = PlaylistInfo.getInfo(ServiceList.YouTube, playlistUrl); + assertTrue("Playlist has fewer items than requested: requested=" + videoCount + + " actual=" + playlist.getRelatedItems().size(), + playlist.getRelatedItems().size() >= videoCount); + + for (int index = 0; index < videoCount; index++) { + final StreamInfoItem item = playlist.getRelatedItems().get(index); + final StreamInfo info = StreamInfo.getInfo(ServiceList.YouTube, item.getUrl()); + assertTrue("Extractor returned no SABR audio stream for item=" + index + + " video=" + info.getId(), + info.getAudioStreams().stream().anyMatch(SabrPlaybackSmokeTest::isSabr)); + runAnonymousAudioWindow(context, info, index, playbackMs); + } + } + @Test public void recoversMissingInitializationFromPump() throws Exception { runSmokeCase(SmokeCase.missingInitialization()); @@ -487,45 +527,59 @@ public void pumpBackoffPublishesStandaloneNotificationWhileBuffering() throws Ex } @Test - public void repeatedProtectedNoMediaResponsesHonorBackoffAndRecover() throws Exception { + public void rejectedAttestationFailsWithoutEnteringBackoff() throws Exception { try (SabrSmokeHarness harness = SabrSmokeHarness.create()) { harness.downloader.enqueue(new UmpFixture() .segment(1, SMOKE_VIDEO_ITAG, 1, 0, 30_000) .bytes()); - for (int i = 0; i < 3; i++) { - harness.downloader.enqueue(new UmpFixture() - .part(SabrResponseDecoder.STREAM_PROTECTION_STATUS, - streamProtection(3, 20)) - .part(SabrResponseDecoder.NEXT_REQUEST_POLICY, - nextRequestPolicy(2_000)) - .bytes()); - } harness.downloader.enqueue(new UmpFixture() - .segment(5, SMOKE_VIDEO_ITAG, 2, 30_000, 5_000) + .part(SabrResponseDecoder.STREAM_PROTECTION_STATUS, + streamProtection(3, 20)) + .part(SabrResponseDecoder.NEXT_REQUEST_POLICY, + nextRequestPolicy(59_000)) .bytes()); - final long elapsedMs = harness.openMediaSegment( - SabrSegmentRequest.media(harness.videoFormat, 2), 10_000); + final long startMs = System.currentTimeMillis(); + harness.openMediaSegmentExpectFailure( + SabrSegmentRequest.media(harness.videoFormat, 2), 5_000); + final long elapsedMs = System.currentTimeMillis() - startMs; final String trace = harness.holder.session.getDiagnosticTrace(); - final List requestTimesMs = harness.downloader.requestTimesSnapshot(); - assertTrue("Repeated protection responses were not exercised: " + trace, + assertTrue("Rejected attestation response was not exercised: " + trace, trace.contains("protection=3/20")); - assertTrue("Expected one initial request, three empty responses, and recovery: " - + requestTimesMs, - requestTimesMs.size() >= 5); - for (int i = 2; i < 5; i++) { - final long retryDelayMs = requestTimesMs.get(i) - requestTimesMs.get(i - 1); - assertTrue("Demand hammered a policy-only response at retry " + i - + ": delayMs=" + retryDelayMs + " trace=" + trace, - retryDelayMs >= 1_500); - } - assertTrue("Server-paced retries completed suspiciously fast: elapsedMs=" + elapsedMs - + " trace=" + trace, - elapsedMs >= 5_500); - assertTrue("Server-paced demand recovery took too long: elapsedMs=" + elapsedMs - + " trace=" + trace, - elapsedMs < 9_000); + assertTrue("Rejected attestation incorrectly entered the 59 second backoff: elapsedMs=" + + elapsedMs + " trace=" + trace, + elapsedMs < 2_000); + assertTrue("Rejected attestation triggered another SABR request: " + + harness.downloader.requestTimesSnapshot(), + harness.downloader.requestTimesSnapshot().size() <= 2); + } + } + + @Test + public void pendingAttestationDoesNotReloadOrFail() throws Exception { + try (SabrSmokeHarness harness = SabrSmokeHarness.create()) { + harness.downloader.enqueue(new UmpFixture() + .part(SabrResponseDecoder.STREAM_PROTECTION_STATUS, + streamProtection(2, 20)) + .part(SabrResponseDecoder.NEXT_REQUEST_POLICY, + nextRequestPolicy(2_000)) + .bytes()); + + final YoutubeSabrSession.DemandResponseResult result = + harness.holder.session.pumpOnceStreamingForDemand( + new Localization("en", "US"), + SabrSegmentRequest.media(harness.videoFormat, 1)); + + final String trace = harness.holder.session.getDiagnosticTrace(); + assertTrue("Pending attestation response was not exercised: " + trace, + trace.contains("protection=2/20")); + assertTrue("Pending attestation did not return through normal response handling", + result.wasRequestPerformed()); + assertEquals("Pending attestation unexpectedly returned media", 0, + result.getSegmentCount()); + assertEquals("Pending attestation triggered an implicit retry", 1, + harness.downloader.requestTimesSnapshot().size()); } } @@ -952,13 +1006,13 @@ public void duplicateMediaHeaderFailsThroughDemandPump() throws Exception { } @Test - public void demandProtectedNoMediaHonorsServerBackoff() throws Exception { + public void demandPendingAttestationHonorsServerBackoff() throws Exception { try (SabrSmokeHarness harness = SabrSmokeHarness.create()) { harness.downloader.enqueue(new UmpFixture() .segment(1, SMOKE_VIDEO_ITAG, 1, 0, 30_000) .bytes()); harness.downloader.enqueue(new UmpFixture() - .part(SabrResponseDecoder.STREAM_PROTECTION_STATUS, streamProtection(3, 7)) + .part(SabrResponseDecoder.STREAM_PROTECTION_STATUS, streamProtection(2, 7)) .part(SabrResponseDecoder.NEXT_REQUEST_POLICY, nextRequestPolicy(3_000)) .bytes()); harness.downloader.enqueue(new UmpFixture() @@ -969,16 +1023,16 @@ public void demandProtectedNoMediaHonorsServerBackoff() throws Exception { SabrSegmentRequest.media(harness.videoFormat, 2), 6_000); final String trace = harness.holder.session.getDiagnosticTrace(); - assertTrue("Protected no-media response was not exercised: " + trace, - trace.contains("protection=3/7")); + assertTrue("Pending attestation response was not exercised: " + trace, + trace.contains("protection=2/7")); final List requestTimesMs = harness.downloader.requestTimesSnapshot(); assertTrue("Expected initial, protected, and target requests: " + requestTimesMs, requestTimesMs.size() >= 3); final long retryDelayMs = requestTimesMs.get(2) - requestTimesMs.get(1); - assertTrue("Protected no-media retry ignored backoff: delayMs=" + retryDelayMs + assertTrue("Pending attestation next request ignored backoff: delayMs=" + retryDelayMs + " trace=" + trace, retryDelayMs >= 2_800); - assertTrue("Protected no-media retry did not honor the server backoff: elapsedMs=" + assertTrue("Pending attestation did not honor the server backoff: elapsedMs=" + elapsedMs, elapsedMs < 5_000); } } @@ -1808,6 +1862,102 @@ private static boolean isSabr(final VideoStream stream) { return stream.getDeliveryMethod() == DeliveryMethod.SABR; } + private static boolean isSabr(final AudioStream stream) { + return stream.getDeliveryMethod() == DeliveryMethod.SABR; + } + + private static void runAnonymousAudioWindow(final Context context, + final StreamInfo info, + final int index, + final long playbackMs) throws Exception { + final PlayerDataSource dataSource = new PlayerDataSource(context, + DownloaderImpl.USER_AGENT, new DefaultBandwidthMeter.Builder(context).build()); + final MediaSource mediaSource = new AudioPlaybackResolver(context, dataSource).resolve(info); + assertNotNull("Audio resolver returned no MediaSource for item=" + index + + " video=" + info.getId(), mediaSource); + + final AtomicReference playerRef = new AtomicReference<>(); + final AtomicReference playerError = new AtomicReference<>(); + final CountDownLatch ready = new CountDownLatch(1); + InstrumentationRegistry.getInstrumentation().runOnMainSync(() -> { + final ExoPlayer player = new ExoPlayer.Builder(context) + .setLoadControl(new LoadController()) + .build(); + player.addListener(new Player.Listener() { + @Override + public void onPlaybackStateChanged(final int state) { + if (state == Player.STATE_READY || state == Player.STATE_ENDED) { + ready.countDown(); + } + } + + @Override + public void onPlayerError(final PlaybackException error) { + playerError.compareAndSet(null, error); + ready.countDown(); + } + }); + player.setVolume(0f); + player.setMediaSource(mediaSource); + player.prepare(); + player.play(); + playerRef.set(player); + }); + + SabrSessionStore.Holder holder = null; + try { + assertTrue("Anonymous audio item did not become ready: index=" + index + + " video=" + info.getId(), + ready.await(PREPARE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertNull("Anonymous audio item failed during startup: index=" + index + + " video=" + info.getId(), playerError.get()); + holder = getHolder(info.getId()); + holder.session.setTraceEnabled(true); + final long durationMs = durationOf(playerRef.get()); + assertTrue("Anonymous probe item is too short to cross 60s: index=" + index + + " video=" + info.getId() + " durationMs=" + durationMs, + durationMs > 61_000); + final long targetMs = Math.min(playbackMs, durationMs - 1_000); + waitForPositionWithSabrProgress(playerRef.get(), info.getId(), targetMs, + TimeUnit.MILLISECONDS.toSeconds(targetMs) + PLAYBACK_TIMEOUT_SECONDS, + playerError); + final String trace = holder.session.getDiagnosticTrace(); + assertNull("Anonymous audio item failed during playback: index=" + index + + " video=" + info.getId() + " trace=" + trace, + playerError.get()); + assertTrue("Anonymous audio item did not reach target: index=" + index + + " video=" + info.getId() + " targetMs=" + targetMs + + " positionMs=" + positionOf(playerRef.get()) + " trace=" + trace, + positionOf(playerRef.get()) >= targetMs); + final int maxProtectionStatus = holder.session.getMaxStreamProtectionStatus(); + assertTrue("Anonymous audio item received unexpected protection status: index=" + + index + " video=" + info.getId() + " maxStatus=" + + maxProtectionStatus + " trace=" + trace, + maxProtectionStatus <= 1); + System.out.println("SABR_ANONYMOUS_SEQUENCE index=" + index + + " video=" + info.getId() + + " positionMs=" + positionOf(playerRef.get()) + + " maxProtectionStatus=" + maxProtectionStatus + + " trace=" + trace); + } catch (final Exception | AssertionError failure) { + final String trace = holder == null ? "" + : holder.session.getDiagnosticTrace(); + System.out.println("SABR_ANONYMOUS_SEQUENCE_FAILURE index=" + index + + " video=" + info.getId() + + " positionMs=" + (playerRef.get() == null ? -1 + : positionOf(playerRef.get())) + + " trace=" + trace); + throw failure; + } finally { + InstrumentationRegistry.getInstrumentation().runOnMainSync(() -> { + if (playerRef.get() != null) { + playerRef.get().release(); + } + }); + SabrSessionStore.evict(info.getId()); + } + } + private static SabrSessionStore.Holder getHolder(final String videoId) throws Exception { final SabrSessionStore.Holder holder = findHolder(videoId); assertNotNull("SABR session was not created", holder); @@ -2127,6 +2277,27 @@ private static void waitForPosition(final ExoPlayer player, final long targetMs, assertEquals("Playback position did not reach target", targetMs, positionOf(player)); } + private static void waitForPositionWithSabrProgress(final ExoPlayer player, + final String videoId, + final long targetMs, + final long timeoutSeconds, + final AtomicReference + playerError) throws Exception { + final long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + while (System.nanoTime() < deadlineNs) { + if (playerError.get() != null) { + return; + } + final long positionMs = positionOf(player); + SabrSessionStore.updatePlayerTime(videoId, positionMs); + if (positionMs >= targetMs) { + return; + } + Thread.sleep(250); + } + assertEquals("Playback position did not reach target", targetMs, positionOf(player)); + } + private static void verifyDemandIntegrityRetry(final String expectedIssue, final UmpFixture brokenResponse) throws Exception { diff --git a/app/src/androidTest/java/org/schabi/newpipe/player/YoutubePlaybackBenchmarkTest.java b/app/src/androidTest/java/org/schabi/newpipe/player/YoutubePlaybackBenchmarkTest.java index 11e8a70dd..85e4ecb2a 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/player/YoutubePlaybackBenchmarkTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/player/YoutubePlaybackBenchmarkTest.java @@ -48,7 +48,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.player.datasource.SabrSessionStore; -import org.schabi.newpipe.player.datasource.SabrPolicyRuntime; import org.schabi.newpipe.player.helper.LegacySubtitleRenderersFactory; import org.schabi.newpipe.player.helper.LoadController; import org.schabi.newpipe.player.helper.PlayerDataSource; @@ -146,10 +145,6 @@ public void compareSabrHlsAndGeneratedDash() throws Exception { final List paths = filterPaths(Arrays.asList( new Path("sabr", "mweb", DeliveryMethod.SABR), - new Path("sabr_builtin", "mweb", DeliveryMethod.SABR, - SabrPolicyRuntime.BenchmarkPolicyMode.BUILTIN), - new Path("sabr_cloud", "mweb", DeliveryMethod.SABR, - SabrPolicyRuntime.BenchmarkPolicyMode.CLOUD), new Path("hls", "web_safari", DeliveryMethod.HLS), new Path("tv_downgraded_generated_dash", "tv_downgraded", DeliveryMethod.PROGRESSIVE_HTTP), @@ -224,16 +219,10 @@ public void compareSabrHlsAndGeneratedDash() throws Exception { Collections.rotate(roundOrder, Math.floorMod(round, roundOrder.size())); for (final Path path : roundOrder) { final boolean warmup = round < 0; - final Result result; - try { - result = runTrial(context, path, extractions.get(path).info, - round, warmup, playSeconds, startPositionMs, seekTargetMs, - maxHeight, targetCodec, url, warmWebViewRuntime, - diagnosticDetails, coldSabrCachesEachTrial); - } finally { - SabrPolicyRuntime.setBenchmarkPolicyMode( - SabrPolicyRuntime.BenchmarkPolicyMode.AUTO); - } + final Result result = runTrial(context, path, extractions.get(path).info, + round, warmup, playSeconds, startPositionMs, seekTargetMs, + maxHeight, targetCodec, url, warmWebViewRuntime, + diagnosticDetails, coldSabrCachesEachTrial); emit("PIPEPIPE_BENCHMARK_RESULT", result.toJson()); emitTrialDetails(result); if (!warmup) { @@ -257,7 +246,6 @@ private static Result runTrial(final Context context, final Path path, final Str final boolean diagnosticDetails, final boolean coldSabrCachesEachTrial) throws Exception { NewPipe.setYoutubePlayerClient(path.client); - SabrPolicyRuntime.setBenchmarkPolicyMode(path.policyMode); if (coldSabrCachesEachTrial && path.sourceDelivery == DeliveryMethod.SABR) { SabrSessionStore.clearBenchmarkCaches(context, info.getId()); } else { @@ -1087,16 +1075,10 @@ private static int effectiveHeight(final VideoStream stream) { private static final class Path { private final String name; private final String client; private final DeliveryMethod sourceDelivery; - private final SabrPolicyRuntime.BenchmarkPolicyMode policyMode; private Path(final String name, final String client, final DeliveryMethod sourceDelivery) { - this(name, client, sourceDelivery, SabrPolicyRuntime.BenchmarkPolicyMode.AUTO); - } - private Path(final String name, final String client, final DeliveryMethod sourceDelivery, - final SabrPolicyRuntime.BenchmarkPolicyMode policyMode) { this.name = name; this.client = client; this.sourceDelivery = sourceDelivery; - this.policyMode = policyMode; } } private static final class CachedExtraction { diff --git a/app/src/androidTest/java/org/schabi/newpipe/player/datasource/QuickJsSabrSessionPolicyTest.kt b/app/src/androidTest/java/org/schabi/newpipe/player/datasource/QuickJsSabrSessionPolicyTest.kt deleted file mode 100644 index d5b384290..000000000 --- a/app/src/androidTest/java/org/schabi/newpipe/player/datasource/QuickJsSabrSessionPolicyTest.kt +++ /dev/null @@ -1,185 +0,0 @@ -package org.schabi.newpipe.player.datasource - -import org.junit.Assert.assertArrayEquals -import org.junit.Assert.assertEquals -import org.junit.Assert.assertThrows -import org.junit.Test -import java.util.Collections -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrDecodedResponse -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrScriptPolicy -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSessionPolicy - -class QuickJsSabrSessionPolicyTest { - @Test - fun changesRequestsBackoffAndMediaProtocol() { - val policy = QuickJsSabrSessionPolicy(script(POLICY)) - val state = SabrSessionPolicy.State(1, 0, 0, 0) - - val request = policy.evaluate( - state, - SabrSessionPolicy.RequestEvent(0, 0, 0, 7, byteArrayOf(1, 2, 3)), - ) - assertArrayEquals(byteArrayOf(1, 7), request.requestBody) - - val response = policy.evaluate( - state, - SabrSessionPolicy.ControlResponseEvent( - 0, - true, - SabrSessionPolicy.ControlMode.PUMP, - SabrDecodedResponse(), - ), - ) - assertEquals(SabrSessionPolicy.ActionType.SLEEP_BACKOFF, response.actions[0].type) - assertEquals(4_321, response.controlDecision!!.backoffTimeMs) - assertEquals(2, response.nextState.redirectCount) - assertEquals(null, response.statePatch) - - val header = policy.mediaProtocol.decodeHeader( - byteArrayOf(0x08, 0x04, 0x18, 0xfb.toByte(), 0x01, 0x48, 0x08), - ) - assertEquals(4, header.headerId) - assertEquals(251, header.itag) - assertEquals(8, header.sequenceNumber) - policy.close() - } - - @Test - fun compiledPoliciesKeepSessionStateIndependent() { - val script = script(POLICY) - val first = QuickJsSabrSessionPolicy(script) - val second = QuickJsSabrSessionPolicy(script) - val state = SabrSessionPolicy.State(1, 0, 0, 0) - val event = SabrSessionPolicy.RequestEvent(0, 0, 0, 3, byteArrayOf(1)) - - assertArrayEquals(byteArrayOf(1, 3), first.evaluate(state, event).requestBody) - assertArrayEquals(byteArrayOf(2, 3), first.evaluate(state, event).requestBody) - assertArrayEquals(byteArrayOf(1, 3), second.evaluate(state, event).requestBody) - first.close() - second.close() - } - - @Test - fun supportsModernJavaScriptSyntax() { - val modern = POLICY.replace( - "headerType: 120", - "headerType: ({ value: 120 })?.value", - ) - - QuickJsSabrSessionPolicy(script(modern)).use { - assertEquals(120, it.mediaProtocol.headerPartType) - } - } - - @Test - fun parsesNormalizedResponseStatePatch() { - val source = POLICY.replace( - "actions:['SLEEP_BACKOFF'],backoffMs:4321,", - "actions:['APPLY_RESPONSE_STATE','SLEEP_BACKOFF'],backoffMs:4321," + - "statePatch:{nextRequest:{targetAudioReadaheadMs:9000}," + - "live:[{headSequenceNumber:77}]," + - "formats:[{itag:251,endSegmentNumber:99}]," + - "contexts:[{type:4,scope:1,value:'AQ==',sendByDefault:true," + - "writePolicy:1}],contextPolicy:{start:[4],stop:[],discard:[]}},", - ) - val policy = QuickJsSabrSessionPolicy(script(source)) - val result = policy.evaluate( - SabrSessionPolicy.State(1, 0, 0, 0), - SabrSessionPolicy.ControlResponseEvent( - 0, - true, - SabrSessionPolicy.ControlMode.PUMP, - SabrDecodedResponse(), - ), - ) - - assertEquals(SabrSessionPolicy.ActionType.APPLY_RESPONSE_STATE, result.actions[0].type) - requireNotNull(result.statePatch) - policy.close() - } - - @Test - fun exposesDemandRouteAndReturnedSegmentIdentities() { - val policy = QuickJsSabrSessionPolicy(script(DEMAND_POLICY)) - val state = SabrSessionPolicy.DemandState(1_000, 1_250, 1, 0) - - assertEquals( - SabrSessionPolicy.DemandRoute.RECOVER_MISSING, - policy.evaluateDemandRoute( - SabrSessionPolicy.DemandRouteEvent(251, 7, 30_000, 25_000, state), - ), - ) - val decision = policy.evaluateDemandResponse( - SabrSessionPolicy.DemandResponseEvent( - 251, - 7, - 30_000, - 25_000, - state, - 1, - 0, - Collections.singletonList( - SabrSessionPolicy.DemandReturnedSegment(398, 9, 30_000, 5_000), - ), - false, - ), - ) - - assertEquals(SabrSessionPolicy.DemandOutcome.CONTINUE, decision.outcome) - assertEquals(321, decision.retryDelayMs) - policy.close() - } - - @Test - fun rejectsMissingPolicyFactory() { - assertThrows(SabrProtocolException::class.java) { - QuickJsSabrSessionPolicy(script("var unrelated = true;")) - } - } - - @Test - fun reusesRuntimeAcrossManySessions() { - val script = script(POLICY) - repeat(200) { - QuickJsSabrSessionPolicy(script).close() - } - } - - private fun script(source: String): SabrScriptPolicy { - val now = System.currentTimeMillis() - return SabrScriptPolicy(10_000, now - 1_000, now + 60_000, source) - } - - private companion object { - const val POLICY = - "function createSabrPolicy(sabr){var count=0;return{" + - "describe:function(){return{media:{headerType: 120,mediaType:121,endType:122," + - "headerDecoder:'js'}}}," + - "initialRequest:function(e){return{body:sabr.base64.encode([++count,e.bufferedRangeCount])}}," + - "followUpRequest:function(e){return{body:sabr.base64.encode([++count,e.bufferedRangeCount])}}," + - "response:function(e){return{actions:['SLEEP_BACKOFF'],backoffMs:4321," + - "state:{redirectCount:2}}}," + - "mediaHeader:function(e){var f=sabr.proto.decode(sabr.base64.decode(e.data)),r={};" + - "for(var i=0;ie.recoveryCount" + - "?'RECOVER_MISSING':'STREAM'}}," + - "demandResponse:function(e){if(e.targetItag!==251||" + - "e.targetSequenceNumber!==7||e.elapsedMs!==250||" + - "e.returnedSegments.length!==1||e.returnedSegments[0].itag!==398||" + - "e.returnedSegments[0].sequenceNumber!==9){throw new Error('bad demand event')}" + - "return{outcome:'CONTINUE',retryDelayMs:321}}" + - "}}" - } -} diff --git a/app/src/androidTest/java/org/schabi/newpipe/player/datasource/SabrPolicyRuntimeTest.java b/app/src/androidTest/java/org/schabi/newpipe/player/datasource/SabrPolicyRuntimeTest.java deleted file mode 100644 index 5c5d306e8..000000000 --- a/app/src/androidTest/java/org/schabi/newpipe/player/datasource/SabrPolicyRuntimeTest.java +++ /dev/null @@ -1,175 +0,0 @@ -package org.schabi.newpipe.player.datasource; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import org.junit.Test; -import androidx.test.core.app.ApplicationProvider; -import androidx.test.platform.app.InstrumentationRegistry; - -import org.schabi.newpipe.BuildConfig; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrScriptPolicy; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrScriptPolicyDocument; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSessionPolicyHost; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSessionPolicy; -import org.schabi.newpipe.extractor.services.youtube.sabr.BuiltinSabrSessionPolicy; -import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters; -import org.bouncycastle.crypto.signers.Ed25519Signer; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.security.SecureRandom; -import java.util.Arrays; - -public final class SabrPolicyRuntimeTest { - @Test - public void installEquivalentBuiltinPolicyForBenchmark() throws Exception { - final String privateKeyBase64 = InstrumentationRegistry.getArguments() - .getString("sabrPolicyPrivateKeyBase64"); - assertTrue("Missing benchmark SABR private key", - privateKeyBase64 != null && !privateKeyBase64.isEmpty()); - assertTrue("Benchmark build has no SABR public key", - !BuildConfig.SABR_POLICY_PUBLIC_KEY_BASE64.isEmpty()); - final android.content.Context context = ApplicationProvider.getApplicationContext(); - context.getFileStreamPath("sabr-cloud-policy.bin").delete(); - context.getFileStreamPath("sabr-cloud-policy.rev").delete(); - final InputStream sourceInput = InstrumentationRegistry.getInstrumentation().getContext() - .getAssets().open("equivalent-builtin-sabr-policy.js"); - final byte[] sourceBytes = new byte[sourceInput.available()]; - int offset = 0; - while (offset < sourceBytes.length) { - final int read = sourceInput.read(sourceBytes, offset, sourceBytes.length - offset); - if (read < 0) break; - offset += read; - } - sourceInput.close(); - assertEquals(sourceBytes.length, offset); - final long now = System.currentTimeMillis(); - final SabrScriptPolicy policy = new SabrScriptPolicy(1_000, now - 60_000, - now + 24 * 60 * 60 * 1000L, - new String(sourceBytes, StandardCharsets.UTF_8)); - final byte[] payload = policy.serialize(); - final Ed25519PrivateKeyParameters privateKey = new Ed25519PrivateKeyParameters( - android.util.Base64.decode(privateKeyBase64, android.util.Base64.DEFAULT)); - final Ed25519Signer signer = new Ed25519Signer(); - signer.init(true, privateKey); - signer.update(payload, 0, payload.length); - SabrPolicyRuntime.initialize(context, BuildConfig.SABR_POLICY_PUBLIC_KEY_BASE64, 0); - SabrPolicyRuntime.install(payload, signer.generateSignature(), now); - - final SabrSessionPolicyHost host = SabrPolicyRuntime.createSessionHost(); - assertEquals(20, host.getMediaProtocol().getHeaderPartType()); - host.close(); - } - - @Test - public void envelopeRoundTripsPayloadAndSignature() throws Exception { - final byte[] payload = new byte[]{1, 2, 3, 4}; - final byte[] signature = new byte[]{5, 6, 7}; - - final SabrPolicyRuntime.Envelope decoded = SabrPolicyRuntime.decodeEnvelope( - SabrPolicyRuntime.encodeEnvelope(payload, signature)); - - assertArrayEquals(payload, decoded.payload); - assertArrayEquals(signature, decoded.signature); - } - - @Test - public void envelopeRejectsTrailingAndUnboundedData() { - final byte[] valid = SabrPolicyRuntime.encodeEnvelope( - new byte[]{1}, new byte[]{2}); - assertThrows(IOException.class, () -> SabrPolicyRuntime.decodeEnvelope( - Arrays.copyOf(valid, valid.length + 1))); - assertThrows(IllegalArgumentException.class, () -> SabrPolicyRuntime.encodeEnvelope( - new byte[512 * 1024 + 1], new byte[]{1})); - assertThrows(IllegalArgumentException.class, () -> SabrPolicyRuntime.encodeEnvelope( - new byte[]{1}, new byte[0])); - } - - @Test - public void signedPolicyPersistsAndRestoresOnAndroid() throws Exception { - final android.content.Context context = ApplicationProvider.getApplicationContext(); - context.getFileStreamPath("sabr-cloud-policy.bin").delete(); - context.getFileStreamPath("sabr-cloud-policy.rev").delete(); - final Ed25519PrivateKeyParameters privateKey = - new Ed25519PrivateKeyParameters(new SecureRandom()); - final long now = System.currentTimeMillis(); - final String source = "function createSabrPolicy(sabr){return{" - + "describe:function(){return{media:{headerType:120,mediaType:121,endType:122}}}," - + "initialRequest:function(e){return{body:e.fallbackBody}}," - + "followUpRequest:function(e){return{body:e.fallbackBody}}," - + "response:function(e){return{actions:['CONTINUE']}}," - + "mediaHeader:function(e){return{headerId:1,itag:1}}}}"; - final SabrScriptPolicy policy = new SabrScriptPolicy( - 5, now - 1_000, now + 60_000, source); - final byte[] payload = policy.serialize(); - final Ed25519Signer signer = new Ed25519Signer(); - signer.init(true, privateKey); - signer.update(payload, 0, payload.length); - - final String publicKey = android.util.Base64.encodeToString( - privateKey.generatePublicKey().getEncoded(), android.util.Base64.NO_WRAP); - SabrPolicyRuntime.initialize(context, publicKey, 0); - SabrPolicyRuntime.installDocument(SabrScriptPolicyDocument.encode( - policy, signer.generateSignature()), now); - SabrPolicyRuntime.initialize(context, publicKey, 0); - - assertEquals(120, SabrPolicyRuntime.createSessionHost() - .getMediaProtocol().getHeaderPartType()); - - SabrPolicyRuntime.initialize(context, publicKey, 0); - final SabrScriptPolicy rollback = new SabrScriptPolicy( - 4, now - 1_000, now + 60_000, source); - final byte[] rollbackPayload = rollback.serialize(); - final Ed25519Signer rollbackSigner = new Ed25519Signer(); - rollbackSigner.init(true, privateKey); - rollbackSigner.update(rollbackPayload, 0, rollbackPayload.length); - assertThrows(IllegalArgumentException.class, () -> SabrPolicyRuntime.installDocument( - SabrScriptPolicyDocument.encode(rollback, - rollbackSigner.generateSignature()), now)); - } - - @Test - public void runtimeFailureDisablesCacheAndFallsBack() throws Exception { - final android.content.Context context = ApplicationProvider.getApplicationContext(); - context.getFileStreamPath("sabr-cloud-policy.bin").delete(); - context.getFileStreamPath("sabr-cloud-policy.rev").delete(); - final Ed25519PrivateKeyParameters privateKey = - new Ed25519PrivateKeyParameters(new SecureRandom()); - final long now = System.currentTimeMillis(); - final String source = "function createSabrPolicy(sabr){return{" - + "describe:function(){return{media:{headerType:20,mediaType:21,endType:22," - + "headerDecoder:'builtin'}}}," - + "initialRequest:function(){throw Error('broken')}," - + "followUpRequest:function(){throw Error('broken')}," - + "response:function(){return{actions:['CONTINUE']}}," - + "mediaHeader:function(){return{headerId:1,itag:1}}}}"; - final SabrScriptPolicy policy = new SabrScriptPolicy( - 12, now - 1_000, now + 60_000, source); - final byte[] payload = policy.serialize(); - final Ed25519Signer signer = new Ed25519Signer(); - signer.init(true, privateKey); - signer.update(payload, 0, payload.length); - final String publicKey = android.util.Base64.encodeToString( - privateKey.generatePublicKey().getEncoded(), android.util.Base64.NO_WRAP); - SabrPolicyRuntime.initialize(context, publicKey, 0); - SabrPolicyRuntime.install(payload, signer.generateSignature(), now); - - final SabrSessionPolicyHost host = SabrPolicyRuntime.createSessionHost(); - final byte[] fallback = new byte[]{4, 5, 6}; - assertArrayEquals(fallback, host.evaluate(new SabrSessionPolicy.State(0, 0, 0, 0), - new SabrSessionPolicy.RequestEvent(0, 0, 0, 0, fallback)).getRequestBody()); - assertTrue(!context.getFileStreamPath("sabr-cloud-policy.bin").exists()); - - final SabrSessionPolicyHost next = SabrPolicyRuntime.createSessionHost(); - final Field policyField = SabrSessionPolicyHost.class.getDeclaredField("policy"); - policyField.setAccessible(true); - assertTrue(policyField.get(next) instanceof BuiltinSabrSessionPolicy); - host.close(); - next.close(); - } -} diff --git a/app/src/main/assets/sabr_po_token.js b/app/src/main/assets/sabr_po_token.js index 129945404..6b8624982 100644 --- a/app/src/main/assets/sabr_po_token.js +++ b/app/src/main/assets/sabr_po_token.js @@ -46,13 +46,19 @@ function loadBotGuard(root, challengeData, onReady, onError) { }; }; + var noOp = function () {}; + var loggerFunctions = [noOp, noOp, noOp, noOp, noOp]; + root.syncSnapshotFunction = root.vm.a( root.program, vmFunctionsCallback, true, root.userInteractionElement, - function () {}, - [[], []] + noOp, + [[], []], + undefined, + false, + loggerFunctions )[0]; root._botGuardPolls = 0; diff --git a/app/src/main/assets/sabr_webpo_client.js b/app/src/main/assets/sabr_webpo_client.js deleted file mode 100644 index aa5e0a1d0..000000000 --- a/app/src/main/assets/sabr_webpo_client.js +++ /dev/null @@ -1,69 +0,0 @@ -(function () { - 'use strict'; - - var contentBinding = window.__SABR_WEBPO_CONTENT_BINDING; - - function stage(name, detail) { - SabrPocBridge.onStage(name, String(detail)); - } - - stage('script_start', 'bindingLength=' + (contentBinding ? contentBinding.length : -1)); - console.info('[sabr-webpo] script start bindingLength=' + - (contentBinding ? contentBinding.length : -1)); - - function report(result) { - console.info('[sabr-webpo] report ok=' + !!result.ok); - SabrPocBridge.onResult(JSON.stringify(result)); - } - - function waitForClient(attempt) { - var type = typeof window.top['havuokmhhs-0']?.bevasrs?.wpc; - stage('client_poll', 'attempt=' + attempt + ' type=' + type); - console.info('[sabr-webpo] client attempt=' + attempt + ' type=' + type); - if (type === 'function') { - return Promise.resolve(); - } - if (attempt >= 10) { - return Promise.reject(new Error('WebPoClient unavailable')); - } - return new Promise(function (resolve) { - setTimeout(resolve, 1000); - }).then(function () { - return waitForClient(attempt + 1); - }); - } - - function mint(attempt) { - stage('mint_start', 'attempt=' + attempt); - console.info('[sabr-webpo] mint attempt=' + attempt); - return window.top['havuokmhhs-0'].bevasrs.wpc().then(function (client) { - stage('client_resolved', 'attempt=' + attempt + ' mws=' + typeof client?.mws); - console.info('[sabr-webpo] client resolved mws=' + typeof client?.mws); - return client.mws({c: contentBinding, mc: false, me: false}); - }).catch(function (error) { - stage('mint_error', 'attempt=' + attempt + ' error=' + String(error)); - console.warn('[sabr-webpo] mint error=' + String(error)); - if (String(error).indexOf('SDF:notready') >= 0 && attempt < 10) { - return new Promise(function (resolve) { - setTimeout(resolve, 1000); - }).then(function () { - return mint(attempt + 1); - }); - } - throw error; - }); - } - - waitForClient(0).then(function () { - return mint(0); - }).then(function (poToken) { - stage('mint_success', 'tokenLength=' + (poToken ? poToken.length : -1)); - console.info('[sabr-webpo] mint success tokenLength=' + - (poToken ? poToken.length : -1)); - report({ok: true, poToken: poToken}); - }).catch(function (error) { - stage('pipeline_error', String(error)); - console.error('[sabr-webpo] pipeline error=' + String(error)); - report({ok: false, error: String(error)}); - }); -})(); diff --git a/app/src/main/java/org/schabi/newpipe/App.java b/app/src/main/java/org/schabi/newpipe/App.java index 57871bfdc..318485019 100644 --- a/app/src/main/java/org/schabi/newpipe/App.java +++ b/app/src/main/java/org/schabi/newpipe/App.java @@ -26,10 +26,9 @@ import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.ktx.ExceptionUtils; import org.schabi.newpipe.player.datasource.LocalDomPoTokenProvider; -import org.schabi.newpipe.player.datasource.SabrPolicyRuntime; -import org.schabi.newpipe.player.datasource.SabrPolicyUpdateWorker; import org.schabi.newpipe.settings.NewPipeSettings; import org.schabi.newpipe.util.*; @@ -39,6 +38,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.Callable; import io.reactivex.rxjava3.exceptions.CompositeException; import io.reactivex.rxjava3.exceptions.MissingBackpressureException; @@ -71,6 +71,8 @@ public class App extends MultiDexApplication { public static final String PACKAGE_NAME = BuildConfig.APPLICATION_ID; private static final String TAG = App.class.toString(); + private static final String YOUTUBE_WEB_CLIENT_NAME = "WEB"; + private static final String YOUTUBE_MWEB_CLIENT_NAME = "MWEB"; private static final String YOUTUBE_ANDROID_VR_CLIENT_NAME = "ANDROID_VR"; private static App app; @@ -131,22 +133,7 @@ public void onChanged(Integer connectionState) { Localization.getPreferredContentCountry(this)); final LocalDomPoTokenProvider sessionPoTokenProvider = LocalDomPoTokenProvider.shared(this); - NewPipe.setYoutubeSessionPoTokenProvider((clientName, localization, contentCountry, - loggedIn) -> { - if (!shouldProvideYoutubeSessionPoToken(clientName, - isYoutubeSessionVisitorDataEnabled(this))) { - return null; - } - return sessionPoTokenProvider.getSessionPoToken(clientName, localization, - contentCountry, loggedIn); - }); - try { - SabrPolicyRuntime.initialize(this, - BuildConfig.SABR_POLICY_PUBLIC_KEY_BASE64, 0); - SabrPolicyUpdateWorker.initialize(this); - } catch (final IllegalArgumentException error) { - Log.e(TAG, "Could not initialize SABR cloud policy; using builtin", error); - } + NewPipe.setYoutubeSessionPoTokenProvider(sessionPoTokenProvider); final AndroidWebViewAvailabilityChecker webViewAvailabilityChecker = new AndroidWebViewAvailabilityChecker(this); NewPipe.setWebViewAvailabilityChecker(webViewAvailabilityChecker); @@ -161,7 +148,6 @@ public void onChanged(Integer connectionState) { initNotificationChannels(); ServiceHelper.initServices(this); - prewarmYoutubeSessionPoToken(); // Initialize image loader final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this); @@ -192,6 +178,7 @@ public void onChanged(Integer connectionState) { prefs.edit().putString(youtubePlayerClientKey, youtubePlayerClient).apply(); } NewPipe.setYoutubePlayerClient(youtubePlayerClient); + prewarmYoutubeSessionPoToken(this); PicassoHelper.init(this); PicassoHelper.setShouldLoadImages( prefs.getBoolean(getString(R.string.download_thumbnail_key), true)); @@ -245,24 +232,69 @@ public void onActivityDestroyed(@NonNull final Activity activity) { } public static void prewarmYoutubeSessionPoToken(@NonNull final Context context) { - LocalDomPoTokenProvider.shared(context).prewarmSessionPoToken( - Localization.getPreferredLocalization(context), - Localization.getPreferredContentCountry(context), - ServiceList.YouTube.hasTokens()); - } - - static boolean shouldProvideYoutubeSessionPoToken(@NonNull final String clientName, - final boolean visitorDataEnabled) { - return visitorDataEnabled || YOUTUBE_ANDROID_VR_CLIENT_NAME.equals(clientName); + final LocalDomPoTokenProvider provider = LocalDomPoTokenProvider.shared(context); + provider.cancelSessionPoTokenPrewarm(); + try { + final YoutubePoTokenClientContext client = resolveYoutubePoTokenClientContext( + NewPipe.getYoutubePlayerClient()); + if (client == null) { + return; + } + provider.prewarmSessionPoToken(client.clientName, client.userAgent, + YoutubeParsingHelper.getPlayerRequestLocalization(), + ServiceList.YouTube.getContentCountry(), ServiceList.YouTube.hasTokens(), + client.clientVersionResolver); + } catch (final RuntimeException e) { + Log.w(TAG, "Could not schedule YouTube session PO token prewarm", e); + } } - private static boolean isYoutubeSessionVisitorDataEnabled(@NonNull final Context context) { - return PreferenceManager.getDefaultSharedPreferences(context).getBoolean( - context.getString(R.string.youtube_session_visitor_data_key), false); + private static YoutubePoTokenClientContext resolveYoutubePoTokenClientContext( + @NonNull final String selectedClient) { + switch (selectedClient) { + case "mweb": + return new YoutubePoTokenClientContext(YOUTUBE_MWEB_CLIENT_NAME, + YoutubeParsingHelper::getClientVersion, + YoutubeParsingHelper.MWEB_USER_AGENT); + case "web": + return new YoutubePoTokenClientContext(YOUTUBE_WEB_CLIENT_NAME, + YoutubeParsingHelper::getClientVersion, + YoutubeParsingHelper.WEB_USER_AGENT); + case "web_safari": + return new YoutubePoTokenClientContext(YOUTUBE_WEB_CLIENT_NAME, + () -> "2.20260114.08.00", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/15.5 Safari/605.1.15,gzip(gfe)"); + case "android_vr": + return new YoutubePoTokenClientContext(YOUTUBE_ANDROID_VR_CLIENT_NAME, + () -> "1.65.10", + "com.google.android.apps.youtube.vr.oculus/1.65.10 " + + "(Linux; U; Android 12L; eureka-user " + + "Build/SQ3A.220605.009.A1) gzip"); + case "tv_simply": + return new YoutubePoTokenClientContext("TVHTML5_SIMPLY", () -> "1.0", + YoutubeParsingHelper.WEB_USER_AGENT); + case "tv_downgraded": + return new YoutubePoTokenClientContext("TVHTML5", () -> "5.20260114", + "Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version"); + default: + return null; + } } - private void prewarmYoutubeSessionPoToken() { - prewarmYoutubeSessionPoToken(this); + private static final class YoutubePoTokenClientContext { + @NonNull private final String clientName; + @NonNull private final Callable clientVersionResolver; + @NonNull private final String userAgent; + + private YoutubePoTokenClientContext(@NonNull final String clientName, + @NonNull final Callable clientVersionResolver, + @NonNull final String userAgent) { + this.clientName = clientName; + this.clientVersionResolver = clientVersionResolver; + this.userAgent = userAgent; + } } @Override diff --git a/app/src/main/java/org/schabi/newpipe/SharedWebViewRuntime.java b/app/src/main/java/org/schabi/newpipe/SharedWebViewRuntime.java index c307a8d9c..f5d3eb8f6 100644 --- a/app/src/main/java/org/schabi/newpipe/SharedWebViewRuntime.java +++ b/app/src/main/java/org/schabi/newpipe/SharedWebViewRuntime.java @@ -62,7 +62,7 @@ public interface SabrLocalDomCallbacks { private static final long DEFAULT_TIMEOUT_MS = 30_000L; private static final long READY_CALLBACK_ATTEMPT_TIMEOUT_MS = 5_000L; private static final int MAX_READY_CALLBACK_ATTEMPTS = 2; - private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3"; private static volatile SharedWebViewRuntime instance; diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenGenerator.kt b/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenGenerator.kt index 782f3c7c5..fab892369 100644 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenGenerator.kt +++ b/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenGenerator.kt @@ -13,6 +13,8 @@ import java.util.concurrent.atomic.AtomicReference internal class LocalDomPoTokenGenerator private constructor( context: Context, private val initialization: InitWaiter, + private val attestationContext: LocalDomPoTokenContext, + private val credentialHeaders: Map>, ) : Closeable { private val appContext = context.applicationContext private val runtime = SharedWebViewRuntime.get(appContext) @@ -107,6 +109,7 @@ internal class LocalDomPoTokenGenerator private constructor( url: String, data: String, contentType: String = "application/json+protobuf", + extraHeaders: Map> = emptyMap(), onSuccess: (String) -> Unit, onError: (Throwable) -> Unit, ) { @@ -117,12 +120,12 @@ internal class LocalDomPoTokenGenerator private constructor( val response = downloader.post( url, mapOf( - "User-Agent" to listOf(USER_AGENT), + "User-Agent" to listOf(SharedWebViewRuntime.USER_AGENT), "Accept" to listOf("application/json"), "Content-Type" to listOf(contentType), - "x-goog-api-key" to listOf(GOOGLE_API_KEY), + "x-goog-api-key" to listOf(LOCAL_DOM_GOOGLE_API_KEY), "x-user-agent" to listOf("grpc-web-javascript/0.1"), - ), + ) + extraHeaders, data.toByteArray(), ) if (response.responseCode() != 200) { @@ -149,7 +152,7 @@ internal class LocalDomPoTokenGenerator private constructor( val response = downloader.get( url, mapOf( - "User-Agent" to listOf(USER_AGENT), + "User-Agent" to listOf(SharedWebViewRuntime.USER_AGENT), "Accept" to listOf("*/*"), ), ) @@ -200,26 +203,25 @@ internal class LocalDomPoTokenGenerator private constructor( private fun downloadAndRunBotguard() { makeBotguardServiceRequest( "https://www.youtube.com/youtubei/v1/att/get?prettyPrint=false", - """{"context":{"client":{"clientName":"WEB","clientVersion":"$ATT_CLIENT_VERSION"}},"engagementType":"ENGAGEMENT_TYPE_UNBOUND"}""", + buildLocalDomAttestationBody(attestationContext), contentType = "application/json", + extraHeaders = buildLocalDomAttestationHeaders( + attestationContext, + credentialHeaders, + ), onSuccess = { body -> try { val challenge = parseSabrAttChallengeData(body) - makeBotguardGetRequest( - challenge.interpreterUrl, - onSuccess = { interpreterJavascript -> - val challengeData = buildSabrAttChallengeData( - challenge, - interpreterJavascript, - ) - runtime.evaluateJavascript( - "pipepipeSabrRunBotguard(" + jsString(sessionId) - + ", " + challengeData + ");", - null, - ) { error -> failInitialization(error) } - }, - onError = ::failInitialization, - ) + val inlineInterpreter = challenge.interpreterJavascript + if (inlineInterpreter != null) { + runBotguard(challenge, inlineInterpreter) + } else { + makeBotguardGetRequest( + requireNotNull(challenge.interpreterUrl), + onSuccess = { runBotguard(challenge, it) }, + onError = ::failInitialization, + ) + } } catch (error: Throwable) { failInitialization(error) } @@ -228,6 +230,17 @@ internal class LocalDomPoTokenGenerator private constructor( ) } + private fun runBotguard( + challenge: SabrAttChallengeData, + interpreterJavascript: String, + ) { + runtime.evaluateJavascript( + "pipepipeSabrRunBotguard(" + jsString(sessionId) + ", " + + buildSabrAttChallengeData(challenge, interpreterJavascript) + ");", + null, + ) { error -> failInitialization(error) } + } + private fun onRunBotguardResult(botguardResponse: String) { makeBotguardServiceRequest( "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT", @@ -287,16 +300,21 @@ internal class LocalDomPoTokenGenerator private constructor( private const val ASSET = "sabr_po_token.js" private const val TOKEN_TIMEOUT_MS = 30_000L private const val INIT_TIMEOUT_MS = 60_000L - private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" - private const val ATT_CLIENT_VERSION = "2.20260227.01.00" - private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" @Throws(SabrProtocolException::class) - fun create(context: Context): LocalDomPoTokenGenerator { + fun create( + context: Context, + attestationContext: LocalDomPoTokenContext, + credentialHeaders: Map>, + ): LocalDomPoTokenGenerator { val init = InitWaiter() - val generator = LocalDomPoTokenGenerator(context, init) + val generator = LocalDomPoTokenGenerator( + context, + init, + attestationContext, + credentialHeaders, + ) generator.loadScriptAndInitialize() try { if (!init.latch.await(INIT_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenProvider.kt b/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenProvider.kt index 564c49480..dec746bbf 100644 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenProvider.kt +++ b/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenProvider.kt @@ -3,6 +3,7 @@ package org.schabi.newpipe.player.datasource import android.content.Context import android.os.Handler import android.os.Looper +import android.os.SystemClock import android.util.Log import org.schabi.newpipe.extractor.ServiceList import org.schabi.newpipe.extractor.services.youtube.sabr.SabrPoTokenProvider @@ -19,10 +20,12 @@ import java.nio.charset.StandardCharsets import java.security.MessageDigest import java.util.Base64 import java.util.HashMap +import java.util.concurrent.CancellationException +import java.util.concurrent.Callable import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ExecutionException import java.util.concurrent.Executors -import java.util.concurrent.FutureTask +import java.util.concurrent.Future internal fun youtubeCredentialIdentity(loggedIn: Boolean, tokens: String?): String { val digest = MessageDigest.getInstance("SHA-256") @@ -55,6 +58,7 @@ class LocalDomPoTokenProvider(context: Context) : val mintedAtMs: Long, val visitorData: String, val credentialIdentity: String, + val clientContextIdentity: String, ) private val appContext = context.applicationContext @@ -63,10 +67,9 @@ class LocalDomPoTokenProvider(context: Context) : private val mintLocks = ConcurrentHashMap() private val generatorLock = Any() private val mainHandler = Handler(Looper.getMainLooper()) - private var generatorVisitorData: String? = null + private var generatorContext: LocalDomPoTokenContext? = null private var generatorCredentialIdentity: String? = null private var generator: LocalDomPoTokenGenerator? = null - private var rawSessionPoToken: ByteArray? = null private val visitorDataLock = Any() private var fetchedVisitorData: String? = null private var fetchedVisitorDataLoggedIn: Boolean? = null @@ -75,115 +78,157 @@ class LocalDomPoTokenProvider(context: Context) : private val credentialIdentityTracker = CredentialIdentityTracker( onChanged = ::invalidateCredentialBoundState, ) - private val prewarmLock = Any() private val prewarmExecutor = Executors.newSingleThreadExecutor { runnable -> - Thread(runnable, "YoutubeSessionPoTokenPrewarm") + Thread(runnable, "YoutubeSessionPoTokenPrewarm").apply { isDaemon = true } } - private var prewarmCredentialIdentity: String? = null - private var prewarmTask: FutureTask? = null + private val sessionPoTokenPrewarmer = + ContextBoundSingleFlight< + YoutubeSessionPoTokenPrewarmContext, + PreparedYoutubeSessionPoToken + >( + prewarmExecutor, + ) override fun getSessionPoToken( clientName: String, + clientVersion: String, + userAgent: String?, localization: Localization, contentCountry: ContentCountry, loggedIn: Boolean, ): YoutubeSessionPoToken? { - val credentialIdentity = currentCredentialIdentity(loggedIn) - val inFlightPrewarm = synchronized(prewarmLock) { - prewarmTask?.takeIf { - prewarmCredentialIdentity == credentialIdentity && !it.isDone - } - } - if (inFlightPrewarm != null) { - try { - return inFlightPrewarm.get() - } catch (error: InterruptedException) { - Thread.currentThread().interrupt() - throw SabrProtocolException( - "Interrupted waiting for session PO token prewarm", - error, - ) - } catch (error: ExecutionException) { - throw SabrProtocolException( - "Session PO token prewarm failed", - error.cause ?: error, - ) - } + if (clientName.isBlank() || clientVersion.isBlank() || userAgent.isNullOrBlank()) { + return null } - return getSessionPoTokenNow( + val credentialIdentity = currentCredentialIdentity(loggedIn) + credentialIdentityTracker.observe(credentialIdentity) + val requestContext = YoutubeSessionPoTokenContext( clientName, + clientVersion, + userAgent, localization, contentCountry, loggedIn, credentialIdentity, ) + sessionPoTokenPrewarmer.inFlight(requestContext.prewarmContext())?.let { + val prepared = awaitSessionPoTokenPrewarm(it) + if (prepared.context == requestContext) { + return prepared.token + } + } + return getSessionPoTokenNow(requestContext) } fun prewarmSessionPoToken( + clientName: String, + userAgent: String?, localization: Localization, contentCountry: ContentCountry, loggedIn: Boolean, + clientVersionResolver: Callable, ) { val credentialIdentity = currentCredentialIdentity(loggedIn) - synchronized(prewarmLock) { - val current = prewarmTask - if (prewarmCredentialIdentity == credentialIdentity && current != null && - !current.isDone - ) { - return - } - val task = FutureTask { - val startMs = android.os.SystemClock.elapsedRealtime() - if (currentCredentialIdentity(ServiceList.YouTube.hasTokens()) != - credentialIdentity - ) { - Log.i(TAG, "session token prewarm skipped after credentials changed") - return@FutureTask null - } - try { - getSessionPoTokenNow( - PREWARM_CLIENT_NAME, - localization, - contentCountry, - loggedIn, - credentialIdentity, - ).also { - Log.i( - TAG, - "session token prewarm ready in " + - "${android.os.SystemClock.elapsedRealtime() - startMs}ms", - ) - } - } catch (error: Throwable) { - Log.w(TAG, "session token prewarm failed", error) - throw error + credentialIdentityTracker.observe(credentialIdentity) + val prewarmContext = YoutubeSessionPoTokenPrewarmContext( + clientName, + userAgent, + localization, + contentCountry, + loggedIn, + credentialIdentity, + ) + sessionPoTokenPrewarmer.start(prewarmContext) { + val startedAtMs = SystemClock.elapsedRealtime() + try { + val requestContext = YoutubeSessionPoTokenContext( + clientName, + clientVersionResolver.call(), + userAgent, + localization, + contentCountry, + loggedIn, + credentialIdentity, + ) + PreparedYoutubeSessionPoToken( + requestContext, + getSessionPoTokenNow(requestContext), + ).also { + Log.i( + TAG, + "session token prewarm ready client=$clientName in " + + "${SystemClock.elapsedRealtime() - startedAtMs}ms", + ) } + } catch (error: Throwable) { + Log.w(TAG, "session token prewarm failed client=$clientName", error) + throw error } - prewarmCredentialIdentity = credentialIdentity - prewarmTask = task - prewarmExecutor.execute(task) + } + } + + fun cancelSessionPoTokenPrewarm() { + sessionPoTokenPrewarmer.cancel() + } + + private fun awaitSessionPoTokenPrewarm( + prewarm: Future, + ): PreparedYoutubeSessionPoToken { + try { + return prewarm.get() + } catch (error: InterruptedException) { + Thread.currentThread().interrupt() + throw SabrProtocolException( + "Interrupted waiting for session PO token prewarm", + error, + ) + } catch (error: CancellationException) { + throw SabrProtocolException("Session PO token prewarm was invalidated", error) + } catch (error: ExecutionException) { + throw SabrProtocolException( + "Session PO token prewarm failed", + error.cause ?: error, + ) } } private fun getSessionPoTokenNow( - clientName: String, - localization: Localization, - contentCountry: ContentCountry, - loggedIn: Boolean, - credentialIdentity: String, - ): YoutubeSessionPoToken? { - credentialIdentityTracker.observe(credentialIdentity) + requestContext: YoutubeSessionPoTokenContext, + ): YoutubeSessionPoToken { + if (!credentialsStillMatch(requestContext.credentialIdentity)) { + throw SabrProtocolException( + "YouTube credentials changed before session PO token initialization", + ) + } val visitorData = getOrFetchVisitorData( - localization, - contentCountry, - loggedIn, - credentialIdentity, + requestContext.localization, + requestContext.contentCountry, + requestContext.loggedIn, + requestContext.credentialIdentity, + ) + val playerContext = createPoTokenContext( + visitorData, + requestContext.clientName, + requestContext.clientVersion, + requestContext.userAgent, + ) + val attestationContext = localDomAttestationContext( + visitorData, + YoutubeParsingHelper.getClientVersion(), + ) + val credentialHeaders = createCredentialHeaders(requestContext.loggedIn) + val rawToken = getOrMintToken( + visitorData, + attestationContext, + requestContext.credentialIdentity, + playerContext.cacheIdentity + ':' + attestationContext.cacheIdentity, + credentialHeaders, ) - val rawToken = getRawSessionPoToken(visitorData, credentialIdentity) val encoded = Base64.getUrlEncoder().withoutPadding().encodeToString(rawToken) Log.i( TAG, - "session token ready client=$clientName loggedIn=$loggedIn bytes=${rawToken.size}", + "session token ready client=${requestContext.clientName} " + + "loggedIn=${requestContext.loggedIn} bytes=${rawToken.size}", ) return YoutubeSessionPoToken(visitorData, encoded) } @@ -191,12 +236,6 @@ class LocalDomPoTokenProvider(context: Context) : override fun getPoToken( info: YoutubeSabrInfo, streamState: YoutubeSabrStreamState, - ): ByteArray? = getPoToken(info, streamState, false) - - override fun getPoToken( - info: YoutubeSabrInfo, - streamState: YoutubeSabrStreamState, - forceRefresh: Boolean, ): ByteArray? { val credentialIdentity = currentCredentialIdentity(ServiceList.YouTube.hasTokens()) credentialIdentityTracker.observe(credentialIdentity) @@ -204,27 +243,67 @@ class LocalDomPoTokenProvider(context: Context) : val visitorData = info.visitorData ?: synchronized(visitorDataLock) { fetchedVisitorData } ?: throw SabrProtocolException("Missing visitorData for Local DOM PO token") - if (forceRefresh) { - cache.remove(videoId) - prefs.edit().remove(videoId).apply() - } - synchronized(mintLocks.computeIfAbsent(videoId) { Any() }) { + val playerContext = createPoTokenContext( + visitorData, + info.profile.clientName, + info.clientVersion, + info.profile.userAgent, + ) + val loggedIn = ServiceList.YouTube.hasTokens() + val attestationContext = localDomAttestationContext( + visitorData, + YoutubeParsingHelper.getClientVersion(), + ) + return getOrMintToken( + videoId, + attestationContext, + credentialIdentity, + playerContext.cacheIdentity + ':' + attestationContext.cacheIdentity, + createCredentialHeaders(loggedIn), + ) + } + + private fun getOrMintToken( + contentBinding: String, + context: LocalDomPoTokenContext, + credentialIdentity: String, + clientContextIdentity: String, + credentialHeaders: Map>, + ): ByteArray { + synchronized(mintLocks.computeIfAbsent(contentBinding) { Any() }) { val now = System.currentTimeMillis() - val cached = cache[videoId] - ?: diskLoad(videoId)?.also { cache[videoId] = it } - if (cached != null && cached.visitorData == visitorData && + val cached = cache[contentBinding] + ?: diskLoad(contentBinding)?.also { cache[contentBinding] = it } + if (cached != null && cached.visitorData == context.visitorData && cached.credentialIdentity == credentialIdentity && + cached.clientContextIdentity == clientContextIdentity && now - cached.mintedAtMs < TOKEN_TTL_MS ) { - Log.i(TAG, "cache hit video=$videoId bytes=${cached.token.size}") - return cached.token + Log.i(TAG, "cache hit bindingBytes=${contentBinding.length} " + + "bytes=${cached.token.size}") + return cached.token.clone() + } + val token = synchronized(generatorLock) { + ensureGenerator(context, credentialIdentity, credentialHeaders) + .generateRawPoToken(contentBinding) } - val token = ensureGenerator(visitorData, credentialIdentity) - .generateRawPoToken(videoId) - cache[videoId] = CachedToken(token, now, visitorData, credentialIdentity) - diskSave(videoId, token, now, visitorData, credentialIdentity) - Log.i(TAG, "mint complete video=$videoId bytes=${token.size}") - return token + cache[contentBinding] = CachedToken( + token, + now, + context.visitorData, + credentialIdentity, + clientContextIdentity, + ) + diskSave( + contentBinding, + token, + now, + context.visitorData, + credentialIdentity, + clientContextIdentity, + ) + Log.i(TAG, "mint complete bindingBytes=${contentBinding.length} bytes=${token.size}") + return token.clone() } } @@ -248,15 +327,15 @@ class LocalDomPoTokenProvider(context: Context) : } private fun ensureGenerator( - visitorData: String, + context: LocalDomPoTokenContext, credentialIdentity: String, + credentialHeaders: Map>, ): LocalDomPoTokenGenerator { synchronized(generatorLock) { val current = generator if (current != null && !current.isExpired() && - generatorVisitorData == visitorData && - generatorCredentialIdentity == credentialIdentity && - rawSessionPoToken != null + generatorContext == context && + generatorCredentialIdentity == credentialIdentity ) { return current } @@ -266,8 +345,11 @@ class LocalDomPoTokenProvider(context: Context) : ) } current?.let { mainHandler.post { it.close() } } - val fresh = LocalDomPoTokenGenerator.create(appContext) - val freshSessionPoToken = fresh.generateRawPoToken(visitorData) + val fresh = LocalDomPoTokenGenerator.create( + appContext, + context, + credentialHeaders, + ) if (!credentialsStillMatch(credentialIdentity)) { mainHandler.post { fresh.close() } throw SabrProtocolException( @@ -275,22 +357,32 @@ class LocalDomPoTokenProvider(context: Context) : ) } generator = fresh - generatorVisitorData = visitorData + generatorContext = context generatorCredentialIdentity = credentialIdentity - rawSessionPoToken = freshSessionPoToken return fresh } } - private fun getRawSessionPoToken( + private fun createPoTokenContext( visitorData: String, - credentialIdentity: String, - ): ByteArray { - synchronized(generatorLock) { - ensureGenerator(visitorData, credentialIdentity) - return rawSessionPoToken?.clone() - ?: throw SabrProtocolException("Local DOM session PO token is missing") + clientName: String, + clientVersion: String, + userAgent: String?, + ): LocalDomPoTokenContext { + if (clientName.isBlank() || clientVersion.isBlank() || userAgent.isNullOrBlank()) { + throw SabrProtocolException("Missing YouTube client context for Local DOM PO token") + } + return LocalDomPoTokenContext(visitorData, clientName, clientVersion, userAgent) + } + + private fun createCredentialHeaders(loggedIn: Boolean): Map> { + val headers = HashMap>() + if (loggedIn) { + YoutubeParsingHelper.addLoggedInHeaders(headers) + } else { + YoutubeParsingHelper.addCookieHeader(headers) } + return headers } private fun getOrFetchVisitorData( @@ -346,28 +438,30 @@ class LocalDomPoTokenProvider(context: Context) : } private fun invalidateCredentialBoundState() { - synchronized(visitorDataLock) { - fetchedVisitorData = null - fetchedVisitorDataLoggedIn = null - fetchedVisitorDataCredentialIdentity = null - visitorDataFetchedAtMs = 0 - } - synchronized(generatorLock) { - generator?.let { mainHandler.post { it.close() } } - generator = null - generatorVisitorData = null - generatorCredentialIdentity = null - rawSessionPoToken = null + sessionPoTokenPrewarmer.cancel() + prewarmExecutor.execute { + synchronized(visitorDataLock) { + fetchedVisitorData = null + fetchedVisitorDataLoggedIn = null + fetchedVisitorDataCredentialIdentity = null + visitorDataFetchedAtMs = 0 + } + synchronized(generatorLock) { + generator?.let { mainHandler.post { it.close() } } + generator = null + generatorContext = null + generatorCredentialIdentity = null + } + cache.clear() + prefs.edit().clear().commit() + Log.i(TAG, "YouTube credentials changed; cleared credential-bound PO token state") } - cache.clear() - prefs.edit().clear().commit() - Log.i(TAG, "YouTube credentials changed; cleared credential-bound PO token state") } private fun diskLoad(videoId: String): CachedToken? { val value = prefs.getString(videoId, null) ?: return null - val parts = value.split('|', limit = 4) - if (parts.size != 4) { + val parts = value.split('|', limit = 5) + if (parts.size != 5) { prefs.edit().remove(videoId).apply() return null } @@ -378,14 +472,18 @@ class LocalDomPoTokenProvider(context: Context) : null } else { val visitorData = String( - Base64.getUrlDecoder().decode(parts[2]), + Base64.getUrlDecoder().decode(parts[3]), StandardCharsets.UTF_8, ) CachedToken( - Base64.getUrlDecoder().decode(parts[3]), + Base64.getUrlDecoder().decode(parts[4]), mintedAt, visitorData, parts[1], + String( + Base64.getUrlDecoder().decode(parts[2]), + StandardCharsets.UTF_8, + ), ) } } catch (error: IllegalArgumentException) { @@ -399,15 +497,20 @@ class LocalDomPoTokenProvider(context: Context) : mintedAt: Long, visitorData: String, credentialIdentity: String, + clientContextIdentity: String, ) { val encoder = Base64.getUrlEncoder().withoutPadding() val encodedVisitorData = encoder.encodeToString( visitorData.toByteArray(StandardCharsets.UTF_8), ) val encodedToken = encoder.encodeToString(token) + val encodedContextIdentity = encoder.encodeToString( + clientContextIdentity.toByteArray(StandardCharsets.UTF_8), + ) prefs.edit().putString( videoId, - "$mintedAt|$credentialIdentity|$encodedVisitorData|$encodedToken", + "$mintedAt|$credentialIdentity|$encodedContextIdentity|" + + "$encodedVisitorData|$encodedToken", ).commit() } @@ -416,8 +519,6 @@ class LocalDomPoTokenProvider(context: Context) : private const val PREFS = "sabr_local_dom_video_token_cache" private const val TOKEN_TTL_MS = 6L * 60L * 60L * 1000L private const val VISITOR_DATA_TTL_MS = 6L * 60L * 60L * 1000L - private const val PREWARM_CLIENT_NAME = "WEB" - @Volatile private var sharedInstance: LocalDomPoTokenProvider? = null diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenRequest.kt b/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenRequest.kt new file mode 100644 index 000000000..a4dbcd115 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/player/datasource/LocalDomPoTokenRequest.kt @@ -0,0 +1,93 @@ +package org.schabi.newpipe.player.datasource + +import org.schabi.newpipe.SharedWebViewRuntime +import java.nio.charset.StandardCharsets +import java.security.MessageDigest +import java.util.Base64 + +internal data class LocalDomPoTokenContext( + val visitorData: String, + val clientName: String, + val clientVersion: String, + val userAgent: String, +) { + val clientId: String + get() = when (clientName) { + "WEB" -> "1" + "MWEB" -> "2" + "WEB_EMBEDDED_PLAYER" -> "56" + "ANDROID" -> "3" + "ANDROID_VR" -> "28" + "IOS" -> "5" + "TVHTML5" -> "7" + else -> throw IllegalArgumentException("Unsupported YouTube client: $clientName") + } + + val cacheIdentity: String + get() { + val digest = MessageDigest.getInstance("SHA-256") + listOf(clientName, clientVersion, visitorData, userAgent).forEach { value -> + val bytes = value.toByteArray(StandardCharsets.UTF_8) + digest.update((bytes.size ushr 24).toByte()) + digest.update((bytes.size ushr 16).toByte()) + digest.update((bytes.size ushr 8).toByte()) + digest.update(bytes.size.toByte()) + digest.update(bytes) + } + return Base64.getUrlEncoder().withoutPadding().encodeToString(digest.digest()) + } +} + +internal fun localDomAttestationContext( + visitorData: String, + clientVersion: String, +): LocalDomPoTokenContext { + return LocalDomPoTokenContext( + visitorData, + "WEB", + clientVersion, + SharedWebViewRuntime.USER_AGENT, + ) +} + +internal fun buildLocalDomAttestationBody(context: LocalDomPoTokenContext): String { + return """{"context":{"client":{"clientName":${jsonString(context.clientName)},"clientVersion":${jsonString(context.clientVersion)},"hl":"en","gl":"US","utcOffsetMinutes":0,"visitorData":${jsonString(context.visitorData)}}},"engagementType":"ENGAGEMENT_TYPE_UNBOUND"}""" +} + +internal fun buildLocalDomAttestationHeaders( + context: LocalDomPoTokenContext, + credentialHeaders: Map>, +): Map> { + return HashMap(credentialHeaders).apply { + put("User-Agent", listOf(context.userAgent)) + put("Accept", listOf("application/json")) + put("Content-Type", listOf("application/json")) + put("Origin", listOf("https://www.youtube.com")) + put("Referer", listOf("https://www.youtube.com/")) + put("X-Goog-Visitor-Id", listOf(context.visitorData)) + put("X-YouTube-Client-Name", listOf(context.clientId)) + put("X-YouTube-Client-Version", listOf(context.clientVersion)) + put("x-goog-api-key", listOf(LOCAL_DOM_GOOGLE_API_KEY)) + put("x-user-agent", listOf("grpc-web-javascript/0.1")) + } +} + +internal const val LOCAL_DOM_GOOGLE_API_KEY = + "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" + +private fun jsonString(value: String): String { + return buildString(value.length + 2) { + append('"') + value.forEach { character -> + when (character) { + '\\' -> append("\\\\") + '"' -> append("\\\"") + '\n' -> append("\\n") + '\r' -> append("\\r") + '\t' -> append("\\t") + else -> append(character) + } + } + append('"') + } +} diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/QuickJsSabrRuntime.kt b/app/src/main/java/org/schabi/newpipe/player/datasource/QuickJsSabrRuntime.kt deleted file mode 100644 index 462cde416..000000000 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/QuickJsSabrRuntime.kt +++ /dev/null @@ -1,113 +0,0 @@ -package org.schabi.newpipe.player.datasource - -import com.dokar.quickjs.QuickJs -import com.dokar.quickjs.binding.function -import com.grack.nanojson.JsonObject -import com.grack.nanojson.JsonParser -import com.grack.nanojson.JsonWriter -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.runBlocking -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrScriptPolicy -import java.util.WeakHashMap - -internal object QuickJsSabrRuntime { - private const val MAX_RESULT_CHARS = 512 * 1024 - private const val MEMORY_LIMIT_BYTES = 32L * 1024 * 1024 - private const val MAX_STACK_BYTES = 512L * 1024 - - private val quickJs = QuickJs.create(Dispatchers.Unconfined) - private val compiledPolicies = WeakHashMap() - private var inputJson = "{}" - private var methodName = "describe" - private var activeSessionId = 0 - private var nextSessionId = 1 - private val invokeBytecode: ByteArray - private val closeBytecode: ByteArray - - init { - quickJs.memoryLimit = MEMORY_LIMIT_BYTES - quickJs.maxStackSize = MAX_STACK_BYTES - quickJs.function("__hostInput") { inputJson } - quickJs.function("__hostMethod") { methodName } - quickJs.function("__hostSession") { activeSessionId } - runBlocking { - quickJs.evaluate(RUNTIME_PRELUDE + "var __sabrPolicies=Object.create(null);") - } - invokeBytecode = quickJs.compile( - "(function(){var p=__sabrPolicies[__hostSession()];" + - "if(!p)throw Error('closed SABR policy');var m=__hostMethod(),f=p[m];" + - "if(typeof f!=='function')throw Error('missing method '+m);" + - "return JSON.stringify(f.call(p,JSON.parse(__hostInput())))})()", - ) - closeBytecode = quickJs.compile("delete __sabrPolicies[__hostSession()]") - } - - @Synchronized - fun createSession(script: SabrScriptPolicy): Int { - val sessionId = nextSessionId++ - if (sessionId <= 0) throw SabrProtocolException("SABR QuickJS session limit reached") - activeSessionId = sessionId - try { - val bootstrap = compiledPolicies[script] ?: quickJs.compile( - "(function(id){\n" + script.source + - "\n;if(typeof createSabrPolicy!=='function')" + - "throw Error('missing createSabrPolicy');" + - "var p=createSabrPolicy(sabr);" + - "if(!p)throw Error('createSabrPolicy returned no policy');" + - "__sabrPolicies[id]=p;})(__hostSession())", - ).also { compiledPolicies[script] = it } - runBlocking { quickJs.evaluate(bootstrap) } - return sessionId - } catch (error: Exception) { - throw SabrProtocolException("Could not initialize SABR QuickJS policy", error) - } - } - - @Synchronized - fun invoke(sessionId: Int, method: String, input: JsonObject): JsonObject { - activeSessionId = sessionId - methodName = method - inputJson = JsonWriter.string(input) - return try { - val result = runBlocking { quickJs.evaluate(invokeBytecode) } - if (result.length > MAX_RESULT_CHARS) { - throw SabrProtocolException("SABR QuickJS result exceeded limit") - } - JsonParser.`object`().from(result) - } catch (error: SabrProtocolException) { - throw error - } catch (error: Exception) { - throw SabrProtocolException("SABR QuickJS method failed: $method", error) - } - } - - @Synchronized - fun closeSession(sessionId: Int) { - activeSessionId = sessionId - runBlocking { quickJs.evaluate(closeBytecode) } - quickJs.gc() - } - - private const val RUNTIME_PRELUDE = - "var sabr=(function(){'use strict';" + - "var A='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';" + - "function b64d(s){var o=[],v=0,n=0,i,c;for(i=0;i=8){n-=8;o.push((v>>n)&255)}}return o}" + - "function b64e(a){var o='',i,v;for(i=0;i>18)&63]+A[(v>>12)&63]+(i+1>6)&63]:'=')" + - "+(i+2=a.length)throw Error('truncated varint');" + - "b=a[p.i++];v+=(b&127)*m;m*=128}while(b&128);return v}" + - "function dec(a){var p={i:0},o=[],k,n,w,l,s;while(p.i127){o.push((v%128)|128);v=Math.floor(v/128)}o.push(v)}" + - "function enc(fs){var o=[],i,f,b;for(i=0;i.evaluateDemandRoute(event) - val output = invokeObject("demandRoute", demandInput(event)) - return try { - SabrSessionPolicy.DemandRoute.valueOf( - output.getString("route") - ?: throw SabrProtocolException("QuickJS demand policy returned no route"), - ) - } catch (error: IllegalArgumentException) { - throw SabrProtocolException("QuickJS demand policy returned unknown route", error) - } - } - - @Synchronized - override fun evaluateDemandResponse( - event: SabrSessionPolicy.DemandResponseEvent, - ): SabrSessionPolicy.DemandResponseDecision { - if (!scriptedDemand) return super.evaluateDemandResponse(event) - val input = demandInput(event) - input["segmentCount"] = event.segmentCount - input["targetTrackSegmentCount"] = event.targetTrackSegmentCount - input["returnedSegmentsTruncated"] = event.areReturnedSegmentsTruncated() - val returned = JsonArray() - for (segment in event.returnedSegments) { - returned.add(JsonObject().apply { - this["itag"] = segment.itag - this["sequenceNumber"] = segment.sequenceNumber - this["startMs"] = segment.startMs - this["durationMs"] = segment.durationMs - }) - } - input["returnedSegments"] = returned - val output = invokeObject("demandResponse", input) - val outcome = try { - SabrSessionPolicy.DemandOutcome.valueOf( - output.getString("outcome") - ?: throw SabrProtocolException("QuickJS demand policy returned no outcome"), - ) - } catch (error: IllegalArgumentException) { - throw SabrProtocolException("QuickJS demand policy returned unknown outcome", error) - } - val retryDelayMs = output.getInt("retryDelayMs", 0) - if (retryDelayMs !in 0..SabrSessionPolicy.MAX_DEMAND_RETRY_DELAY_MS || - outcome != SabrSessionPolicy.DemandOutcome.CONTINUE && retryDelayMs != 0 - ) { - throw SabrProtocolException("QuickJS demand policy returned invalid retry delay") - } - return SabrSessionPolicy.DemandResponseDecision( - outcome, - retryDelayMs, - ) - } - - @Synchronized - override fun evaluate( - state: SabrSessionPolicy.State, - event: SabrSessionPolicy.Event, - ): SabrSessionPolicy.Result { - ensureOpen() - if (event is SabrSessionPolicy.RequestEvent) { - val input = stateJson(state) - input["playerTimeMs"] = event.playerTimeMs - input["bufferedEdgeMs"] = event.bufferedEdgeMs - input["poTokenBytes"] = event.poTokenBytes - input["bufferedRangeCount"] = event.bufferedRangeCount - input["fallbackBody"] = Base64.encodeToString(event.proposedBody, Base64.NO_WRAP) - val output = invokeObject( - if (state.requestNumber == 0) "initialRequest" else "followUpRequest", - input, - ) - val body = output.getString("body") - ?: throw SabrProtocolException("QuickJS policy returned no request body") - val bytes = try { - Base64.decode(body, Base64.DEFAULT) - } catch (error: IllegalArgumentException) { - throw SabrProtocolException("Invalid QuickJS request body", error) - } - return SabrSessionPolicy.Result.request( - state, - if (state.requestNumber == 0) { - SabrSessionPolicy.ActionType.SEND_INITIAL_REQUEST - } else { - SabrSessionPolicy.ActionType.SEND_FOLLOW_UP_REQUEST - }, - bytes, - ) - } - return control(state, event as SabrSessionPolicy.ControlResponseEvent) - } - - private fun control( - state: SabrSessionPolicy.State, - event: SabrSessionPolicy.ControlResponseEvent, - ): SabrSessionPolicy.Result { - val input = stateJson(state) - input["segmentCount"] = event.segmentCount - input["honorBackoff"] = event.shouldHonorBackoff() - input["mode"] = event.mode.name - val parts = JsonArray() - for (part in event.response.parts) { - if (part.type == mediaProtocol.mediaPartType) continue - val item = JsonObject() - item["type"] = part.type - item["data"] = Base64.encodeToString(part.data, Base64.NO_WRAP) - parts.add(item) - } - input["parts"] = parts - val builtin = JsonObject() - builtin["error"] = event.response.sabrErrorDetails != null - builtin["reload"] = event.response.isReloadRequested - builtin["protection"] = event.response.isProtectionBoundaryNoMediaResponse - builtin["redirectUrl"] = event.response.redirectUrl - builtin["backoffMs"] = maxOf(0, event.response.backoffTimeMs) - input["builtin"] = builtin - - val output = invokeObject("response", input) - val outputActions = output.getArray("actions") - if (outputActions == null || outputActions.isEmpty()) { - throw SabrProtocolException("QuickJS policy returned no actions") - } - val actions = ArrayList(outputActions.size) - for (index in outputActions.indices) { - try { - actions.add( - SabrSessionPolicy.Action( - SabrSessionPolicy.ActionType.valueOf(outputActions.getString(index)), - ), - ) - } catch (error: RuntimeException) { - throw SabrProtocolException("QuickJS policy returned unknown action", error) - } - } - val next = output.getObject("state") - val nextState = if (next == null) state else SabrSessionPolicy.State( - state.requestNumber, - next.getInt("redirectCount", state.redirectCount), - next.getInt("poTokenRefreshes", state.poTokenRefreshes), - state.reloads, - ) - return SabrSessionPolicy.Result.control( - nextState, - actions, - SabrSessionPolicy.ControlDecision( - output.getInt("backoffMs", 0), - output.getString("redirectUrl"), - output.getString("errorDetails"), - ), - if (output.has("statePatch")) { - parseStatePatch(output.getObject("statePatch"), event) - } else { - null - }, - ) - } - - private fun parseStatePatch( - value: JsonObject?, - event: SabrSessionPolicy.ControlResponseEvent, - ): SabrResponseStatePatch? { - if (value == null) return null - val builder = SabrResponseStatePatch.builder() - val next = value.getObject("nextRequest") - if (next != null) { - builder.setNextRequestPolicy( - SabrNextRequestPolicy.normalized( - next.getInt("targetAudioReadaheadMs", -1), - next.getInt("targetVideoReadaheadMs", -1), - next.getInt("maxTimeSinceLastRequestMs", -1), - next.getInt("backoffTimeMs", -1), - next.getInt("minAudioReadaheadMs", -1), - next.getInt("minVideoReadaheadMs", -1), - decodeOptional(next.getString("playbackCookie")), - next.getString("videoId"), - ), - ) - } - val live = value.getArray("live") - if (live != null) { - for (index in live.indices) { - val item = live.getObject(index) - builder.addLiveMetadata( - SabrLiveMetadata.normalized( - item.getString("broadcastId"), - item.getLong("headSequenceNumber", -1), - item.getLong("headTimeMs", -1), - item.getLong("wallTimeMs", -1), - item.getString("videoId"), - item.getBoolean("postLiveDvr", false), - item.getLong("headm", -1), - item.getLong("minSeekableTimeTicks", -1), - item.getInt("minSeekableTimescale", -1), - item.getLong("maxSeekableTimeTicks", -1), - item.getInt("maxSeekableTimescale", -1), - ), - ) - } - } - val formats = value.getArray("formats") - if (formats != null) { - for (index in formats.indices) { - val item = formats.getObject(index) - builder.addFormatMetadata( - SabrFormatInitializationMetadata.normalized( - item.getString("videoId"), - item.getInt("itag", -1), - item.getLong("lastModified", -1), - item.getString("xtags"), - item.getLong("endTimeMs", -1), - item.getLong("endSegmentNumber", -1), - item.getString("mimeType"), - item.getLong("initRangeStart", -1), - item.getLong("initRangeEnd", -1), - item.getLong("indexRangeStart", -1), - item.getLong("indexRangeEnd", -1), - item.getLong("field8", -1), - item.getLong("durationUnits", -1), - item.getLong("durationTimescale", -1), - ), - ) - } - } - val contexts = value.getArray("contexts") - if (contexts != null) { - for (index in contexts.indices) { - val item = contexts.getObject(index) - builder.addContextUpdate( - SabrContextUpdate.normalized( - item.getInt("type", -1), - item.getInt("scope", -1), - decodeRequired(item.getString("value"), "context value"), - item.getBoolean("sendByDefault", false), - item.getInt("writePolicy", -1), - ), - ) - } - } - val contextPolicy = value.getObject("contextPolicy") - if (contextPolicy != null) { - builder.setContextSendingPolicy( - SabrContextSendingPolicy.normalized( - intList(contextPolicy.getArray("start")), - intList(contextPolicy.getArray("stop")), - intList(contextPolicy.getArray("discard")), - ), - ) - } - for (header in event.response.mediaHeaders) builder.addMediaHeader(header) - return builder.build() - } - - private fun intList(values: JsonArray?): List { - if (values == null) return emptyList() - return values.indices.map { values.getInt(it) } - } - - private fun decodeOptional(value: String?): ByteArray? = - value?.let { decodeRequired(it, "optional bytes") } - - private fun decodeRequired(value: String?, name: String): ByteArray { - if (value == null) throw SabrProtocolException("QuickJS policy returned no $name") - return try { - Base64.decode(value, Base64.DEFAULT) - } catch (error: IllegalArgumentException) { - throw SabrProtocolException("QuickJS policy returned invalid $name", error) - } - } - - @Synchronized - private fun invokeObject(method: String, input: JsonObject): JsonObject { - ensureOpen() - return QuickJsSabrRuntime.invoke(sessionId, method, input) - } - - private fun stateJson(state: SabrSessionPolicy.State) = JsonObject().apply { - this["requestNumber"] = state.requestNumber - this["redirectCount"] = state.redirectCount - this["poTokenRefreshes"] = state.poTokenRefreshes - this["reloads"] = state.reloads - } - - private fun demandInput(event: SabrSessionPolicy.DemandEvent) = JsonObject().apply { - this["targetItag"] = event.targetItag - this["targetSequenceNumber"] = event.targetSequenceNumber - this["targetStartMs"] = event.targetStartMs - this["bufferedEdgeMs"] = event.bufferedEdgeMs - this["createdAtMs"] = event.state.createdAtMs - this["nowMs"] = event.state.nowMs - this["elapsedMs"] = event.state.elapsedMs - this["responsesWithoutDemandedSegment"] = - event.state.responsesWithoutDemandedSegment - this["recoveryCount"] = event.state.recoveryCount - } - - private fun ensureOpen() { - if (closed) throw SabrProtocolException("SABR QuickJS policy is closed") - } - - @Synchronized - override fun close() { - if (!closed) { - closed = true - closeCreatedSession() - } - } - - private fun closeCreatedSession() { - if (sessionId >= 0) { - QuickJsSabrRuntime.closeSession(sessionId) - sessionId = -1 - } - } - - private inner class ScriptMediaProtocol( - private val headerType: Int, - private val mediaType: Int, - private val endType: Int, - private val builtinHeaderDecoder: Boolean, - ) : SabrMediaProtocol { - init { - if (headerType < 0 || mediaType < 0 || endType < 0 || - headerType == mediaType || headerType == endType || mediaType == endType - ) { - throw IllegalArgumentException("Invalid QuickJS media protocol types") - } - } - - override fun getHeaderPartType() = headerType - override fun getMediaPartType() = mediaType - override fun getEndPartType() = endType - - override fun decodeHeader(payload: ByteArray): SabrMediaHeader { - if (builtinHeaderDecoder) return SabrMediaProtocol.builtin().decodeHeader(payload) - val input = JsonObject() - input["data"] = Base64.encodeToString(payload, Base64.NO_WRAP) - val value = invokeObject("mediaHeader", input) - val headerId = value.getInt("headerId", -1) - val itag = value.getInt("itag", -1) - if (headerId !in 0..255 || itag <= 0) { - throw SabrProtocolException("QuickJS policy returned invalid media header identity") - } - return SabrMediaHeader.normalized( - headerId, - value.getString("videoId"), - itag, - value.getLong("lastModified", -1), - value.getString("xtags"), - value.getLong("startRange", -1), - value.getInt("compressionAlgorithm", -1), - value.getBoolean("initSegment", false), - value.getInt("sequenceNumber", -1), - value.getLong("bitrateBps", -1), - value.getLong("startMs", -1), - value.getLong("durationMs", -1), - value.getLong("contentLength", -1), - value.getLong("timeRangeStartTicks", -1), - value.getLong("timeRangeDurationTicks", -1), - value.getInt("timeRangeTimescale", -1), - value.getLong("sequenceLastModified", -1), - ) - } - } - -} diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/SabrLocalDomPoTokenUtil.kt b/app/src/main/java/org/schabi/newpipe/player/datasource/SabrLocalDomPoTokenUtil.kt index 8c027d634..17a6cff50 100644 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/SabrLocalDomPoTokenUtil.kt +++ b/app/src/main/java/org/schabi/newpipe/player/datasource/SabrLocalDomPoTokenUtil.kt @@ -8,21 +8,29 @@ import java.util.Base64 internal data class SabrAttChallengeData( val program: String, val globalName: String, - val interpreterUrl: String, + val interpreterJavascript: String?, + val interpreterUrl: String?, ) internal fun parseSabrAttChallengeData(rawAttestationData: String): SabrAttChallengeData { val challenge = JsonParser.`object`().from(rawAttestationData).getObject("bgChallenge") - val interpreterUrl = challenge.getObject("interpreterUrl") - .getString("privateDoNotAccessOrElseTrustedResourceUrlWrappedValue") + val interpreterJavascript = challenge.getObject("interpreterJavascript") + ?.getString("privateDoNotAccessOrElseSafeScriptWrappedValue") + ?.takeIf { it.isNotEmpty() } + val rawInterpreterUrl = challenge.getObject("interpreterUrl") + ?.getString("privateDoNotAccessOrElseTrustedResourceUrlWrappedValue") + ?.takeIf { it.isNotEmpty() } + val interpreterUrl = rawInterpreterUrl?.let { + if (it.startsWith("//")) "https:$it" else it + } + require(interpreterJavascript != null || interpreterUrl != null) { + "Attestation challenge has no interpreter script or URL" + } return SabrAttChallengeData( program = challenge.getString("program"), globalName = challenge.getString("globalName"), - interpreterUrl = if (interpreterUrl.startsWith("//")) { - "https:$interpreterUrl" - } else { - interpreterUrl - }, + interpreterJavascript = interpreterJavascript, + interpreterUrl = interpreterUrl, ) } @@ -33,10 +41,9 @@ internal fun buildSabrAttChallengeData( return JsonWriter.string( JsonObject.builder() .`object`("interpreterJavascript") - .value("privateDoNotAccessOrElseSafeScriptWrappedValue", interpreterJavascript) .value( - "privateDoNotAccessOrElseTrustedResourceUrlWrappedValue", - challengeData.interpreterUrl, + "privateDoNotAccessOrElseSafeScriptWrappedValue", + interpreterJavascript, ) .end() .value("program", challengeData.program) diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/SabrPolicyRuntime.java b/app/src/main/java/org/schabi/newpipe/player/datasource/SabrPolicyRuntime.java deleted file mode 100644 index ae8884725..000000000 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/SabrPolicyRuntime.java +++ /dev/null @@ -1,377 +0,0 @@ -package org.schabi.newpipe.player.datasource; - -import android.content.Context; -import android.util.AtomicFile; -import android.util.Base64; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import org.schabi.newpipe.BuildConfig; -import org.schabi.newpipe.extractor.services.youtube.sabr.BuiltinSabrSessionPolicy; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaProtocol; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrScriptPolicy; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrScriptPolicyDocument; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrScriptPolicyManager; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSessionPolicy; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSessionPolicyHost; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSessionPolicyTranscript; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.security.PublicKey; - -/** Single construction boundary for the bundled SABR policy set and its per-session transcripts. */ -public final class SabrPolicyRuntime { - public enum BenchmarkPolicyMode { AUTO, BUILTIN, CLOUD } - - private static final int SESSION_TRANSCRIPT_CAPACITY = 512; - private static final int CACHE_MAGIC = 0x53504348; - private static final int CACHE_VERSION = 1; - private static final int MAX_PAYLOAD_BYTES = 512 * 1024; - private static final int MAX_SIGNATURE_BYTES = 1024; - private static final String CACHE_FILE = "sabr-cloud-policy.bin"; - private static final String REVISION_FILE = "sabr-cloud-policy.rev"; - @NonNull private static final SabrSessionPolicy FALLBACK = new BuiltinSabrSessionPolicy(); - @Nullable private static volatile SabrScriptPolicyManager cloudPolicies; - @Nullable private static volatile AtomicFile policyCache; - @Nullable private static volatile AtomicFile revisionCache; - @NonNull private static volatile BenchmarkPolicyMode benchmarkPolicyMode = - BenchmarkPolicyMode.AUTO; - - private SabrPolicyRuntime() { - } - - @NonNull - public static SabrSessionPolicyHost createSessionHost() { - final BenchmarkPolicyMode benchmarkMode = benchmarkPolicyMode; - if (benchmarkMode == BenchmarkPolicyMode.BUILTIN) { - return createHost(FALLBACK); - } - final SabrScriptPolicyManager manager = cloudPolicies; - SabrSessionPolicy policy = FALLBACK; - final SabrScriptPolicy script = manager == null - ? null : manager.current(System.currentTimeMillis()); - if (script != null) { - try { - policy = new FailoverPolicy(script, createScriptPolicy(script)); - } catch (final SabrProtocolException ignored) { - policy = FALLBACK; - } - } - if (benchmarkMode == BenchmarkPolicyMode.CLOUD && policy == FALLBACK) { - throw new IllegalStateException("No active SABR cloud policy for benchmark"); - } - return createHost(policy); - } - - public static void setBenchmarkPolicyMode(@NonNull final BenchmarkPolicyMode mode) { - if (!BuildConfig.DEBUG) { - throw new IllegalStateException("SABR benchmark policy override requires debug build"); - } - benchmarkPolicyMode = mode; - } - - @NonNull - private static SabrSessionPolicyHost createHost(@NonNull final SabrSessionPolicy policy) { - return new SabrSessionPolicyHost(policy, - new SabrSessionPolicyTranscript(SESSION_TRANSCRIPT_CAPACITY)); - } - - /** Configures cloud policy verification and restores the last verified cached envelope. */ - public static synchronized void initialize(@NonNull final Context context, - @NonNull final PublicKey publicKey, - final long minimumRevision) { - final SabrScriptPolicyManager manager = new SabrScriptPolicyManager(publicKey, - Math.max(minimumRevision, readHighestRevision(context))); - initialize(context, manager); - } - - private static synchronized void initialize(@NonNull final Context context, - @NonNull final SabrScriptPolicyManager manager) { - final AtomicFile cache = new AtomicFile(new File( - context.getApplicationContext().getFilesDir(), CACHE_FILE)); - final AtomicFile revisions = new AtomicFile(new File( - context.getApplicationContext().getFilesDir(), REVISION_FILE)); - try { - final Envelope envelope = decodeEnvelope(cache.readFully()); - final SabrScriptPolicy verified = manager.verify( - envelope.payload, envelope.signature, System.currentTimeMillis()); - createScriptPolicy(verified).close(); - manager.activate(verified); - } catch (final IOException | IllegalArgumentException | SabrProtocolException ignored) { - // Missing, expired, or invalid cache must never prevent startup or builtin playback. - } - cloudPolicies = manager; - policyCache = cache; - revisionCache = revisions; - } - - /** Initializes from a deployment-provided raw 32-byte Ed25519 key. Empty means builtin only. */ - public static synchronized void initialize(@NonNull final Context context, - @Nullable final String publicKeyBase64, - final long minimumRevision) { - if (publicKeyBase64 == null || publicKeyBase64.isEmpty()) { - cloudPolicies = null; - policyCache = null; - revisionCache = null; - return; - } - try { - final byte[] key = Base64.decode(publicKeyBase64, Base64.DEFAULT); - final SabrScriptPolicyManager manager = new SabrScriptPolicyManager(key, - Math.max(minimumRevision, readHighestRevision(context))); - initialize(context, manager); - } catch (final IllegalArgumentException error) { - throw new IllegalArgumentException("Invalid SABR cloud policy public key", error); - } - } - - /** Verifies, activates, and atomically persists a downloaded policy envelope. */ - public static synchronized void install(@NonNull final byte[] payload, - @NonNull final byte[] signature, - final long nowMs) throws IOException { - final SabrScriptPolicyManager manager = cloudPolicies; - final AtomicFile cache = policyCache; - final AtomicFile revisions = revisionCache; - if (manager == null || cache == null || revisions == null) { - throw new IllegalStateException("SABR cloud policy runtime is not initialized"); - } - final SabrScriptPolicy verified = manager.verify(payload, signature, nowMs); - try { - createScriptPolicy(verified).close(); - } catch (final SabrProtocolException error) { - throw new IOException("Invalid SABR JavaScript policy", error); - } - writeRevision(revisions, verified.getRevision()); - try { - writeEnvelope(cache, encodeEnvelope(payload, signature)); - } catch (final IOException error) { - cache.delete(); - throw error; - } - manager.activate(verified); - } - - /** Installs a signed, human-readable JSON policy document received from remote delivery. */ - public static void installDocument(@NonNull final byte[] encoded, final long nowMs) - throws IOException { - final SabrScriptPolicyDocument.Parsed document; - try { - document = SabrScriptPolicyDocument.decode(encoded); - } catch (final IllegalArgumentException error) { - throw new IOException("Invalid SABR cloud policy document", error); - } - install(document.getPayload(), document.getSignature(), nowMs); - } - - @NonNull - private static SabrSessionPolicy createScriptPolicy(@NonNull final SabrScriptPolicy script) - throws SabrProtocolException { - return new QuickJsSabrSessionPolicy(script); - } - - private static synchronized void disable(@NonNull final SabrScriptPolicy script) { - final SabrScriptPolicyManager manager = cloudPolicies; - if (manager != null) manager.deactivate(script); - final AtomicFile cache = policyCache; - if (cache != null) cache.delete(); - } - - private static final class FailoverPolicy implements SabrSessionPolicy { - @NonNull private final SabrScriptPolicy script; - @NonNull private final SabrSessionPolicy primary; - @NonNull private final SabrMediaProtocol mediaProtocol; - private boolean failed; - - private FailoverPolicy(@NonNull final SabrScriptPolicy script, - @NonNull final SabrSessionPolicy primary) { - this.script = script; - this.primary = primary; - final SabrMediaProtocol primaryMedia = primary.getMediaProtocol(); - mediaProtocol = new SabrMediaProtocol() { - @Override public int getHeaderPartType() { - return currentMediaProtocol(primaryMedia).getHeaderPartType(); - } - @Override public int getMediaPartType() { - return currentMediaProtocol(primaryMedia).getMediaPartType(); - } - @Override public int getEndPartType() { - return currentMediaProtocol(primaryMedia).getEndPartType(); - } - @NonNull @Override public SabrMediaHeader decodeHeader(@NonNull final byte[] payload) - throws SabrProtocolException { - if (failed) return SabrMediaProtocol.builtin().decodeHeader(payload); - try { - return primaryMedia.decodeHeader(payload); - } catch (final RuntimeException | SabrProtocolException error) { - fail(); - return SabrMediaProtocol.builtin().decodeHeader(payload); - } - } - }; - } - - @NonNull - @Override - public SabrMediaProtocol getMediaProtocol() { - return mediaProtocol; - } - - @NonNull - @Override - public Result evaluate(@NonNull final State state, @NonNull final Event event) - throws SabrProtocolException { - if (failed) return FALLBACK.evaluate(state, event); - try { - return primary.evaluate(state, event); - } catch (final RuntimeException | SabrProtocolException error) { - fail(); - return FALLBACK.evaluate(state, event); - } - } - - @NonNull - @Override - public DemandRoute evaluateDemandRoute(@NonNull final DemandRouteEvent event) - throws SabrProtocolException { - if (failed) return FALLBACK.evaluateDemandRoute(event); - try { - return primary.evaluateDemandRoute(event); - } catch (final RuntimeException | SabrProtocolException error) { - fail(); - return FALLBACK.evaluateDemandRoute(event); - } - } - - @NonNull - @Override - public DemandResponseDecision evaluateDemandResponse( - @NonNull final DemandResponseEvent event) throws SabrProtocolException { - if (failed) return FALLBACK.evaluateDemandResponse(event); - try { - return primary.evaluateDemandResponse(event); - } catch (final RuntimeException | SabrProtocolException error) { - fail(); - return FALLBACK.evaluateDemandResponse(event); - } - } - - @NonNull - private SabrMediaProtocol currentMediaProtocol(@NonNull final SabrMediaProtocol primaryMedia) { - return failed ? SabrMediaProtocol.builtin() : primaryMedia; - } - - private void fail() { - if (!failed) { - failed = true; - disable(script); - try { - primary.close(); - } catch (final RuntimeException ignored) { - } - } - } - - @Override - public void close() { - primary.close(); - } - } - - @NonNull - static byte[] encodeEnvelope(@NonNull final byte[] payload, - @NonNull final byte[] signature) { - validateLengths(payload.length, signature.length); - try { - final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - final DataOutputStream output = new DataOutputStream(bytes); - output.writeInt(CACHE_MAGIC); - output.writeByte(CACHE_VERSION); - output.writeInt(payload.length); - output.writeInt(signature.length); - output.write(payload); - output.write(signature); - output.flush(); - return bytes.toByteArray(); - } catch (final IOException impossible) { - throw new IllegalStateException(impossible); - } - } - - @NonNull - static Envelope decodeEnvelope(@NonNull final byte[] encoded) throws IOException { - final DataInputStream input = new DataInputStream(new ByteArrayInputStream(encoded)); - if (input.readInt() != CACHE_MAGIC || input.readUnsignedByte() != CACHE_VERSION) { - throw new IOException("Unsupported SABR policy cache"); - } - final int payloadLength = input.readInt(); - final int signatureLength = input.readInt(); - validateLengths(payloadLength, signatureLength); - final byte[] payload = new byte[payloadLength]; - final byte[] signature = new byte[signatureLength]; - input.readFully(payload); - input.readFully(signature); - if (input.available() != 0) throw new IOException("Trailing SABR policy cache bytes"); - return new Envelope(payload, signature); - } - - private static void validateLengths(final int payloadLength, final int signatureLength) { - if (payloadLength <= 0 || payloadLength > MAX_PAYLOAD_BYTES - || signatureLength <= 0 || signatureLength > MAX_SIGNATURE_BYTES) { - throw new IllegalArgumentException("Invalid SABR policy cache size"); - } - } - - private static void writeEnvelope(@NonNull final AtomicFile cache, - @NonNull final byte[] encoded) throws IOException { - FileOutputStream output = null; - try { - output = cache.startWrite(); - output.write(encoded); - output.flush(); - cache.finishWrite(output); - } catch (final IOException error) { - if (output != null) cache.failWrite(output); - throw error; - } - } - - private static long readHighestRevision(@NonNull final Context context) { - final AtomicFile file = new AtomicFile(new File( - context.getApplicationContext().getFilesDir(), REVISION_FILE)); - try { - final DataInputStream input = new DataInputStream( - new ByteArrayInputStream(file.readFully())); - final long revision = input.readLong(); - return input.available() == 0 && revision >= 0 ? revision : 0; - } catch (final IOException ignored) { - return 0; - } - } - - private static void writeRevision(@NonNull final AtomicFile file, final long revision) - throws IOException { - final ByteArrayOutputStream bytes = new ByteArrayOutputStream(Long.BYTES); - final DataOutputStream output = new DataOutputStream(bytes); - output.writeLong(revision); - output.flush(); - writeEnvelope(file, bytes.toByteArray()); - } - - static final class Envelope { - @NonNull final byte[] payload; - @NonNull final byte[] signature; - private Envelope(@NonNull final byte[] payload, @NonNull final byte[] signature) { - this.payload = payload; - this.signature = signature; - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/SabrPolicyUpdateWorker.kt b/app/src/main/java/org/schabi/newpipe/player/datasource/SabrPolicyUpdateWorker.kt deleted file mode 100644 index 1c64d1df9..000000000 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/SabrPolicyUpdateWorker.kt +++ /dev/null @@ -1,85 +0,0 @@ -package org.schabi.newpipe.player.datasource - -import android.content.Context -import android.util.Log -import androidx.work.Constraints -import androidx.work.ExistingPeriodicWorkPolicy -import androidx.work.ExistingWorkPolicy -import androidx.work.NetworkType -import androidx.work.OneTimeWorkRequestBuilder -import androidx.work.PeriodicWorkRequestBuilder -import androidx.work.WorkManager -import androidx.work.Worker -import androidx.work.WorkerParameters -import org.schabi.newpipe.BuildConfig -import org.schabi.newpipe.DownloaderImpl -import java.io.IOException -import java.util.concurrent.TimeUnit - -class SabrPolicyUpdateWorker( - context: Context, - params: WorkerParameters, -) : Worker(context, params) { - override fun doWork(): Result { - val url = BuildConfig.SABR_POLICY_URL - if (url.isEmpty() || BuildConfig.SABR_POLICY_PUBLIC_KEY_BASE64.isEmpty()) { - return Result.success() - } - return try { - val response = DownloaderImpl.getInstance().get(url) - when (response.responseCode()) { - 200 -> { - val body = response.rawResponseBody() - ?: throw IOException("SABR policy response had no body") - SabrPolicyRuntime.installDocument(body, System.currentTimeMillis()) - Result.success() - } - 204, 304 -> Result.success() - in 500..599 -> Result.retry() - else -> { - Log.w(TAG, "SABR policy update failed with HTTP ${response.responseCode()}") - Result.failure() - } - } - } catch (error: IOException) { - Log.w(TAG, "Could not update SABR policy", error) - Result.retry() - } catch (error: RuntimeException) { - Log.e(TAG, "Rejected SABR policy update", error) - Result.failure() - } - } - - companion object { - private const val TAG = "SabrPolicyUpdate" - private const val IMMEDIATE_WORK = "sabr-policy-update-now" - private const val PERIODIC_WORK = "sabr-policy-update-periodic" - - @JvmStatic - fun initialize(context: Context) { - if (BuildConfig.SABR_POLICY_URL.isEmpty() || - BuildConfig.SABR_POLICY_PUBLIC_KEY_BASE64.isEmpty() - ) return - val constraints = Constraints.Builder() - .setRequiredNetworkType(NetworkType.CONNECTED) - .build() - val immediate = OneTimeWorkRequestBuilder() - .setConstraints(constraints) - .build() - val periodic = PeriodicWorkRequestBuilder(6, TimeUnit.HOURS) - .setConstraints(constraints) - .build() - val workManager = WorkManager.getInstance(context) - workManager.enqueueUniqueWork( - IMMEDIATE_WORK, - ExistingWorkPolicy.KEEP, - immediate, - ) - workManager.enqueueUniquePeriodicWork( - PERIODIC_WORK, - ExistingPeriodicWorkPolicy.KEEP, - periodic, - ) - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/SabrSessionStore.java b/app/src/main/java/org/schabi/newpipe/player/datasource/SabrSessionStore.java index 0d2a2cbee..d24c71d7f 100644 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/SabrSessionStore.java +++ b/app/src/main/java/org/schabi/newpipe/player/datasource/SabrSessionStore.java @@ -11,7 +11,6 @@ import org.schabi.newpipe.player.PlaybackStartupTrace; import org.schabi.newpipe.player.SabrBackoffCoordinator; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.youtube.sabr.SabrPoTokenProvider; import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment; @@ -673,10 +672,10 @@ public static SabrSourceSpec createSourceSpec(@NonNull final String videoId, PlaybackStartupTrace.markForVideoId(videoId, "sabr_source_spec_started"); final String preferredAudioTrackId = PREFERRED_AUDIO.get(videoId); final Localization localization = new Localization("en", "US"); - final ContentCountry contentCountry = new ContentCountry("US"); - final YoutubeSabrInfo info = isUsableExtractorInfo(extractorInfo, videoId) - ? extractorInfo - : YoutubeSabrProbeFetch(videoId, localization, contentCountry); + if (!isUsableExtractorInfo(extractorInfo, videoId)) { + throw new IOException("SABR extractor info is missing for " + videoId); + } + final YoutubeSabrInfo info = Objects.requireNonNull(extractorInfo); final YoutubeSabrFormat audioFormat = pickAudioFormat(info, preferredAudioTrackId); final YoutubeSabrFormat videoFormat = pickVideoFormat(info, preferredVideoItag); if (audioFormat == null || videoFormat == null) { @@ -768,7 +767,7 @@ private static BootstrapResult createBootstrap(@NonNull final Context context, final File spoolDirectory = new File(context.getApplicationContext().getCacheDir(), "sabr-bootstrap/" + info.getVideoId() + '-' + System.nanoTime()); final YoutubeSabrSession session = new YoutubeSabrSession(info, audioFormat, videoFormat, - sessionProvider, spoolDirectory, SabrPolicyRuntime.createSessionHost()); + sessionProvider, spoolDirectory); session.setBackoffListener(backoffState); boolean handedOff = false; try { @@ -836,7 +835,7 @@ private static BootstrapResult createAdaptiveInitialization( @NonNull final byte[] poToken) throws IOException, ExtractionException { final YoutubeSabrSession session = new YoutubeSabrSession(info, audioFormat, videoFormat, - null, null, SabrPolicyRuntime.createSessionHost()); + null, null); final Future audio = INITIALIZATION_EXECUTOR.submit(() -> session.fetchInitializationData(audioFormat, localization, 2_000, poToken)); final Future video = INITIALIZATION_EXECUTOR.submit(() -> @@ -898,11 +897,18 @@ private static BootstrapResult awaitBootstrap(@NonNull final String key, private static String bootstrapKey(@NonNull final YoutubeSabrInfo info, @NonNull final YoutubeSabrFormat audioFormat, @NonNull final YoutubeSabrFormat videoFormat) { - return info.getVideoId() + '#' + info.getProfile() + '#' + return tokenIdentityKey(info) + '#' + audioFormat.getItag() + ':' + audioFormat.getLastModified() + '#' + videoFormat.getItag() + ':' + videoFormat.getLastModified(); } + @NonNull + private static String tokenIdentityKey(@NonNull final YoutubeSabrInfo info) { + return info.getVideoId() + '#' + info.getProfile() + '#' + info.getClientVersion() + '#' + + Objects.toString(info.getVisitorData(), "") + '#' + + Objects.toString(info.getProfile().getUserAgent(), ""); + } + @NonNull private static BootstrapResult cacheBootstrap(@NonNull final String key, @NonNull final BootstrapResult result) { @@ -914,15 +920,15 @@ private static void startTokenWarmup(@NonNull final Context context, @NonNull final YoutubeSabrInfo info, @NonNull final YoutubeSabrFormat audioFormat, @NonNull final YoutubeSabrFormat videoFormat) { - final String videoId = info.getVideoId(); + final String tokenKey = tokenIdentityKey(info); final FutureTask created = new FutureTask(() -> provider(context).getPoToken( info, new YoutubeSabrStreamState(audioFormat, videoFormat))) { @Override protected void done() { - TOKEN_IN_FLIGHT.remove(videoId, this); + TOKEN_IN_FLIGHT.remove(tokenKey, this); } }; - if (TOKEN_IN_FLIGHT.putIfAbsent(videoId, created) == null) { + if (TOKEN_IN_FLIGHT.putIfAbsent(tokenKey, created) == null) { TOKEN_EXECUTOR.execute(created); } } @@ -952,8 +958,7 @@ static Lease acquire(@NonNull final Context context, @NonNull final SabrSourceSp session.addDiagnosticEvent("bootstrap_session_handoff"); } else { session = new YoutubeSabrSession(spec.getInfo(), spec.getAudioFormat(), - spec.getVideoFormat(), sessionProvider, spoolDirectory, - SabrPolicyRuntime.createSessionHost()); + spec.getVideoFormat(), sessionProvider, spoolDirectory); attachPoToken(spec.getVideoId(), spec.getInfo(), sessionProvider, session); } final Holder holder = new Holder(context, spec, session); @@ -1022,7 +1027,8 @@ private static byte[] awaitWarmedToken(@NonNull final String videoId, @NonNull final org.schabi.newpipe.extractor.services .youtube.sabr.YoutubeSabrStreamState state) throws IOException, ExtractionException { - final Future future = TOKEN_IN_FLIGHT.get(videoId); + final String tokenKey = tokenIdentityKey(info); + final Future future = TOKEN_IN_FLIGHT.get(tokenKey); if (future == null) { PlaybackStartupTrace.markForVideoId(videoId, "sabr_token_mint_started"); final byte[] token = provider.getPoToken(info, state); @@ -1047,7 +1053,7 @@ private static byte[] awaitWarmedToken(@NonNull final String videoId, } throw new IOException("Could not prewarm SABR token for " + videoId, cause); } finally { - TOKEN_IN_FLIGHT.remove(videoId, future); + TOKEN_IN_FLIGHT.remove(tokenKey, future); } } @@ -1060,15 +1066,6 @@ private static boolean isUsableExtractorInfo(@Nullable final YoutubeSabrInfo inf && !info.getFormats().isEmpty(); } - @NonNull - private static YoutubeSabrInfo YoutubeSabrProbeFetch(@NonNull final String videoId, - @NonNull final Localization localization, - @NonNull final ContentCountry contentCountry) - throws IOException, ExtractionException { - return org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe.fetchSabrInfo( - videoId, YoutubeSabrClientProfile.WEB, localization, contentCountry); - } - private static YoutubeSabrFormat pickAudioFormat(@NonNull final YoutubeSabrInfo info, @Nullable final String preferredTrackId) { if (preferredTrackId == null) { @@ -1147,9 +1144,11 @@ public static void clearBenchmarkCaches(@NonNull final Context context, } } } - final Future tokenFuture = TOKEN_IN_FLIGHT.remove(videoId); - if (tokenFuture != null) { - tokenFuture.cancel(true); + for (final Map.Entry> entry : TOKEN_IN_FLIGHT.entrySet()) { + if (entry.getKey().startsWith(videoId + '#')) { + entry.getValue().cancel(true); + TOKEN_IN_FLIGHT.remove(entry.getKey(), entry.getValue()); + } } provider(context).clearCachedToken(videoId); } diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/WebViewPoTokenProvider.java b/app/src/main/java/org/schabi/newpipe/player/datasource/WebViewPoTokenProvider.java deleted file mode 100644 index 7653ab877..000000000 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/WebViewPoTokenProvider.java +++ /dev/null @@ -1,540 +0,0 @@ -package org.schabi.newpipe.player.datasource; - -import android.annotation.SuppressLint; -import android.content.Context; -import android.os.Build; -import android.os.Handler; -import android.os.Looper; -import android.util.Log; -import android.webkit.ConsoleMessage; -import android.webkit.JavascriptInterface; -import android.webkit.WebChromeClient; -import android.webkit.WebResourceError; -import android.webkit.WebResourceRequest; -import android.webkit.WebResourceResponse; -import android.webkit.WebSettings; -import android.webkit.WebView; -import android.webkit.WebViewClient; - -import androidx.annotation.Nullable; - -import org.json.JSONObject; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrPoTokenProvider; -import org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException; -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo; -import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState; - -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Legacy YouTube-page WebPoClient SABR PO-token provider. - * - *

The app no longer calls this provider. Normal playback and download use - * {@link LocalDomPoTokenProvider}, which avoids loading the YouTube page and reuses the shared local - * JavaScript runtime. This class is kept only as a legacy fallback/debug reference for the old - * page-loaded WebPoClient pipeline.

- */ -@Deprecated -public final class WebViewPoTokenProvider implements SabrPoTokenProvider { - - private static final String TAG = "WebViewPoToken"; - private static final String ASSET = "sabr_webpo_client.js"; - private static final String DESKTOP_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " - + "(KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"; - private static final long TOKEN_TTL_MS = 6L * 60L * 60L * 1000L; // 6 hours - // The WebView mint can occasionally run long. 60s + one retry avoids a token-less cold start. - private static final long PIPELINE_TIMEOUT_MS = 60_000L; - // Persist minted tokens across process restarts so an app cold-start can reuse a valid token. - private static final String PREFS = "sabr_webpo_video_token_cache"; - private static final int READY_RETRIES = 20; - private static final long READY_POLL_MS = 250L; - - private static final class CachedToken { - private final byte[] token; - private final long mintedAtMs; - - CachedToken(final byte[] token, final long mintedAtMs) { - this.token = token; - this.mintedAtMs = mintedAtMs; - } - } - - private final Context appContext; - private final Handler mainHandler; - private final android.content.SharedPreferences prefs; - private final Map cache = new ConcurrentHashMap<>(); - // one lock per videoId so two callers (pre-warm + pump) don't both fire the ~45s WebView mint - // for the same video. second one just waits and takes the cached token. - private final Map mintLocks = new ConcurrentHashMap<>(); - public WebViewPoTokenProvider(final Context context) { - this.appContext = context.getApplicationContext(); - this.mainHandler = new Handler(Looper.getMainLooper()); - this.prefs = this.appContext.getSharedPreferences(PREFS, Context.MODE_PRIVATE); - } - - @Nullable - @Override - public byte[] getPoToken(final YoutubeSabrInfo info, final YoutubeSabrStreamState streamState) - throws SabrProtocolException { - return getPoToken(info, streamState, false); - } - - @Nullable - @Override - public byte[] getPoToken(final YoutubeSabrInfo info, final YoutubeSabrStreamState streamState, - final boolean forceRefresh) throws SabrProtocolException { - final String videoId = info.getVideoId(); - Log.i(TAG, "get video=" + videoId + " force=" + forceRefresh - + " thread=" + Thread.currentThread().getName()); - if (forceRefresh) { - // Server rejected the cached token: drop it (memory + disk) and mint fresh. - cache.remove(videoId); - prefs.edit().remove(videoId).apply(); - } - synchronized (mintLocks.computeIfAbsent(videoId, k -> new Object())) { - final long now = System.currentTimeMillis(); - CachedToken cached = cache.get(videoId); - if (cached == null) { - cached = diskLoad(videoId); // survive process restart, skip the ~45s mint - if (cached != null) { - cache.put(videoId, cached); - } - } - if (cached != null && now - cached.mintedAtMs < TOKEN_TTL_MS) { - Log.i(TAG, "cache hit video=" + videoId + " bytes=" + cached.token.length - + " ageMs=" + (now - cached.mintedAtMs)); - return cached.token; - } - if (Thread.currentThread().isInterrupted()) { - Log.w(TAG, "mint skipped: interrupted video=" + videoId); - throw new SabrProtocolException("PO token mint interrupted before start"); - } - // One retry avoids failing playback on a transient WebPoClient error. - final String contentBinding = info.getVideoId(); - Log.i(TAG, "mint start video=" + videoId + " binding=video_id"); - String tokenB64 = mintBlocking(contentBinding); - if (Thread.currentThread().isInterrupted()) { - throw new SabrProtocolException("PO token mint interrupted after pipeline"); - } - if (tokenB64 == null || tokenB64.isEmpty()) { - Log.w(TAG, "PO token mint returned null, retrying once for " + videoId); - tokenB64 = mintBlocking(contentBinding); - if (Thread.currentThread().isInterrupted()) { - throw new SabrProtocolException("PO token mint interrupted after retry"); - } - } - if (tokenB64 == null || tokenB64.isEmpty()) { - Log.e(TAG, "PO token mint failed after retry video=" + videoId); - return null; - } - final byte[] token; - try { - token = Base64.getUrlDecoder().decode(tokenB64); - } catch (final IllegalArgumentException e) { - Log.e(TAG, "could not decode PO token", e); - return null; - } - cache.put(videoId, new CachedToken(token, now)); - diskSave(videoId, tokenB64, now); - Log.i(TAG, "mint complete video=" + videoId + " bytes=" + token.length); - return token; - } - } - - /** - * True if a non-expired PO token for this video is already in memory or on disk, WITHOUT minting. - * Lets a caller pre-load metadata cheaply when we've recently played this video (cold-restore / - * re-resolve) while NOT blocking the first-ever play on the ~45s mint. - */ - public boolean hasCachedToken(final String videoId) { - final CachedToken mem = cache.get(videoId); - if (mem != null && System.currentTimeMillis() - mem.mintedAtMs < TOKEN_TTL_MS) { - return true; - } - return diskLoad(videoId) != null; - } - - @Nullable - private CachedToken diskLoad(final String videoId) { - final String v = prefs.getString(videoId, null); - if (v == null) { - return null; - } - final int sep = v.indexOf('|'); - if (sep <= 0) { - return null; - } - try { - final long mintedAt = Long.parseLong(v.substring(0, sep)); - if (System.currentTimeMillis() - mintedAt >= TOKEN_TTL_MS) { - prefs.edit().remove(videoId).apply(); - return null; - } - return new CachedToken(Base64.getUrlDecoder().decode(v.substring(sep + 1)), mintedAt); - } catch (final IllegalArgumentException e) { - return null; - } - } - - private void diskSave(final String videoId, final String tokenB64, final long mintedAt) { - // commit() (sync) not apply(): the token must hit disk before a fast force-stop/process kill, - // else an app cold-start re-mints (~45s) even though a valid token was just minted. - prefs.edit().putString(videoId, mintedAt + "|" + tokenB64).commit(); - } - - @Nullable - private String mintBlocking(final String contentBinding) throws SabrProtocolException { - final CountDownLatch latch = new CountDownLatch(1); - final AtomicBoolean canceled = new AtomicBoolean(false); - final AtomicReference tokenRef = new AtomicReference<>(); - final AtomicReference webViewRef = new AtomicReference<>(); - final AtomicReference stage = new AtomicReference<>("posting_create"); - final AtomicReference detail = new AtomicReference<>("none"); - final AtomicReference failureRef = new AtomicReference<>(); - final long startedAt = System.currentTimeMillis(); - - mainHandler.post(() -> { - if (canceled.get()) { - Log.w(TAG, "create canceled before main-thread start"); - latch.countDown(); - return; - } - try { - stage.set("creating_webview"); - Log.i(TAG, "creating WebView mainThread=" - + (Looper.myLooper() == Looper.getMainLooper())); - final WebView webView = createWebView(contentBinding, tokenRef, latch, canceled, - stage, detail, failureRef); - if (canceled.get()) { - Log.w(TAG, "create completed after cancellation"); - destroyWebView(webView); - latch.countDown(); - } else { - webViewRef.set(webView); - Log.i(TAG, "WebView created and load requested"); - } - } catch (final Exception e) { - stage.set("create_failed"); - failureRef.set(e); - Log.e(TAG, "failed to start WebView pipeline", e); - latch.countDown(); - } - }); - - try { - if (!latch.await(PIPELINE_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { - Log.e(TAG, "pipeline timeout stage=" + stage.get() - + " elapsedMs=" + (System.currentTimeMillis() - startedAt) - + " webView=" + (webViewRef.get() != null) - + " detail=" + detail.get()); - throw new SabrProtocolException("PO token pipeline timed out at " + stage.get() - + ", detail=" + detail.get()); - } - Log.i(TAG, "pipeline released stage=" + stage.get() - + " elapsedMs=" + (System.currentTimeMillis() - startedAt) - + " token=" + (tokenRef.get() == null ? "null" : "present")); - final Throwable failure = failureRef.get(); - if (failure != null) { - throw new SabrProtocolException("PO token pipeline failed at " + stage.get() - + ", detail=" + detail.get() + ": " + failure.getMessage(), failure); - } - if (tokenRef.get() == null || tokenRef.get().isEmpty()) { - throw new SabrProtocolException("PO token pipeline returned no token at " - + stage.get() + ", detail=" + detail.get()); - } - } catch (final InterruptedException e) { - final String interruptedStage = stage.get(); - Log.w(TAG, "pipeline interrupted stage=" + interruptedStage - + " elapsedMs=" + (System.currentTimeMillis() - startedAt) - + " detail=" + detail.get(), e); - Thread.currentThread().interrupt(); - throw new SabrProtocolException("PO token pipeline interrupted at " - + interruptedStage + ", detail=" + detail.get(), e); - } finally { - canceled.set(true); - mainHandler.post(() -> destroyWebView(webViewRef.getAndSet(null))); - } - return tokenRef.get(); - } - - @SuppressLint("SetJavaScriptEnabled") - private WebView createWebView(final String contentBinding, - final AtomicReference tokenRef, - final CountDownLatch latch, - final AtomicBoolean canceled, - final AtomicReference stage, - final AtomicReference detail, - final AtomicReference failureRef) { - final WebView webView = new WebView(appContext); - final AtomicBoolean injected = new AtomicBoolean(false); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O - && WebView.getCurrentWebViewPackage() != null) { - Log.i(TAG, "WebView package=" + WebView.getCurrentWebViewPackage().packageName - + " version=" + WebView.getCurrentWebViewPackage().versionName); - detail.set("webView=" + WebView.getCurrentWebViewPackage().packageName + '/' - + WebView.getCurrentWebViewPackage().versionName); - } - final WebSettings settings = webView.getSettings(); - settings.setJavaScriptEnabled(true); - settings.setDomStorageEnabled(true); - settings.setUserAgentString(DESKTOP_UA); - webView.addJavascriptInterface(new Bridge(tokenRef, latch, canceled, stage, detail, - failureRef), - "SabrPocBridge"); - webView.setWebChromeClient(new WebChromeClient() { - @Override - public boolean onConsoleMessage(final ConsoleMessage message) { - Log.i(TAG, "console " + message.messageLevel() + " " + message.message() - + " @" + message.sourceId() + ':' + message.lineNumber()); - detail.set("console=" + limit(message.message(), 300) - + " level=" + message.messageLevel() + " line=" + message.lineNumber()); - return true; - } - }); - webView.setWebViewClient(new WebViewClient() { - @Override - public WebResourceResponse shouldInterceptRequest(final WebView view, - final WebResourceRequest request) { - final String url = request.getUrl().toString(); - if (url.contains("/js/th/")) { - Log.i(TAG, "intercept interpreter url=" + url); - return fetchWithCors(url); - } - return super.shouldInterceptRequest(view, request); - } - - @Override - public void onPageFinished(final WebView view, final String url) { - super.onPageFinished(view, url); - Log.i(TAG, "page finished url=" + url + " canceled=" + canceled.get()); - if (canceled.get() || url == null || !url.contains("youtube.com") - || !injected.compareAndSet(false, true)) { - return; - } - stage.set("page_finished"); - waitForReadyThenInject(view, contentBinding, 0, canceled, stage, detail, - failureRef, latch); - } - - @Override - public void onPageCommitVisible(final WebView view, final String url) { - super.onPageCommitVisible(view, url); - Log.i(TAG, "page commit url=" + url + " canceled=" + canceled.get()); - if (canceled.get() || url == null || !url.contains("youtube.com") - || !injected.compareAndSet(false, true)) { - return; - } - stage.set("page_committed"); - waitForReadyThenInject(view, contentBinding, 0, canceled, stage, detail, - failureRef, latch); - } - - @Override - public void onReceivedError(final WebView view, final WebResourceRequest request, - final WebResourceError error) { - super.onReceivedError(view, request, error); - Log.w(TAG, "resource error main=" + request.isForMainFrame() - + " code=" + error.getErrorCode() + " description=" - + error.getDescription() + " url=" + request.getUrl()); - if (request.isForMainFrame() && failureRef.compareAndSet(null, - new IllegalStateException("WebView main page error " - + error.getErrorCode() + ": " + error.getDescription()))) { - stage.set("main_page_failed"); - latch.countDown(); - } - } - }); - stage.set("loading_page"); - Log.i(TAG, "load page"); - webView.loadUrl("https://www.youtube.com?themeRefresh=1"); - return webView; - } - - private void waitForReadyThenInject(final WebView view, final String contentBinding, - final int attempt, - final AtomicBoolean canceled, - final AtomicReference stage, - final AtomicReference detail, - final AtomicReference failureRef, - final CountDownLatch latch) { - if (canceled.get()) { - Log.w(TAG, "ready poll canceled attempt=" + attempt); - return; - } - stage.set("ready_poll_" + attempt); - view.evaluateJavascript("document.readyState", value -> { - if (canceled.get()) { - Log.w(TAG, "ready result canceled attempt=" + attempt); - return; - } - final boolean complete = value != null && value.contains("complete"); - Log.i(TAG, "ready attempt=" + attempt + " value=" + value - + " complete=" + complete); - detail.set("readyState=" + value + " attempt=" + attempt); - if (complete || attempt >= READY_RETRIES) { - stage.set("injecting_binding"); - view.evaluateJavascript( - "window.__SABR_WEBPO_CONTENT_BINDING=" + jsString(contentBinding) + ";", - result -> Log.i(TAG, "binding injected result=" + result)); - stage.set("injecting_script"); - final String script = loadPipelineScript(); - if (script.isEmpty()) { - failureRef.compareAndSet(null, - new IllegalStateException("WebPo pipeline asset is empty")); - stage.set("script_load_failed"); - latch.countDown(); - return; - } - view.evaluateJavascript(script, result -> { - stage.set("waiting_bridge"); - Log.i(TAG, "script injected result=" + result); - }); - } else { - mainHandler.postDelayed( - () -> waitForReadyThenInject(view, contentBinding, attempt + 1, canceled, - stage, detail, failureRef, latch), - READY_POLL_MS); - } - }); - } - - private static void destroyWebView(@Nullable final WebView webView) { - if (webView == null) { - return; - } - try { - webView.stopLoading(); - webView.loadUrl("about:blank"); - webView.removeAllViews(); - webView.destroy(); - } catch (final Exception ignored) { - // best effort - } - } - - private static String jsString(final String value) { - return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; - } - - private static String limit(@Nullable final String value, final int maxLength) { - if (value == null || value.length() <= maxLength) { - return value; - } - return value.substring(0, maxLength); - } - - private String loadPipelineScript() { - try (InputStream in = appContext.getAssets().open(ASSET); - ByteArrayOutputStream out = new ByteArrayOutputStream()) { - final byte[] chunk = new byte[8192]; - int read; - while ((read = in.read(chunk)) != -1) { - out.write(chunk, 0, read); - } - return out.toString("UTF-8"); - } catch (final Exception e) { - Log.e(TAG, "could not read pipeline asset", e); - return ""; - } - } - - @Nullable - private static WebResourceResponse fetchWithCors(final String url) { - try { - final HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); - connection.setRequestProperty("User-Agent", DESKTOP_UA); - connection.setConnectTimeout(15000); - connection.setReadTimeout(15000); - final int code = connection.getResponseCode(); - final InputStream body = code >= 400 - ? connection.getErrorStream() : connection.getInputStream(); - final String contentType = connection.getContentType(); - String mime = "application/javascript"; - if (contentType != null) { - final int sep = contentType.indexOf(';'); - mime = sep > 0 ? contentType.substring(0, sep).trim() : contentType.trim(); - } - final Map headers = new HashMap<>(); - headers.put("Access-Control-Allow-Origin", "*"); - final WebResourceResponse response = new WebResourceResponse(mime, "UTF-8", body); - response.setStatusCodeAndReasonPhrase(code, code >= 400 ? "ERROR" : "OK"); - response.setResponseHeaders(headers); - return response; - } catch (final Exception e) { - Log.e(TAG, "interpreter native fetch failed", e); - return null; - } - } - - private static final class Bridge { - private final AtomicReference tokenRef; - private final CountDownLatch latch; - private final AtomicBoolean canceled; - private final AtomicReference stage; - private final AtomicReference detail; - private final AtomicReference failureRef; - - Bridge(final AtomicReference tokenRef, final CountDownLatch latch, - final AtomicBoolean canceled, final AtomicReference stage, - final AtomicReference detail, - final AtomicReference failureRef) { - this.tokenRef = tokenRef; - this.latch = latch; - this.canceled = canceled; - this.stage = stage; - this.detail = detail; - this.failureRef = failureRef; - } - - @JavascriptInterface - public void onStage(final String nextStage, final String nextDetail) { - stage.set("js_" + limit(nextStage, 80)); - detail.set(limit(nextDetail, 500)); - Log.i(TAG, "JS stage=" + stage.get() + " detail=" + detail.get()); - } - - @JavascriptInterface - public void onResult(final String json) { - stage.set("bridge_called"); - Log.i(TAG, "bridge called canceled=" + canceled.get() - + " jsonLength=" + (json == null ? -1 : json.length())); - try { - if (canceled.get()) { - Log.w(TAG, "bridge result ignored after cancellation"); - return; - } - final JSONObject obj = new JSONObject(json); - if (obj.optBoolean("ok", false)) { - final String token = obj.optString("poToken", null); - tokenRef.set(token); - stage.set("bridge_success"); - Log.i(TAG, "bridge success tokenB64Length=" - + (token == null ? -1 : token.length())); - } else { - stage.set("bridge_failed"); - final String error = obj.optString("error", "unknown"); - failureRef.compareAndSet(null, new IllegalStateException(error)); - Log.w(TAG, "PO token pipeline failed: " + error); - } - } catch (final Exception e) { - stage.set("bridge_parse_failed"); - failureRef.compareAndSet(null, e); - Log.e(TAG, "could not parse pipeline result", e); - } finally { - latch.countDown(); - } - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeSessionPoTokenPrewarmer.kt b/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeSessionPoTokenPrewarmer.kt new file mode 100644 index 000000000..a26418ecd --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeSessionPoTokenPrewarmer.kt @@ -0,0 +1,92 @@ +package org.schabi.newpipe.player.datasource + +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import java.util.concurrent.Executor +import java.util.concurrent.Future +import java.util.concurrent.FutureTask + +internal data class YoutubeSessionPoTokenContext( + val clientName: String, + val clientVersion: String, + val userAgent: String?, + val localization: Localization, + val contentCountry: ContentCountry, + val loggedIn: Boolean, + val credentialIdentity: String, +) + +internal data class YoutubeSessionPoTokenPrewarmContext( + val clientName: String, + val userAgent: String?, + val localization: Localization, + val contentCountry: ContentCountry, + val loggedIn: Boolean, + val credentialIdentity: String, +) + +internal data class PreparedYoutubeSessionPoToken( + val context: YoutubeSessionPoTokenContext, + val token: YoutubeSessionPoToken, +) + +internal fun YoutubeSessionPoTokenContext.prewarmContext() = + YoutubeSessionPoTokenPrewarmContext( + clientName, + userAgent, + localization, + contentCountry, + loggedIn, + credentialIdentity, + ) + +internal class ContextBoundSingleFlight(private val executor: Executor) { + private data class Entry(val key: K, val task: FutureTask) + + private val lock = Any() + private var current: Entry? = null + + fun start(key: K, operation: () -> V): Boolean { + val created = object : FutureTask(operation) { + override fun done() { + synchronized(lock) { + if (current?.task === this) { + current = null + } + } + } + } + val replaced = synchronized(lock) { + val existing = current + if (existing != null && existing.key == key && !existing.task.isDone) { + return false + } + current = Entry(key, created) + existing?.task + } + replaced?.cancel(true) + try { + executor.execute(created) + } catch (error: RuntimeException) { + synchronized(lock) { + if (current?.task === created) { + current = null + } + } + throw error + } + return true + } + + fun inFlight(key: K): Future? = synchronized(lock) { + current?.takeIf { it.key == key && !it.task.isCancelled }?.task + } + + fun cancel() { + val task = synchronized(lock) { + current?.task.also { current = null } + } + task?.cancel(true) + } +} diff --git a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt index 54cf79f38..9b750a9d6 100644 --- a/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt +++ b/app/src/main/java/us/shandian/giga/get/SabrDownloader.kt @@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession import org.schabi.newpipe.player.datasource.LocalDomPoTokenProvider -import org.schabi.newpipe.player.datasource.SabrPolicyRuntime import java.io.File import java.io.FileOutputStream import java.io.IOException @@ -97,7 +96,6 @@ internal class SabrDownloader( SabrDownloadFormatResolver.selectedVideoFormat(info, recoveries), LocalDomPoTokenProvider(mission.context), null, - SabrPolicyRuntime.createSessionHost(), ) val workDir = prepareWorkDirectory() val targets = SabrDownloadFormatResolver.buildTargets(info, recoveries, workDir) diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index dfe5a3fab..7a0ee2e5d 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -982,8 +982,7 @@ YouTube-Wiedergabe wartet Benachrichtigungen, wenn YouTube verlangt, dass eine Wiedergabeanfrage wartet Warten auf YouTube -Besucherdaten an YouTube senden. Hilft dabei, vorübergehende IP-Sperren zu vermeiden, aber Sie müssen möglicherweise häufiger auf Werbung warten. YouTube zwingt uns, %1$d Sekunden zu warten, bevor die Wiedergabe fortgesetzt wird Wischen zur Steuerung der Wiedergabegeschwindigkeit In der Mitte wischen, um die Wiedergabegeschwindigkeit zu steuern. Deaktiviert die Vollbild-Gestensteuerung. - \ No newline at end of file + diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 2eb9235d0..9a8b066d2 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -1038,8 +1038,7 @@ Reproducción de YouTube en espera Notificaciones cuando YouTube requiere que una solicitud de reproducción espere Esperando a YouTube -Enviar datos de visitantes a YouTube. Ayuda a evitar bloqueos temporales de IP, pero es posible que te veas obligado a esperar por los anuncios con mayor frecuencia. YouTube nos obliga a esperar %1$d segundos antes de continuar la reproducción Desliza para controlar la velocidad de reproducción Desliza en el centro para controlar la velocidad de reproducción. Desactiva el control por gestos en pantalla completa. - \ No newline at end of file + diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index f4565e95a..ff450206e 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -999,8 +999,7 @@ Voulez-vous continuer? Lecture YouTube en attente Notifications lorsque YouTube exige qu\'une demande de lecture soit mise en attente En attente de YouTube -Envoyer les données des visiteurs à YouTube. Aide à éviter les blocages temporaires d\'IP, mais vous pourriez être forcé d\'attendre les annonces plus fréquemment. YouTube nous oblige à attendre %1$d secondes avant de reprendre la lecture Balayez pour contrôler la vitesse de lecture Glissez au milieu pour contrôler la vitesse de lecture. Désactive le contrôle gestuel en plein écran. - \ No newline at end of file + diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index f5411cb89..8ea2f4b8a 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -976,8 +976,7 @@ Riproduzione YouTube in attesa Notifiche quando YouTube richiede l\'attesa per una richiesta di riproduzione In attesa di YouTube -Invia i dati dei visitatori a YouTube. Aiuta a evitare blocchi temporanei dell\'IP, ma potresti dover attendere gli annunci più frequentemente. YouTube ci costringe ad aspettare %1$d secondi prima di continuare la riproduzione Scorri per controllare la velocità di riproduzione Scorri al centro per controllare la velocità di riproduzione. Disabilita il controllo dei gesti a schermo intero - \ No newline at end of file + diff --git a/app/src/main/res/values-ja/strings.xml b/app/src/main/res/values-ja/strings.xml index a6f7f4330..22e09dbfd 100644 --- a/app/src/main/res/values-ja/strings.xml +++ b/app/src/main/res/values-ja/strings.xml @@ -1005,8 +1005,7 @@ YouTube再生待機 YouTubeで再生リクエストの待機が必要な場合の通知 YouTubeの待機中 -YouTubeに訪問者データを送信します。一時的なIPブロックの回避に役立ちますが、より頻繁に広告を待たされる可能性があります。 YouTubeの制限により、再生を続行する前に %1$d 秒待機する必要があります スワイプで再生速度を調整 中央をスワイプして再生速度を調整します。フルスクリーンのジェスチャー操作は無効になります。 - \ No newline at end of file + diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index 38d5e1196..b1c9a5142 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -1062,8 +1062,7 @@ Oczekiwanie na odtwarzanie YouTube Powiadomienia, gdy YouTube wymaga oczekiwania na żądanie odtwarzania Oczekiwanie na YouTube -Wyślij dane odwiedzającego do YouTube. Pomaga uniknąć tymczasowych blokad IP, ale możesz być zmuszony do częstszego oczekiwania na reklamy. YouTube wymusza na nas odczekanie %1$d sekund przed wznowieniem odtwarzania Przesuń, aby sterować prędkością odtwarzania Przesuń pośrodku, aby sterować prędkością odtwarzania. Wyłącza sterowanie gestami na pełnym ekranie - \ No newline at end of file + diff --git a/app/src/main/res/values-vi/strings.xml b/app/src/main/res/values-vi/strings.xml index c64f6b847..e1596d8f8 100644 --- a/app/src/main/res/values-vi/strings.xml +++ b/app/src/main/res/values-vi/strings.xml @@ -1091,8 +1091,7 @@ nSự khác biệt giữa hai loại này là cái nào nhanh thường thiếu Đang chờ phát lại YouTube Thông báo khi YouTube yêu cầu chờ phát lại Đang chờ YouTube -Gửi dữ liệu người truy cập đến YouTube. Giúp tránh việc bị chặn IP tạm thời, nhưng bạn có thể sẽ phải chờ quảng cáo thường xuyên hơn. YouTube buộc chúng ta phải đợi %1$d giây trước khi tiếp tục phát Vuốt để điều khiển tốc độ phát lại Vuốt ở giữa để điều khiển tốc độ phát. Vô hiệu hóa điều khiển bằng cử chỉ toàn màn hình - \ No newline at end of file + diff --git a/app/src/main/res/values-zh-rCN/strings.xml b/app/src/main/res/values-zh-rCN/strings.xml index 045695004..a4dd66e81 100644 --- a/app/src/main/res/values-zh-rCN/strings.xml +++ b/app/src/main/res/values-zh-rCN/strings.xml @@ -1046,8 +1046,7 @@ YouTube 播放等待 YouTube 强制播放请求等待时显示通知 正在等待 YouTube -向 YouTube 发送访客数据。有助于避免临时的 IP 封禁，但您可能会被迫更频繁地等待广告。 YouTube 强制我们等待 %1$d 秒后才能继续播放 滑动控制播放速度 在中间滑动以控制播放速度。将禁用全屏手势控制 - \ No newline at end of file + diff --git a/app/src/main/res/values-zh-rTW/strings.xml b/app/src/main/res/values-zh-rTW/strings.xml index 77b65e053..cb3efe046 100644 --- a/app/src/main/res/values-zh-rTW/strings.xml +++ b/app/src/main/res/values-zh-rTW/strings.xml @@ -1036,8 +1036,7 @@ YouTube 播放等待 YouTube 強制播放請求等待時顯示通知 正在等待 YouTube -將訪客資料傳送至 YouTube。這有助於避免暫時性的 IP 封鎖，但您可能需要更頻繁地等待廣告。 YouTube 強制我們等待 %1$d 秒後才能繼續播放 滑動以控制播放速度 在中間滑動以控制播放速度。會停用全螢幕手勢控制。 - \ No newline at end of file + diff --git a/app/src/main/res/values/settings_keys.xml b/app/src/main/res/values/settings_keys.xml index 98dd90281..ec9a0db4c 100644 --- a/app/src/main/res/values/settings_keys.xml +++ b/app/src/main/res/values/settings_keys.xml @@ -1784,6 +1784,5 @@ android_vr tv_downgraded - youtube_session_visitor_data sabr_backoff diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index f21a218dc..cbbacbb36 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1084,6 +1084,5 @@ YouTube playback waiting Notifications when YouTube requires a playback request to wait Waiting for YouTube -Send visitor data to YouTube. Help avoid temporary IP blocks, but you may be forced to wait for ads more frequently YouTube forces us to wait %1$d seconds before continuing playback - \ No newline at end of file + diff --git a/app/src/main/res/xml/advanced_settings.xml b/app/src/main/res/xml/advanced_settings.xml index 604b3f30f..5ac029a6d 100644 --- a/app/src/main/res/xml/advanced_settings.xml +++ b/app/src/main/res/xml/advanced_settings.xml @@ -75,14 +75,6 @@ app:singleLineTitle="false" app:iconSpaceReserved="false" /> - - (executor) + val started = CountDownLatch(1) + val release = CountDownLatch(1) + val calls = AtomicInteger() + try { + assertTrue(prewarmer.start("context") { + calls.incrementAndGet() + started.countDown() + release.await() + "token" + }) + assertTrue(started.await(2, TimeUnit.SECONDS)) + val shared = prewarmer.inFlight("context") + + assertFalse(prewarmer.start("context") { + calls.incrementAndGet() + "duplicate" + }) + release.countDown() + + assertEquals("token", shared?.get(2, TimeUnit.SECONDS)) + assertEquals(1, calls.get()) + } finally { + release.countDown() + executor.shutdownNow() + } + } + + @Test(timeout = 5_000) + fun replacingContextCancelsOldTask() { + val executor = Executors.newSingleThreadExecutor() + val prewarmer = ContextBoundSingleFlight(executor) + val started = CountDownLatch(1) + val replacementStarted = CountDownLatch(1) + val replacementRelease = CountDownLatch(1) + try { + assertTrue(prewarmer.start("old") { + started.countDown() + CountDownLatch(1).await() + "old-token" + }) + assertTrue(started.await(2, TimeUnit.SECONDS)) + val old = prewarmer.inFlight("old") + + assertTrue(prewarmer.start("new") { + replacementStarted.countDown() + replacementRelease.await() + "new-token" + }) + assertTrue(replacementStarted.await(2, TimeUnit.SECONDS)) + val replacement = prewarmer.inFlight("new") + + assertTrue(old?.isCancelled == true) + replacementRelease.countDown() + assertEquals("new-token", replacement?.get(2, TimeUnit.SECONDS)) + assertNull(prewarmer.inFlight("old")) + } finally { + replacementRelease.countDown() + executor.shutdownNow() + } + } + + @Test(timeout = 5_000) + fun foregroundSharesTaskWhileClientVersionResolutionIsBlocked() { + val prewarmExecutor = Executors.newSingleThreadExecutor() + val foregroundExecutor = Executors.newSingleThreadExecutor() + val prewarmer = ContextBoundSingleFlight< + YoutubeSessionPoTokenPrewarmContext, + PreparedYoutubeSessionPoToken + >(prewarmExecutor) + val requestContext = YoutubeSessionPoTokenContext( + "MWEB", + "2.test", + "test-user-agent", + Localization("en", "US"), + ContentCountry("US"), + false, + "credential-a", + ) + val versionResolutionStarted = CountDownLatch(1) + val versionResolutionRelease = CountDownLatch(1) + val foregroundStarted = CountDownLatch(1) + val initializations = AtomicInteger() + val synchronousInitializations = AtomicInteger() + try { + assertTrue(prewarmer.start(requestContext.prewarmContext()) { + versionResolutionStarted.countDown() + versionResolutionRelease.await() + initializations.incrementAndGet() + PreparedYoutubeSessionPoToken( + requestContext, + YoutubeSessionPoToken("visitor-data", "prewarmed-token"), + ) + }) + assertTrue(versionResolutionStarted.await(2, TimeUnit.SECONDS)) + + val foreground = foregroundExecutor.submit { + foregroundStarted.countDown() + val prepared = prewarmer.inFlight(requestContext.prewarmContext())?.get() + if (prepared?.context == requestContext) { + prepared.token + } else { + synchronousInitializations.incrementAndGet() + YoutubeSessionPoToken("visitor-data", "synchronous-token") + } + } + assertTrue(foregroundStarted.await(2, TimeUnit.SECONDS)) + assertFalse(foreground.isDone) + assertEquals(0, initializations.get()) + assertEquals(0, synchronousInitializations.get()) + + versionResolutionRelease.countDown() + + assertEquals( + "prewarmed-token", + foreground.get(2, TimeUnit.SECONDS).poToken, + ) + assertEquals(1, initializations.get()) + assertEquals(0, synchronousInitializations.get()) + } finally { + versionResolutionRelease.countDown() + prewarmExecutor.shutdownNow() + foregroundExecutor.shutdownNow() + } + } + + @Test + fun fullPlayerContextControlsTaskIdentity() { + val context = YoutubeSessionPoTokenContext( + "MWEB", + "2.test", + "test-user-agent", + Localization("en", "US"), + ContentCountry("US"), + false, + "credential-a", + ) + + assertNotEquals(context, context.copy(clientName = "WEB")) + assertNotEquals(context, context.copy(clientVersion = "3.test")) + assertEquals( + context.prewarmContext(), + context.copy(clientVersion = "3.test").prewarmContext(), + ) + assertNotEquals(context, context.copy(userAgent = "other-user-agent")) + assertNotEquals(context, context.copy(localization = Localization("zh", "CN"))) + assertNotEquals(context, context.copy(contentCountry = ContentCountry("CN"))) + assertNotEquals(context, context.copy(loggedIn = true)) + assertNotEquals(context, context.copy(credentialIdentity = "credential-b")) + } +}