diff --git a/gradle.properties b/gradle.properties index 137ab2ae..aa33b2fc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.5.1 +appVersion=1.6.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 diff --git a/openapi.yaml b/openapi.yaml index f551cc64..37be9bbe 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -43,6 +43,10 @@ paths: /playlist: { $ref: ./openapi/paths/playlists.yaml#/Playlist } /saved-playlists: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylists } /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } + /subscriptions: { $ref: ./openapi/paths/subscriptions.yaml#/Subscriptions } + /subscriptions/groups: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroups } + /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } + /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } /rss/feeds: { $ref: ./openapi/paths/rss.yaml#/RssFeeds } /rss/feeds/{id}: { $ref: ./openapi/paths/rss.yaml#/RssFeed } @@ -146,6 +150,10 @@ components: $ref: ./openapi/components/media.yaml#/PublicPlaylistItem SavedPlaylistItem: { $ref: ./openapi/components/media.yaml#/SavedPlaylistItem } SavedPlaylistRequest: { $ref: ./openapi/components/media.yaml#/SavedPlaylistRequest } + SubscriptionItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionItem } + SubscriptionGroupItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupItem } + SubscriptionGroupRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupRequest } + SubscriptionGroupMembershipRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index 900de1fe..5e3d7d25 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -13,6 +13,7 @@ SettingsItem: autoplay: { type: boolean, default: true } volume: { type: number, format: double, default: 1.0 } muted: { type: boolean, default: false } + notificationPopupsEnabled: { type: boolean, default: true } subtitlesEnabled: { type: boolean, default: false } defaultSubtitleLanguage: { type: string, default: "" } defaultAudioLanguage: { type: string, default: "" } @@ -34,6 +35,7 @@ SettingsItem: hideComments: { type: boolean, default: false } hideShorts: { type: boolean, default: false } hideSubscriptionLiveStreams: { type: boolean, default: false } + hideMembersOnlyContent: { type: boolean, default: false } accessMode: type: string enum: [unrestricted, allow_list] diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index b0e7ad59..5894cc5f 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -1,3 +1,37 @@ +SubscriptionItem: + type: object + required: [channelUrl, name, avatarUrl, subscribedAt] + properties: + channelUrl: { type: string, minLength: 1 } + name: { type: string } + avatarUrl: { type: string } + subscribedAt: { type: integer, format: int64 } +SubscriptionCreateRequest: + type: object + required: [channelUrl, name, avatarUrl] + properties: + channelUrl: { type: string, minLength: 1 } + name: { type: string } + avatarUrl: { type: string } +SubscriptionGroupItem: + type: object + required: [id, name, channelCount, createdAt, updatedAt] + properties: + id: { type: string, format: uuid } + name: { type: string, minLength: 1, maxLength: 100 } + channelCount: { type: integer, minimum: 0 } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } +SubscriptionGroupRequest: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 100 } +SubscriptionGroupMembershipRequest: + type: object + required: [channelUrl] + properties: + channelUrl: { type: string, minLength: 1 } SubscriptionFeedResponse: type: object required: [videos, nextpage, generation, generatedAt, refreshing] diff --git a/openapi/components/user-backup.yaml b/openapi/components/user-backup.yaml index 36f5a1d4..76428dd8 100644 --- a/openapi/components/user-backup.yaml +++ b/openapi/components/user-backup.yaml @@ -23,6 +23,17 @@ TypeTypeContentFiltersBackup: allowedPlaylists: type: array items: { $ref: ./access-control.yaml#/AllowedPlaylistItem } +SubscriptionGroupBackupItem: + type: object + required: [name, channelUrls, createdAt, updatedAt] + properties: + name: { type: string, minLength: 1, maxLength: 100 } + channelUrls: + type: array + uniqueItems: true + items: { type: string, minLength: 1 } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } TypeTypeBackupItem: type: object required: [format, version, exportedAt, categories] @@ -47,6 +58,7 @@ TypeTypeBackupItem: - settings - contentFilters subscriptions: { type: array, nullable: true, items: { type: object, additionalProperties: true } } + subscriptionGroups: { type: array, nullable: true, items: { $ref: '#/SubscriptionGroupBackupItem' } } history: { type: array, nullable: true, items: { type: object, additionalProperties: true } } playlists: { type: array, nullable: true, items: { type: object, additionalProperties: true } } watchLater: { type: array, nullable: true, items: { type: object, additionalProperties: true } } diff --git a/openapi/components/youtube-session.yaml b/openapi/components/youtube-session.yaml index 8ce67139..963d9ab3 100644 --- a/openapi/components/youtube-session.yaml +++ b/openapi/components/youtube-session.yaml @@ -14,7 +14,7 @@ YoutubeRemoteBrowserStartResponse: expiresAt: { type: integer, format: int64 } YoutubeRemoteBrowserCompleteRequest: type: object - required: [sessionId, tokenSessionId, status, cookies, poToken, capturedAt] + required: [sessionId, tokenSessionId, status, cookies, poToken, authUser, capturedAt] properties: sessionId: { type: string } tokenSessionId: { type: string } @@ -23,4 +23,5 @@ YoutubeRemoteBrowserCompleteRequest: enum: [completed] cookies: { type: string } poToken: { type: string } + authUser: { type: integer, minimum: 0, maximum: 99 } capturedAt: { type: integer, format: int64 } diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml index 33aad372..a0bab142 100644 --- a/openapi/paths/streams.yaml +++ b/openapi/paths/streams.yaml @@ -18,7 +18,11 @@ YoutubeSabrStreams: schema: $ref: ../components/streams.yaml#/StreamResponse '400': - $ref: ../components/common.yaml#/JsonError + description: Invalid request or YouTube account connection required (`youtube_session_required`). + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse '401': $ref: ../components/common.yaml#/JsonError '403': @@ -45,7 +49,11 @@ YoutubeSabrBootstrap: schema: $ref: ../components/streams.yaml#/StreamResponse '400': - $ref: ../components/common.yaml#/JsonError + description: Invalid request or YouTube account connection required (`youtube_session_required`). + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse '401': $ref: ../components/common.yaml#/JsonError '403': diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index f0dcc65c..9f9b4863 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -1,8 +1,161 @@ +Subscriptions: + get: + tags: [user-data] + summary: List the current user's subscriptions + description: Omit both filters for the global list. Use groupId for one named group or ungrouped=true for subscriptions in no groups. + parameters: + - name: groupId + in: query + required: false + schema: { type: string, format: uuid } + - name: ungrouped + in: query + required: false + schema: { type: boolean, default: false } + responses: + '200': + description: The selected subscription projection. + content: + application/json: + schema: + type: array + items: { $ref: ../components/subscriptions.yaml#/SubscriptionItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Subscribe to a channel + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionCreateRequest } + responses: + '201': + description: Subscription created. + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Unsubscribe from a channel + parameters: + - name: url + in: query + required: true + schema: { type: string, minLength: 1 } + responses: + '204': { description: Subscription deleted. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroups: + get: + tags: [user-data] + summary: List the current user's subscription groups + responses: + '200': + description: Account-scoped named groups. + content: + application/json: + schema: + type: array + items: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupItem } + '401': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Create a subscription group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupRequest } + responses: + '201': + description: Group created. + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroup: + parameters: + - name: groupId + in: path + required: true + schema: { type: string, format: uuid } + put: + tags: [user-data] + summary: Rename a subscription group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupRequest } + responses: + '204': { description: Group renamed. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Delete a subscription group + responses: + '204': { description: Group and its memberships deleted. } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroupChannels: + parameters: + - name: groupId + in: path + required: true + schema: { type: string, format: uuid } + put: + tags: [user-data] + summary: Add a subscribed channel to a group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + responses: + '204': { description: Membership exists. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Remove a subscribed channel from a group + parameters: + - name: url + in: query + required: true + schema: { type: string, minLength: 1 } + responses: + '204': { description: Membership deleted. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } SubscriptionFeed: get: tags: [user-data] summary: Read a stable page from the current user's subscription feed snapshot parameters: + - name: groupId + in: query + required: false + description: Restrict the snapshot projection to subscriptions in one account-owned group. + schema: { type: string, format: uuid } + - name: ungrouped + in: query + required: false + description: Restrict the snapshot projection to subscriptions in no groups. + schema: { type: boolean, default: false } - name: page in: query required: false @@ -16,7 +169,7 @@ SubscriptionFeed: - name: cursor in: query required: false - description: Opaque continuation returned in nextpage. + description: Opaque continuation returned in nextpage and bound to the selected membership snapshot. schema: { type: string } responses: '200': @@ -45,6 +198,8 @@ SubscriptionFeed: $ref: ../components/common.yaml#/JsonError '401': $ref: ../components/common.yaml#/JsonError + '404': + $ref: ../components/common.yaml#/JsonError '409': description: The cursor references a snapshot generation that is no longer retained. headers: @@ -54,3 +209,12 @@ SubscriptionFeed: application/json: schema: $ref: ../components/common.yaml#/ErrorResponse + '429': + description: The account already has the maximum number of active filtered cursor sessions. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse diff --git a/openapi/paths/youtube-session.yaml b/openapi/paths/youtube-session.yaml index 2e22552b..a491b069 100644 --- a/openapi/paths/youtube-session.yaml +++ b/openapi/paths/youtube-session.yaml @@ -69,7 +69,7 @@ InternalBrowserComplete: application/json: schema: type: object - required: [sessionId, tokenSessionId, status, cookies, poToken, capturedAt] + required: [sessionId, tokenSessionId, status, cookies, poToken, authUser, capturedAt] properties: sessionId: { type: string } tokenSessionId: { type: string } @@ -78,6 +78,7 @@ InternalBrowserComplete: enum: [completed] cookies: { type: string } poToken: { type: string } + authUser: { type: integer, minimum: 0, maximum: 99 } capturedAt: { type: integer, format: int64 } responses: '204': { description: YouTube credentials stored. } diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 41e04025..1976d9cf 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -92,6 +92,10 @@ internal fun Application.installApplicationRoutes( svc.accessControlService, adminSettingsService, svc.audioOnlyMediaTokenService, + svc.authenticatedSabrInfoService, + svc.youtubeSessionSabrStreamService?.let { service -> + { userId, url -> service.getStreamInfo(userId, url) } + }, ) } downloaderGatewayRoutes(downloaderGatewayService) diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index 681158ef..1c3fd0e5 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -31,6 +31,9 @@ internal fun Route.installStreamRoutes( blockedService = svc.blockedService, publicHlsManifestTokenService = svc.publicHlsManifestTokenService, sabrStreamContractFilter = { url, data -> data.withPlayableSabrStreams(url, svc.sabrSessionStore) }, + youtubeSessionSabrStreamInfo = svc.youtubeSessionSabrStreamService?.let { service -> + { userId, url -> service.getStreamInfo(userId, url) } + }, ) audioOnlyContractRoutes( streamService = svc.streamService, diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 1fe29a3e..5791d986 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -3,6 +3,7 @@ package dev.typetype.server import dev.typetype.server.cache.DragonflyService import dev.typetype.server.services.BilibiliRelatedService import dev.typetype.server.services.BilibiliTrendingService +import dev.typetype.server.services.AuthenticatedSabrInfoService import dev.typetype.server.services.CachedChannelService import dev.typetype.server.services.CachedCommentService import dev.typetype.server.services.CachedManifestService @@ -33,6 +34,7 @@ import dev.typetype.server.services.SabrBootstrapStreamService import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.SignedHlsManifestTokenService import dev.typetype.server.services.TypetypeTokenYoutubeSessionClient +import dev.typetype.server.services.TypetypeTokenSabrTokenClient import dev.typetype.server.services.YouTubeSubtitleService import dev.typetype.server.services.YouTubeSubtitleCache import dev.typetype.server.services.YouTubeSubtitleDeliveryService @@ -40,7 +42,6 @@ import dev.typetype.server.services.OkHttpYouTubeSubtitleContentFetcher import dev.typetype.server.services.StreamYouTubeSubtitleResolver import dev.typetype.server.services.TokenYouTubeSubtitleContentFetcher import dev.typetype.server.services.YoutubePlayerClient -import dev.typetype.server.services.YoutubePlayerClientFallbackStreamService import dev.typetype.server.services.YoutubePlayerClientStreamService import dev.typetype.server.services.YoutubeScopedChannelService import dev.typetype.server.services.YoutubeScopedCommentService @@ -53,6 +54,7 @@ import dev.typetype.server.services.YoutubeSessionCrypto import dev.typetype.server.services.YoutubeSessionHlsManifestService import dev.typetype.server.services.YoutubeSessionService import dev.typetype.server.services.YoutubeSessionStreamService +import dev.typetype.server.services.YoutubeSessionSabrStreamService import okhttp3.ConnectionPool import okhttp3.Dispatcher import okhttp3.OkHttpClient @@ -95,20 +97,27 @@ internal class ExtractionServiceRegistry( directPipePipeStreamService, YoutubePlayerClient.VISIONOS, ) - private val authenticatedStreamService = YoutubePlayerClientFallbackStreamService( + private val authenticatedStreamService = YoutubePlayerClientStreamService( directPipePipeStreamService, - listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.VISIONOS), + YoutubePlayerClient.MWEB, ) private val sabrPublicStreamService = YoutubePlayerClientStreamService( sabrPipePipeStreamService, YoutubePlayerClient.MWEB, ) val youtubeSessionService = YoutubeSessionService(youtubeSessionSecret?.let(YoutubeSessionCrypto::fromSecret)) + val authenticatedSabrInfoService = AuthenticatedSabrInfoService( + youtubeSessionService, + TypetypeTokenSabrTokenClient(subtitleServiceUrl, httpClient), + ) private val hlsTokenService = youtubeSessionSecret?.let(::SignedHlsManifestTokenService) private val tokenYoutubeSessionClient = TypetypeTokenYoutubeSessionClient(subtitleServiceUrl, httpClient) val youtubeSessionStreamService = hlsTokenService?.let { YoutubeSessionStreamService(authenticatedStreamService, youtubeSessionService, cache, it) } + val youtubeSessionSabrStreamService = youtubeSessionStreamService?.let { + YoutubeSessionSabrStreamService(it, authenticatedSabrInfoService) + } val youtubeSabrStreamService = CachedStreamService( YoutubeScopedStreamService( SabrFallbackStreamService(sabrPublicStreamService, sabrSessionStore, tokenYoutubeSessionClient), diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index a6ee84ed..de8e6a0b 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -29,6 +29,7 @@ import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionShortsBlendService import dev.typetype.server.services.SubscriptionShortsFeedService import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation import dev.typetype.server.services.SubscriptionFeedCacheInvalidator import dev.typetype.server.services.TypeTypeBackupService @@ -57,7 +58,9 @@ internal class ServiceRegistry( youtubeProxySelector, ) val youtubeSessionService = extraction.youtubeSessionService + val authenticatedSabrInfoService = extraction.authenticatedSabrInfoService val youtubeSessionStreamService = extraction.youtubeSessionStreamService + val youtubeSessionSabrStreamService = extraction.youtubeSessionSabrStreamService val youtubeSabrStreamService = extraction.youtubeSabrStreamService val youtubeSabrBootstrapStreamService = extraction.youtubeSabrBootstrapStreamService val nicoNicoStreamService = extraction.nicoNicoStreamService @@ -82,6 +85,7 @@ internal class ServiceRegistry( val sabrSessionStore = extraction.sabrSessionStore val historyService = HistoryService() val subscriptionsService = SubscriptionsService() + val subscriptionGroupsService = SubscriptionGroupsService() val subscriptionFeedService = SubscriptionFeedService(subscriptionsService, channelService, cache) val subscriptionShortsFeedService = SubscriptionShortsFeedService( subscriptionsService, diff --git a/src/main/kotlin/dev/typetype/server/UserDataRateLimitKey.kt b/src/main/kotlin/dev/typetype/server/UserDataRateLimitKey.kt index 454edc98..e5720ec9 100644 --- a/src/main/kotlin/dev/typetype/server/UserDataRateLimitKey.kt +++ b/src/main/kotlin/dev/typetype/server/UserDataRateLimitKey.kt @@ -3,11 +3,11 @@ package dev.typetype.server import dev.typetype.server.services.AuthService import io.ktor.server.application.ApplicationCall -fun userDataRateLimitKey(call: ApplicationCall, authService: AuthService): String { +suspend fun userDataRateLimitKey(call: ApplicationCall, authService: AuthService): String { val bearerToken = call.request.headers["Authorization"] ?.takeIf { it.startsWith("Bearer ") } ?.substringAfter("Bearer ") - val userId = bearerToken?.let(authService::verify) + val userId = bearerToken?.let { authService.verify(it) } if (userId != null) return "user:$userId" return "ip:${call.request.headers["X-Real-IP"] ?: call.request.local.remoteHost}" } diff --git a/src/main/kotlin/dev/typetype/server/cache/CacheService.kt b/src/main/kotlin/dev/typetype/server/cache/CacheService.kt index cd67eb79..40f16770 100644 --- a/src/main/kotlin/dev/typetype/server/cache/CacheService.kt +++ b/src/main/kotlin/dev/typetype/server/cache/CacheService.kt @@ -3,5 +3,9 @@ package dev.typetype.server.cache interface CacheService { suspend fun get(key: String): String? suspend fun set(key: String, value: String, ttlSeconds: Long) + suspend fun setIfAbsent(key: String, value: String, ttlSeconds: Long): Boolean = + throw UnsupportedOperationException("Atomic set-if-absent is not supported") + suspend fun refreshIfValueMatches(key: String, value: String, ttlSeconds: Long): Boolean = + throw UnsupportedOperationException("Atomic compare-and-expire is not supported") suspend fun delete(key: String) } diff --git a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt index 32aa51b2..d8517ccd 100644 --- a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt +++ b/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt @@ -1,6 +1,8 @@ package dev.typetype.server.cache import io.lettuce.core.RedisClient +import io.lettuce.core.ScriptOutputType +import io.lettuce.core.SetArgs import io.lettuce.core.api.StatefulRedisConnection import io.lettuce.core.api.async.RedisAsyncCommands import kotlinx.coroutines.future.await @@ -17,8 +19,25 @@ class DragonflyService(url: String) : CacheService { override suspend fun set(key: String, value: String, ttlSeconds: Long): Unit = async.setex(key, ttlSeconds, value).await().let {} + override suspend fun setIfAbsent(key: String, value: String, ttlSeconds: Long): Boolean = + async.set(key, value, SetArgs.Builder.nx().ex(ttlSeconds)).await() == "OK" + + override suspend fun refreshIfValueMatches(key: String, value: String, ttlSeconds: Long): Boolean = + async.eval( + REFRESH_IF_VALUE_MATCHES, + ScriptOutputType.INTEGER, + arrayOf(key), + value, + ttlSeconds.toString(), + ).await() == 1L + override suspend fun delete(key: String): Unit = async.del(key).await().let {} suspend fun ping(): Boolean = async.ping().await() == "PONG" + + private companion object { + const val REFRESH_IF_VALUE_MATCHES = + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('expire', KEYS[1], ARGV[2]) end return 0" + } } diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index f964f6db..0ffc8fa9 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -16,6 +16,8 @@ import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SettingsTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.UserAvatarsTable import dev.typetype.server.db.tables.WatchLaterTable @@ -38,6 +40,9 @@ import org.jetbrains.exposed.v1.jdbc.SchemaUtils import org.jetbrains.exposed.v1.jdbc.transactions.transaction object DatabaseFactory { + private const val POOL_SIZE = 10 + private val queryDispatcher = Dispatchers.IO.limitedParallelism(POOL_SIZE, "database") + fun init(url: String, user: String, password: String) { val dbPassword = password val config = HikariConfig().apply { @@ -45,7 +50,7 @@ object DatabaseFactory { username = user this.password = dbPassword driverClassName = "org.postgresql.Driver" - maximumPoolSize = 10 + maximumPoolSize = POOL_SIZE minimumIdle = 2 } Database.connect(HikariDataSource(config)) @@ -57,6 +62,8 @@ object DatabaseFactory { AdminSettingsTable, HistoryTable, SubscriptionsTable, + SubscriptionGroupsTable, + SubscriptionGroupMembershipsTable, PlaylistsTable, PlaylistVideosTable, WatchLaterTable, @@ -119,6 +126,7 @@ object DatabaseFactory { exec("ALTER TABLE users ADD COLUMN IF NOT EXISTS public_username TEXT") exec("ALTER TABLE users ADD COLUMN IF NOT EXISTS bio TEXT") exec("ALTER TABLE youtube_takeout_import_jobs ADD COLUMN IF NOT EXISTS preview_json TEXT") + exec("ALTER TABLE youtube_sessions ADD COLUMN IF NOT EXISTS auth_user INTEGER NOT NULL DEFAULT 0") exec("ALTER TABLE bug_reports ALTER COLUMN github_issue_url TYPE TEXT") DatabaseSessionAuthMigration.apply() DatabaseOidcMigration.apply() @@ -131,7 +139,10 @@ object DatabaseFactory { DatabaseCollectionMetadataMigration.apply() } } - suspend fun query(block: () -> T): T = withContext(Dispatchers.IO) { transaction { block() } } + suspend fun query(block: () -> T): T = blocking { transaction { block() } } + + suspend fun blocking(block: () -> T): T = withContext(queryDispatcher) { block() } + fun healthCheck(): Boolean = runCatching { transaction { exec("SELECT 1") { it.next() } == true } }.getOrDefault(false) diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index 45566d0f..e49baf63 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -5,6 +5,7 @@ import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager object SettingsSchemaMigrations { fun apply() { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subtitles_enabled BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS notification_popups_enabled BOOLEAN NOT NULL DEFAULT true") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_playback_speed DOUBLE PRECISION NOT NULL DEFAULT 1.0") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_subtitle_language TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_audio_language TEXT NOT NULL DEFAULT ''") @@ -25,6 +26,7 @@ object SettingsSchemaMigrations { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_comments BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_shorts BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_subscription_live_streams BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_members_only_content BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS disable_watch_history BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS skip_playlist_autoplay_screen BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index 4744441f..253b348f 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -12,6 +12,7 @@ object SettingsTable : Table("settings") { val skipPlaylistAutoplayScreen = bool("skip_playlist_autoplay_screen").default(false) val volume = double("volume").default(1.0) val muted = bool("muted").default(false) + val notificationPopupsEnabled = bool("notification_popups_enabled").default(true) val subtitlesEnabled = bool("subtitles_enabled").default(false) val defaultSubtitleLanguage = text("default_subtitle_language").default("") val defaultAudioLanguage = text("default_audio_language").default("") @@ -33,6 +34,7 @@ object SettingsTable : Table("settings") { val hideComments = bool("hide_comments").default(false) val hideShorts = bool("hide_shorts").default(false) val hideSubscriptionLiveStreams = bool("hide_subscription_live_streams").default(false) + val hideMembersOnlyContent = bool("hide_members_only_content").default(false) val disableWatchHistory = bool("disable_watch_history").default(false) val deArrowEnabled = bool("dearrow_enabled").default(false) val deArrowTitleMode = text("dearrow_title_mode").default("dearrow") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt new file mode 100644 index 00000000..93931a89 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.ReferenceOption +import org.jetbrains.exposed.v1.core.Table + +object SubscriptionGroupMembershipsTable : Table("subscription_group_memberships") { + val groupId = text("group_id").references(SubscriptionGroupsTable.id, onDelete = ReferenceOption.CASCADE) + val userId = text("user_id") + val channelUrl = text("channel_url") + val addedAt = long("added_at") + + init { + index(false, userId, channelUrl) + } + + override val primaryKey = PrimaryKey(groupId, channelUrl) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt new file mode 100644 index 00000000..13b89b20 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object SubscriptionGroupsTable : Table("subscription_groups") { + val id = text("id") + val userId = text("user_id") + val name = text("name") + val normalizedName = text("normalized_name") + val createdAt = long("created_at") + val updatedAt = long("updated_at") + + init { + uniqueIndex(userId, normalizedName) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt index a904aadb..3d05bf41 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt @@ -6,6 +6,7 @@ object YoutubeSessionsTable : Table("youtube_sessions") { val userId = text("user_id") val encryptedCookies = text("encrypted_cookies") val encryptedPoToken = text("encrypted_po_token") + val authUser = integer("auth_user").default(0) val status = text("status") val createdAt = long("created_at") val updatedAt = long("updated_at") diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt b/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt index de00fc06..1e9072ca 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt +++ b/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt @@ -40,6 +40,7 @@ class OkHttpDownloader private constructor( } private const val STREAMING_READ_TIMEOUT_MS = 30_000L + private const val YOUTUBE_AUTH_USER_HEADER = "X-Goog-AuthUser" } override fun execute(request: ExtractorRequest): Response { @@ -102,15 +103,19 @@ class OkHttpDownloader private constructor( private fun buildOkHttpRequest(request: ExtractorRequest): Request { val method = request.httpMethod() val dataToSend = request.dataToSend() + val normalizedUrl = normalizeExtractorUrl(request.url()) val body = dataToSend?.toRequestBody() ?: if (method == "POST" || method == "PUT" || method == "PATCH") ByteArray(0).toRequestBody() else null val builder = Request.Builder() - .url(normalizeExtractorUrl(request.url())) + .url(normalizedUrl) .method(method, body) request.headers().forEach { (name, values) -> values.forEach { value -> builder.addHeader(name, value) } } + YoutubeAuthUserContext.headerFor(normalizedUrl)?.let { + builder.header(YOUTUBE_AUTH_USER_HEADER, it) + } return builder.build() } diff --git a/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt b/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt new file mode 100644 index 00000000..ad03ff60 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.downloader + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull + +internal object YoutubeAuthUserContext { + @Volatile private var value: Int? = null + + fun set(authUser: Int?): Unit { + value = authUser + } + + internal fun headerFor(url: String): String? { + val parsed = url.toHttpUrlOrNull() ?: return null + val host = parsed.host.lowercase() + val isYoutube = host == "youtube.com" || host.endsWith(".youtube.com") + return value?.toString()?.takeIf { + isYoutube && parsed.encodedPath.startsWith("/youtubei/v1/") + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index b6f46a83..5d5d7447 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -12,6 +12,7 @@ data class SettingsItem( val skipPlaylistAutoplayScreen: Boolean = false, val volume: Double = 1.0, val muted: Boolean = false, + val notificationPopupsEnabled: Boolean = true, val subtitlesEnabled: Boolean = false, val defaultSubtitleLanguage: String = "", val defaultAudioLanguage: String = "", @@ -33,6 +34,7 @@ data class SettingsItem( val hideComments: Boolean = false, val hideShorts: Boolean = false, val hideSubscriptionLiveStreams: Boolean = false, + val hideMembersOnlyContent: Boolean = false, val disableWatchHistory: Boolean = false, val deArrowEnabled: Boolean = false, val deArrowTitleMode: String = "dearrow", diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt new file mode 100644 index 00000000..0e530fdf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionCreateRequest( + val channelUrl: String, + val name: String, + val avatarUrl: String, +) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt new file mode 100644 index 00000000..efae0cab --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupBackupItem( + val name: String, + val channelUrls: List, + val createdAt: Long, + val updatedAt: Long, +) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt new file mode 100644 index 00000000..684382bb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupItem( + val id: String, + val name: String, + val channelCount: Int, + val createdAt: Long, + val updatedAt: Long, +) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt new file mode 100644 index 00000000..9a2fcb5f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupMembershipRequest(val channelUrl: String) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt new file mode 100644 index 00000000..136b831c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupRequest(val name: String) diff --git a/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt b/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt index 4c9300b9..82cb9f40 100644 --- a/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt @@ -9,6 +9,7 @@ data class TypeTypeBackupItem( val exportedAt: Long, val categories: List, val subscriptions: List? = null, + val subscriptionGroups: List? = null, val history: List? = null, val playlists: List? = null, val watchLater: List? = null, diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt b/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt index 42534723..f45ee274 100644 --- a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt +++ b/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt @@ -9,5 +9,6 @@ data class YoutubeRemoteBrowserCompleteRequest( val status: String, val cookies: String, val poToken: String, + val authUser: Int = 0, val capturedAt: Long, ) diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt b/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt index 261033fc..50855e4b 100644 --- a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt +++ b/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt @@ -7,4 +7,5 @@ data class YoutubeSessionCompleteRequest( val code: String, val cookies: String, val poToken: String, + val authUser: Int = 0, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt index 19bfda0f..d791eeaf 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt @@ -114,5 +114,5 @@ fun Route.authRoutes( } } -private fun String.warm(authService: AuthService, warmupService: HomeRecommendationWarmup): Unit = +private suspend fun String.warm(authService: AuthService, warmupService: HomeRecommendationWarmup): Unit = authService.verify(this)?.let(warmupService::markActive) ?: Unit diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt index 18ca2142..44b4e912 100644 --- a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt @@ -102,13 +102,16 @@ private fun shouldProxyArtifact(path: String, response: dev.typetype.server.serv if (!path.endsWith("/artifact")) return false if (response.status != 302 && response.status != 307) return false val location = headerValue(response, "Location") ?: return false - return isInternalHost(location) + val markedInternal = headerValue(response, INTERNAL_ARTIFACT_PROXY_HEADER) == "1" + return markedInternal || isLegacyInternalHost(location) } private fun headerValue(response: dev.typetype.server.services.DownloaderGatewayResponse, name: String): String? = response.headers.firstOrNull { it.first.equals(name, ignoreCase = true) }?.second -private fun isInternalHost(location: String): Boolean { +private fun isLegacyInternalHost(location: String): Boolean { val host = runCatching { URI(location).host }.getOrNull() ?: return false return host.equals("garage", ignoreCase = true) } + +private const val INTERNAL_ARTIFACT_PROXY_HEADER = "X-TypeType-Artifact-Proxy" diff --git a/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt index ac9f1f4a..c3b59832 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt @@ -23,7 +23,7 @@ fun Route.registerRoutes( get("/auth/register/status") { val bootstrapAvailable = !authService.hasAdmin() val settings = adminSettingsService.get() - call.respond( + call.respondNoStore( RegisterStatusResponse( allowRegistration = settings.allowRegistration, bootstrapAvailable = bootstrapAvailable, diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt new file mode 100644 index 00000000..4cbf4b00 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt @@ -0,0 +1,25 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.services.StreamService +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR +import dev.typetype.server.services.requiresYoutubeSession + +internal class SabrPlaybackAccessValidator( + private val publicStreamService: StreamService, + private val youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)?, +) { + suspend fun resolve(userId: String?, videoId: String): ExtractionResult { + val url = "https://www.youtube.com/watch?v=$videoId" + val publicResult = publicStreamService.getStreamInfo(url) + val authenticatedResult = userId?.let { id -> youtubeSessionStreamInfo?.invoke(id, url) } + if (authenticatedResult != null) return authenticatedResult + return if (publicResult.requiresYoutubeSession() && youtubeSessionStreamInfo != null) { + ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE) + } else { + publicResult + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index c4abe087..0e544d4b 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -2,12 +2,15 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AuthenticatedSabrInfoService import dev.typetype.server.services.AuthService import dev.typetype.server.services.SabrPreparedInfo import dev.typetype.server.services.SabrPlaybackSegmentResult import dev.typetype.server.services.SabrPlaybackSessionService +import dev.typetype.server.services.SabrPlaybackInfoResolver import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService @@ -23,15 +26,19 @@ internal class SabrPlaybackHandler( private val authService: AuthService?, private val accessControlService: AccessControlService?, private val adminSettingsService: AdminSettingsService?, + authenticatedSabrInfoService: AuthenticatedSabrInfoService? = null, + youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, ) { private val playbackService = SabrPlaybackSessionService(sabrSessionStore) + private val infoResolver = SabrPlaybackInfoResolver(sabrSessionStore, authenticatedSabrInfoService) + private val accessValidator = SabrPlaybackAccessValidator(streamService, youtubeSessionStreamInfo) suspend fun create(call: ApplicationCall, videoId: String) { val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return if (!validateAccess(call, videoId, access)) return val request = call.playbackRequest() val startTimeMs = request.effectiveStartTimeMs() - val prepared = sabrSessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = true) + val prepared = infoResolver.initial(access.userId, videoId, startTimeMs) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) val audio = selectAudio(call, prepared, request) ?: return val video = selectVideo(call, prepared, request) ?: return @@ -58,7 +65,7 @@ internal class SabrPlaybackHandler( val preparation = playbackService.seekExisting(holder, playerTimeMs, request.audioOnly) return respondPrepared(call, holder, holder.key.videoId, preparation.startTimeMs, preparation.ready) } - val prepared = sabrSessionStore.fetchInfo(holder.key.videoId, playerTimeMs, cachedFirst = true) + val prepared = infoResolver.replacement(holder, playerTimeMs) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) val audio = SabrFormatSelector.audio( prepared.info, @@ -121,20 +128,21 @@ internal class SabrPlaybackHandler( } private suspend fun validateAccess(call: ApplicationCall, videoId: String, access: AccessRouteProfile): Boolean { - if (!access.profile.enabled) return true - return when (val result = streamService.getStreamInfo("https://www.youtube.com/watch?v=$videoId")) { + return when (val result = accessValidator.resolve(access.userId, videoId)) { is ExtractionResult.Success -> { - if (access.profile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) true else { + val allowed = !access.profile.enabled || + access.profile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName) + if (allowed) true else { call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) false } } is ExtractionResult.Failure -> { - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) false } is ExtractionResult.BadRequest -> { - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) false } } diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt index 2f4abe7f..8f99d264 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt @@ -1,10 +1,13 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AudioOnlyMediaTokenService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.AuthenticatedSabrInfoService import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService import io.ktor.http.HttpStatusCode @@ -20,6 +23,8 @@ internal fun Route.sabrRoutes( accessControlService: AccessControlService?, adminSettingsService: AdminSettingsService?, audioOnlyTokenService: AudioOnlyMediaTokenService?, + authenticatedSabrInfoService: AuthenticatedSabrInfoService? = null, + youtubeSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, ) { val sessionHandler = SabrSessionDescriptorHandler( sabrSessionStore, @@ -49,6 +54,8 @@ internal fun Route.sabrRoutes( authService, accessControlService, adminSettingsService, + authenticatedSabrInfoService, + youtubeSessionStreamInfo, ) val playbackStateHandler = SabrPlaybackStateHandler(sabrSessionStore) val playbackWindowHandler = SabrPlaybackWindowHandler(sabrSessionStore) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt index 6598572a..177533c7 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt @@ -1,5 +1,6 @@ package dev.typetype.server.routes +import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService @@ -14,4 +15,5 @@ internal data class StreamRouteDependencies( val blockedService: BlockedService?, val publicHlsManifestTokenService: PublicHlsManifestTokenService?, val sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)?, + val youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)?, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index 81f60626..9f315c08 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -10,8 +10,11 @@ import dev.typetype.server.services.BlockedContentProfile import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.StreamService -import dev.typetype.server.services.filterBlocked +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR import dev.typetype.server.services.filterAllowed +import dev.typetype.server.services.filterBlocked +import dev.typetype.server.services.requiresYoutubeSession import dev.typetype.server.services.withSabrManifestUrls import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode @@ -32,6 +35,7 @@ fun Route.streamRoutes( nicoNicoStreamService: StreamService = streamService, bilibiliStreamService: StreamService = streamService, sabrBootstrapStreamService: StreamService = streamService, + youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)? = null, ) { val dependencies = StreamRouteDependencies( @@ -41,6 +45,7 @@ fun Route.streamRoutes( blockedService = blockedService, publicHlsManifestTokenService = publicHlsManifestTokenService, sabrStreamContractFilter = sabrStreamContractFilter, + youtubeSessionSabrStreamInfo = youtubeSessionSabrStreamInfo, ) streamRoute("/streams/youtube/sabr", StreamDeliveryMode.YoutubeSabr, streamService, dependencies) streamRoute( @@ -83,7 +88,9 @@ private fun Route.streamRoute( ErrorResponse("Video is blocked", "content_blocked"), ) } - when (val result = streamService.getStreamInfo(url)) { + val publicResult = streamService.getStreamInfo(url) + val resolution = resolveStreamInfo(url, deliveryMode, access.userId, publicResult, dependencies) + when (val result = resolution.result) { is ExtractionResult.Success -> { if (!accessProfile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) @@ -106,7 +113,11 @@ private fun Route.streamRoute( deliveryMode.isSabr() && selected.isLive || access.userId != null && !access.allowGuest, dependencies.publicHlsManifestTokenService, ) - val data = if (!deliveryMode.isSabr() || dependencies.sabrStreamContractFilter == null) { + val data = if ( + !deliveryMode.isSabr() || + resolution.authenticated || + dependencies.sabrStreamContractFilter == null + ) { filtered } else { dependencies.sabrStreamContractFilter.invoke(url, filtered) @@ -131,6 +142,33 @@ private fun Route.streamRoute( } } +private data class StreamResolution( + val result: ExtractionResult, + val authenticated: Boolean = false, +) + +private suspend fun resolveStreamInfo( + url: String, + deliveryMode: StreamDeliveryMode, + userId: String?, + publicResult: ExtractionResult, + dependencies: StreamRouteDependencies, +): StreamResolution { + val authenticatedInfo = dependencies.youtubeSessionSabrStreamInfo + if (!deliveryMode.isSabr() || authenticatedInfo == null) { + return StreamResolution(publicResult) + } + val authenticatedResult = userId?.let { authenticatedInfo(it, url) } + if (authenticatedResult != null) return StreamResolution(authenticatedResult, authenticated = true) + return if (publicResult.requiresYoutubeSession()) { + StreamResolution( + ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE), + ) + } else { + StreamResolution(publicResult) + } +} + private fun StreamResponse.hasPlayableSource(): Boolean = videoStreams.isNotEmpty() || videoOnlyStreams.isNotEmpty() || diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index 07154402..abdfb136 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -2,9 +2,13 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.SubscriptionFeedPreparingResponse +import dev.typetype.server.preserveTooManyRequestsBody import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionFeedVisibility +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.SettingsService import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode @@ -18,15 +22,37 @@ fun Route.subscriptionFeedRoutes( feedService: SubscriptionFeedService, authService: AuthService, settingsService: SettingsService? = null, + groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), ) { get("/subscriptions/feed") { call.withJwtAuth(authService) { userId -> + val parsed = call.parseSubscriptionSelection() + if (parsed !is SubscriptionSelectionParseResult.Valid) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid subscription filter")) + } + val selection = parsed.selection + val cursor = call.request.queryParameters["cursor"] + if (cursor == null && selection is SubscriptionSelection.Group && !groupsService.exists(userId, selection.id)) { + return@withJwtAuth call.respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } val page = call.request.queryParameters["page"]?.toIntOrNull()?.coerceIn(0, MAX_FEED_PAGE) ?: 0 val limit = call.request.queryParameters["limit"]?.toIntOrNull()?.coerceIn(1, 100) ?: 30 - val cursor = call.request.queryParameters["cursor"] - val hideLiveStreams = settingsService?.hidesSubscriptionLiveStreams(userId) ?: false + val visibility = settingsService?.subscriptionFeedVisibility(userId) ?: SubscriptionFeedVisibility() call.response.headers.append(HttpHeaders.CacheControl, "no-store") - when (val result = feedService.getPage(userId, page, limit, cursor, hideLiveStreams)) { + when ( + val result = feedService.getPage( + userId, + page, + limit, + cursor, + visibility.hideLiveStreams, + visibility.hideMembersOnlyContent, + selection, + ) + ) { is SubscriptionFeedPageResult.Ready -> call.respond(result.response) is SubscriptionFeedPageResult.Preparing -> { call.response.headers.append(HttpHeaders.RetryAfter, "1") @@ -43,6 +69,13 @@ fun Route.subscriptionFeedRoutes( HttpStatusCode.Conflict, ErrorResponse("Subscription feed generation is no longer available", "subscription_feed_stale_generation"), ) + SubscriptionFeedPageResult.CursorCapacityReached -> { + call.preserveTooManyRequestsBody() + call.respond( + HttpStatusCode.TooManyRequests, + ErrorResponse("Too many active subscription feed cursors", "subscription_feed_cursor_capacity"), + ) + } } } } diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt new file mode 100644 index 00000000..c80acb3d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt @@ -0,0 +1,114 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.SubscriptionGroupMembershipRequest +import dev.typetype.server.models.SubscriptionGroupRequest +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put + +fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, authService: AuthService) { + get("/subscriptions/groups") { + call.withJwtAuth(authService) { userId -> call.respond(groupsService.getAll(userId)) } + } + post("/subscriptions/groups") { + call.withJwtAuth(authService) { userId -> + val request = call.receiveGroupRequest() ?: return@withJwtAuth + call.respondGroupWrite(groupsService.create(userId, request.name), created = true) + } + } + put("/subscriptions/groups/{groupId}") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val request = call.receiveGroupRequest() ?: return@withJwtAuth + call.respondGroupWrite(groupsService.rename(userId, groupId, request.name), created = false) + } + } + delete("/subscriptions/groups/{groupId}") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + if (groupsService.delete(userId, groupId)) call.respond(HttpStatusCode.NoContent) else { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Subscription group not found", "subscription_group_not_found")) + } + } + } + put("/subscriptions/groups/{groupId}/channels") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val request = runCatching { call.receive() }.getOrElse { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + if (request.channelUrl.isBlank()) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("channelUrl must not be blank")) + } + call.respondMembership(groupsService.addSubscription(userId, groupId, request.channelUrl)) + } + } + delete("/subscriptions/groups/{groupId}/channels") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val channelUrl = call.request.queryParameters["url"]?.takeIf(String::isNotBlank) + ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) + call.respondMembership(groupsService.removeSubscription(userId, groupId, channelUrl)) + } + } +} + +private fun ApplicationCall.groupId(): String? = parameters["groupId"]?.takeIf(String::isNotBlank) + +private suspend fun ApplicationCall.receiveGroupRequest(): SubscriptionGroupRequest? = + runCatching { receive() }.getOrElse { + respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + null + } + +private suspend fun ApplicationCall.respondGroupWrite(result: SubscriptionGroupWriteResult, created: Boolean) { + when (result) { + is SubscriptionGroupWriteResult.Success -> if (created) respond(HttpStatusCode.Created, result.group) else { + respond(HttpStatusCode.NoContent) + } + SubscriptionGroupWriteResult.InvalidName -> respond( + HttpStatusCode.BadRequest, + ErrorResponse("Group name must contain 1 to 100 characters", "subscription_group_invalid_name"), + ) + SubscriptionGroupWriteResult.DuplicateName -> respond( + HttpStatusCode.Conflict, + ErrorResponse("A subscription group with this name already exists", "subscription_group_name_conflict"), + ) + SubscriptionGroupWriteResult.NotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } +} + +private suspend fun ApplicationCall.respondMembership(result: SubscriptionGroupMembershipResult) { + when (result) { + SubscriptionGroupMembershipResult.Success -> respond(HttpStatusCode.NoContent) + SubscriptionGroupMembershipResult.GroupNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + SubscriptionGroupMembershipResult.SubscriptionNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription not found", "subscription_not_found"), + ) + SubscriptionGroupMembershipResult.MembershipNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group membership not found", "subscription_group_membership_not_found"), + ) + } +} + +private suspend fun ApplicationCall.respondMissingGroupId() = + respond(HttpStatusCode.BadRequest, ErrorResponse("Missing groupId")) diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt new file mode 100644 index 00000000..a847aab0 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SubscriptionSelection +import io.ktor.server.application.ApplicationCall + +internal sealed interface SubscriptionSelectionParseResult { + data class Valid(val selection: SubscriptionSelection) : SubscriptionSelectionParseResult + data object Invalid : SubscriptionSelectionParseResult +} + +internal fun ApplicationCall.parseSubscriptionSelection(): SubscriptionSelectionParseResult { + val rawGroupId = request.queryParameters["groupId"] + val groupId = rawGroupId?.takeIf(String::isNotBlank) + if (rawGroupId != null && groupId == null) return SubscriptionSelectionParseResult.Invalid + val rawUngrouped = request.queryParameters["ungrouped"] + val ungrouped = when (rawUngrouped) { + null -> false + "true" -> true + "false" -> false + else -> return SubscriptionSelectionParseResult.Invalid + } + if (groupId != null && ungrouped) return SubscriptionSelectionParseResult.Invalid + val selection = when { + groupId != null -> SubscriptionSelection.Group(groupId) + ungrouped -> SubscriptionSelection.Ungrouped + else -> SubscriptionSelection.All + } + return SubscriptionSelectionParseResult.Valid(selection) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index 3d7b40b2..becd5d9b 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -1,11 +1,14 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.SubscriptionCreateRequest import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.services.AuthService import dev.typetype.server.services.HomeRecommendationWarmup import dev.typetype.server.services.NoopHomeRecommendationWarmup import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -18,16 +21,37 @@ import io.ktor.server.routing.post import java.net.URLDecoder import java.nio.charset.StandardCharsets -fun Route.subscriptionsRoutes(subscriptionsService: SubscriptionsService, authService: AuthService, warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup) { +fun Route.subscriptionsRoutes( + subscriptionsService: SubscriptionsService, + authService: AuthService, + warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, + groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), +) { get("/subscriptions") { - call.withJwtAuth(authService) { userId -> call.respond(subscriptionsService.getAll(userId)) } + call.withJwtAuth(authService) { userId -> + val parsed = call.parseSubscriptionSelection() + if (parsed !is SubscriptionSelectionParseResult.Valid) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid subscription filter")) + } + val selection = parsed.selection + if (selection is SubscriptionSelection.Group && !groupsService.exists(userId, selection.id)) { + return@withJwtAuth call.respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } + call.respond(subscriptionsService.getAll(userId, selection)) + } } post("/subscriptions") { call.withJwtAuth(authService) { userId -> - val item = runCatching { call.receive() }.getOrElse { + val request = runCatching { call.receive() }.getOrElse { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) } - val subscription = subscriptionsService.add(userId, item) + val subscription = subscriptionsService.add( + userId, + SubscriptionItem(request.channelUrl, request.name, request.avatarUrl), + ) warmupService.invalidateAndWarm(userId) call.respond(HttpStatusCode.Created, subscription) } diff --git a/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt b/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt index 5970649f..6aaf8259 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt @@ -21,7 +21,7 @@ suspend fun ApplicationCall.withJwtAuth(authService: AuthService, block: suspend block(userId) } -fun ApplicationCall.optionalJwtUserId(authService: AuthService): String? { +suspend fun ApplicationCall.optionalJwtUserId(authService: AuthService): String? { val authHeader = request.headers["Authorization"] if (authHeader == null || !authHeader.startsWith("Bearer ")) return null val token = authHeader.substringAfter("Bearer ") diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 74baab91..1ff721c8 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -17,8 +17,19 @@ internal fun Route.userDataRoutes( restoreService: PipePipeBackupImporterService, ) { historyRoutes(svc.historyService, authService, svc.settingsService) - subscriptionsRoutes(svc.subscriptionsService, authService, svc.homeRecommendationWarmupService) - subscriptionFeedRoutes(svc.subscriptionFeedService, authService, svc.settingsService) + subscriptionGroupsRoutes(svc.subscriptionGroupsService, authService) + subscriptionsRoutes( + svc.subscriptionsService, + authService, + svc.homeRecommendationWarmupService, + svc.subscriptionGroupsService, + ) + subscriptionFeedRoutes( + svc.subscriptionFeedService, + authService, + svc.settingsService, + svc.subscriptionGroupsService, + ) subscriptionShortsFeedRoutes(svc.subscriptionShortsFeedService, authService) rssFeedRoutes(svc.rssFeedManagementService, authService) playlistRoutes(svc.playlistService, authService, svc.videoMetadataRepairService) diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/src/main/kotlin/dev/typetype/server/services/AuthService.kt index 21a5eb59..54a81a8e 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AuthService.kt @@ -3,7 +3,10 @@ package dev.typetype.server.services import com.auth0.jwt.JWT import com.auth0.jwt.algorithms.Algorithm import com.password4j.Password +import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.UsersTable +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.lowerCase import org.jetbrains.exposed.v1.jdbc.insert @@ -24,8 +27,8 @@ open class AuthService( private val sessionVerifier = AuthSessionVerifier(accessCodec, sessionStore) private val sessionRevoker = AuthSessionRevoker(sessionStore) - fun register(email: String, password: String, name: String): AuthSessionTokens { - val hashed = Password.hash(password).withArgon2().result + suspend fun register(email: String, password: String, name: String): AuthSessionTokens { + val hashed = withContext(passwordDispatcher) { Password.hash(password).withArgon2().result } val userId = UUID.randomUUID().toString() val now = System.currentTimeMillis() @@ -33,7 +36,7 @@ open class AuthService( val role = if (needsAdmin) "admin" else "user" val publicUsername = name.trim().takeIf(ProfileService::isValidPublicUsername) - transaction { + DatabaseFactory.query { UsersTable.insert { it[UsersTable.id] = userId it[UsersTable.email] = email @@ -45,13 +48,15 @@ open class AuthService( it[UsersTable.updatedAt] = now } } - return tokenIssuer.issue(userId) ?: throw IllegalStateException("Failed to create session") + return DatabaseFactory.blocking { + tokenIssuer.issue(userId) ?: throw IllegalStateException("Failed to create session") + } } - fun login(identifier: String, password: String): AuthSessionTokens? { + suspend fun login(identifier: String, password: String): AuthSessionTokens? { val normalizedIdentifier = identifier.trim().lowercase() if (normalizedIdentifier.isBlank()) return null - val user = transaction { + val user = DatabaseFactory.query { val query = UsersTable.selectAll().where { if (normalizedIdentifier.contains("@")) { UsersTable.email.lowerCase() eq normalizedIdentifier @@ -63,22 +68,26 @@ open class AuthService( } ?: return null val hashed = user[UsersTable.passwordHash] - val verified = Password.check(password, hashed).withArgon2() + val verified = withContext(passwordDispatcher) { Password.check(password, hashed).withArgon2() } if (!verified) return null - return tokenIssuer.issue(user[UsersTable.id]) + return DatabaseFactory.blocking { tokenIssuer.issue(user[UsersTable.id]) } } - fun refreshSession(refreshToken: String): AuthSessionTokens? = sessionRefresher.refresh(refreshToken) + suspend fun refreshSession(refreshToken: String): AuthSessionTokens? = DatabaseFactory.blocking { + sessionRefresher.refresh(refreshToken) + } - fun issueSession(userId: String): AuthSessionTokens? = tokenIssuer.issue(userId) + suspend fun issueSession(userId: String): AuthSessionTokens? = DatabaseFactory.blocking { + tokenIssuer.issue(userId) + } - fun logout(refreshToken: String?) { + suspend fun logout(refreshToken: String?): Unit = DatabaseFactory.blocking { sessionRevoker.revokeByRefreshToken(refreshToken) } - open fun verify(token: String): String? { - return sessionVerifier.verifyUserId(token) + open suspend fun verify(token: String): String? = DatabaseFactory.blocking { + sessionVerifier.verifyUserId(token) } fun guestLogin(): String { @@ -91,28 +100,33 @@ open class AuthService( .sign(Algorithm.HMAC256(jwtSecret)) } - fun getUserRole(userId: String): String? { + suspend fun getUserRole(userId: String): String? { if (userId.startsWith("guest:")) return "user" - return transaction { + return DatabaseFactory.query { UsersTable.selectAll().where { UsersTable.id eq userId }.singleOrNull() }?.get(UsersTable.role) } - fun hasUsers(): Boolean = hasUsersProbe?.invoke() ?: transaction { UsersTable.selectAll().empty().not() } + suspend fun hasUsers(): Boolean = DatabaseFactory.blocking { + hasUsersProbe?.invoke() ?: transaction { UsersTable.selectAll().empty().not() } + } + + suspend fun hasAdmin(): Boolean = DatabaseFactory.blocking { hasAdminBlocking() } - fun hasAdmin(): Boolean = hasUsersProbe?.invoke() ?: transaction { + private fun hasAdminBlocking(): Boolean = hasUsersProbe?.invoke() ?: transaction { UsersTable.selectAll().where { UsersTable.role eq "admin" }.empty().not() } companion object { private const val GUEST_TTL_MS = 7 * 24 * 60 * 60 * 1000L + private val passwordDispatcher = Dispatchers.Default.limitedParallelism(2, "password-hashing") fun fixed(userId: String): AuthService = object : AuthService("test") { - override fun verify(token: String): String? = if (token == "test-jwt") userId else null + override suspend fun verify(token: String): String? = if (token == "test-jwt") userId else null } fun fixed(userId: String, hasUsers: Boolean): AuthService = object : AuthService("test", { hasUsers }) { - override fun verify(token: String): String? = if (token == "test-jwt") userId else null + override suspend fun verify(token: String): String? = if (token == "test-jwt") userId else null } } } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt new file mode 100644 index 00000000..063e63ce --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CompletableDeferred +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap + +internal class AuthenticatedSabrInfoCache( + ttl: Duration = Duration.ofMinutes(5), + maxEntries: Int = 256, +) { + private val items = BoundedExpiringCache( + maxEntries = maxEntries, + ttl = ttl, + ) + private val inFlight = ConcurrentHashMap>() + + suspend fun getOrLoad( + credentials: YoutubeSessionCredentials, + videoId: String, + loader: suspend () -> AuthenticatedSabrInfoResult, + ): AuthenticatedSabrInfoResult { + val key = Key(credentials.userId, credentials.fingerprint, videoId) + items.get(key)?.let { return AuthenticatedSabrInfoResult.Ready(it) } + val pending = CompletableDeferred() + val existing = inFlight.putIfAbsent(key, pending) + if (existing != null) return existing.await() + return try { + val result = loader() + if (result is AuthenticatedSabrInfoResult.Ready) items.put(key, result.prepared) + pending.complete(result) + result + } catch (error: Throwable) { + pending.completeExceptionally(error) + throw error + } finally { + inFlight.remove(key, pending) + } + } + + private data class Key( + val userId: String, + val credentialFingerprint: String, + val videoId: String, + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt new file mode 100644 index 00000000..b31ddb9f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt @@ -0,0 +1,90 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe +import org.slf4j.LoggerFactory + +internal class AuthenticatedSabrInfoService( + private val youtubeSessionService: YoutubeSessionService, + private val tokenClient: TypetypeTokenSabrTokenClient, + private val visitorDataFetcher: () -> String = AuthenticatedYoutubeVisitorData::fetch, + private val probe: AuthenticatedSabrProbe = PipePipeAuthenticatedSabrProbe, + private val cache: AuthenticatedSabrInfoCache = AuthenticatedSabrInfoCache(), +) { + suspend fun fetch(userId: String?, videoId: String): AuthenticatedSabrInfoResult { + if (userId == null || userId.startsWith("guest:")) return AuthenticatedSabrInfoResult.NotConnected + val credentials = youtubeSessionService.connectedCredentials(userId) + ?: return AuthenticatedSabrInfoResult.NotConnected + return cache.getOrLoad(credentials, videoId) { fetchUncached(credentials, videoId) } + } + + private suspend fun fetchUncached( + credentials: YoutubeSessionCredentials, + videoId: String, + ): AuthenticatedSabrInfoResult { + return try { + val prepared = YoutubeSessionTokenScope.withCredentials(credentials) { + withContext(Dispatchers.IO) { + val sessionBinding = visitorDataFetcher() + val token = tokenClient.fetchSession(videoId, sessionBinding) + ?: error("Token service did not return authenticated SABR tokens") + val info = probe.fetch(videoId, token.youtubeSessionPoToken()) + SabrPreparedInfo( + info = info, + initialToken = token, + source = SabrPreparedSource.AUTHENTICATED_YOUTUBE, + ).takeIf(SabrPreparedInfo::hasAudioAndVideoFormats) + ?: error("Authenticated SABR response has no audio and video formats") + } + } + youtubeSessionService.markUsed(credentials.userId) + AuthenticatedSabrInfoResult.Ready(prepared) + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + logger.warn( + "authenticated_sabr_probe event=failed videoId={} errorType={} error={}", + videoId, + error.javaClass.simpleName, + error.message, + ) + AuthenticatedSabrInfoResult.Failed + } + } + + private companion object { + val logger = LoggerFactory.getLogger(AuthenticatedSabrInfoService::class.java) + } +} + +internal sealed interface AuthenticatedSabrInfoResult { + data object NotConnected : AuthenticatedSabrInfoResult + data object Failed : AuthenticatedSabrInfoResult + data class Ready(val prepared: SabrPreparedInfo) : AuthenticatedSabrInfoResult +} + +internal fun interface AuthenticatedSabrProbe { + fun fetch(videoId: String, token: YoutubeSessionPoToken): YoutubeSabrInfo +} + +private object PipePipeAuthenticatedSabrProbe : AuthenticatedSabrProbe { + private val localization = Localization("en", "US") + private val contentCountry = ContentCountry("US") + + override fun fetch(videoId: String, token: YoutubeSessionPoToken): YoutubeSabrInfo = + YoutubeSabrProbe.fetchSabrInfo( + videoId, + YoutubeSabrClientProfile.WEB, + localization, + contentCountry, + token.poToken, + token.visitorData, + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt b/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt new file mode 100644 index 00000000..2c2d55db --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt @@ -0,0 +1,27 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper + +internal object AuthenticatedYoutubeVisitorData { + fun fetch( + localization: Localization = Localization("en", "US"), + contentCountry: ContentCountry = ContentCountry("US"), + ): String { + val headers = HashMap>() + YoutubeParsingHelper.addYoutubeHeaders(headers) + headers["Content-Type"] = listOf("application/json") + YoutubeParsingHelper.addLoggedInHeaders(headers) + return YoutubeParsingHelper.getVisitorDataFromInnertube( + InnertubeClientRequestInfo.ofWebClient(), + localization, + contentCountry, + headers, + YoutubeParsingHelper.YOUTUBEI_V1_URL, + null, + false, + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt b/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt index 93ab24a8..b6cc04b7 100644 --- a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt +++ b/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt @@ -17,10 +17,12 @@ object NewPipeInitializer { val normalizedUrl = tokenServiceUrl?.trim()?.takeIf { it.isNotBlank() } if (normalizedUrl != null && normalizedUrl != decoderServiceUrl) { YoutubeApiDecoder.setLocalDecoder(TypetypeTokenYoutubeJavaScriptDecoder(normalizedUrl)) - NewPipe.setYoutubeSessionPoTokenProvider( + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider( TypetypeTokenYoutubeSessionPoTokenProvider(normalizedUrl), ) decoderServiceUrl = normalizedUrl + } else if (normalizedUrl == null) { + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(null) } if (!initialized) { NewPipe.init(OkHttpDownloader.instance(youtubeProxySelector)) diff --git a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt b/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt index 920031e0..d6c50d06 100644 --- a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt @@ -11,7 +11,7 @@ import org.jetbrains.exposed.v1.jdbc.update import java.util.UUID class OidcUserService(private val authService: AuthService) { - fun login(identity: OidcIdentity): AuthSessionTokens { + suspend fun login(identity: OidcIdentity): AuthSessionTokens { val userId = transaction { resolveUserId(identity) } return authService.issueSession(userId) ?: throw IllegalStateException("Failed to create session") } diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt index 9378fb54..4ee60d97 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt @@ -16,12 +16,14 @@ import java.util.UUID class PipePipeBackupPersisterService { suspend fun persist(userId: String, snapshot: PipePipeBackupSnapshotItem): PipePipeBackupRestoreResult = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) clearUserData(userId) val avatarsByChannel = snapshot.subscriptions .mapNotNull { item -> item.url.takeIf { it.isNotBlank() }?.let { url -> url to item.avatarUrl } } .toMap() val history = insertHistory(userId, snapshot.history, avatarsByChannel) val subscriptions = insertSubscriptions(userId, snapshot.subscriptions) + SubscriptionGroupMembershipCleaner.retain(userId, snapshot.subscriptions.map { it.url }) val (playlists, playlistVideos) = insertPlaylists(userId, snapshot.playlists) val progress = insertProgress(userId, snapshot.progress) val searchHistory = insertSearchHistory(userId, snapshot.searchHistory) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt b/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt index 6024be9d..15c3ed98 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt @@ -17,7 +17,9 @@ internal class SabrDownloadStreamer( private val pumpTimeoutMs: Long = PUMP_TIMEOUT_MS, ) { private val localization = Localization("en", "US") - private val unauthorizedRecovery = SabrUnauthorizedResponseRecovery(store::refreshVideoPoToken) + private val unauthorizedRecovery = SabrUnauthorizedResponseRecovery { holder -> + store.refreshVideoPoToken(holder.key.videoId) + } suspend fun stream( holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt b/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt index 6d6f8e0f..2b690d87 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt @@ -8,8 +8,8 @@ internal inline fun withLiveContinuationRequestShape( block: () -> T, ): T { val ranges = buildList { - holder.observedRange(holder.audioFormat)?.takeIf { holder.isAudioActive() }?.let(::add) - holder.observedRange(holder.videoFormat)?.takeIf { holder.isVideoActive() }?.let(::add) + holder.continuationRange(holder.audioFormat)?.takeIf { holder.isAudioActive() }?.let(::add) + holder.continuationRange(holder.videoFormat)?.takeIf { holder.isVideoActive() }?.let(::add) } if (ranges.isEmpty()) return block() val state = holder.session.streamState @@ -24,20 +24,18 @@ internal inline fun withLiveContinuationRequestShape( } } -private fun SabrSessionHolder.observedRange(format: YoutubeSabrFormat): SabrBufferedRange? { - val header = observedMediaSegment(format)?.header ?: return null - val sequence = header.sequenceNumber.takeIf { it > 0 } ?: return null - val startMs = header.startMs.takeIf { it >= 0L } ?: return null - val durationMs = header.durationMs.takeIf { it > 0L } - ?: playbackSegmentDurationMs(format, sequence) - val bufferedEndMs = startMs + durationMs +private fun SabrSessionHolder.continuationRange(format: YoutubeSabrFormat): SabrBufferedRange? { + observedMediaSegment(format) ?: return null + val sequence = lastServedSequence(format) + ?: (playbackStartSequence(format, requestedSeekTimeMs() ?: playerTimeMs()) - 1).coerceAtLeast(0) + val bufferedEndMs = playbackSegmentEndMs(format, sequence).coerceAtLeast(1L) return SabrBufferedRange( format.itag, format.lastModified, format.xtags, 0L, bufferedEndMs, - 1, + if (sequence > 0) 1 else 0, sequence, TIMESCALE, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index 4d21ed59..f01c902e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -75,8 +75,16 @@ private fun SabrSessionHolder.availableLiveMediaStartMs(): Long? { internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false - livePlaybackSnapshot()?.takeIf { it.active } ?: return false + val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false if (session.getCachedSegment(request) != null) return false + if (request.format.itag == videoFormat.itag && + live.headSequence > 0L && + request.sequenceNumber.toLong() < live.headSequence + ) return false + if (observedMediaSegment(request.format) != null && + playbackSegmentEndMs(request.format, request.sequenceNumber) < + live.headTimeMs - LIVE_HISTORICAL_REQUEST_TOLERANCE_MS + ) return false if (session.getReadableSegment(request) != null && !isHistoricalLiveRequest(request)) return true val state = session.streamState observedMediaSegment(request.format)?.let { observed -> @@ -100,13 +108,13 @@ internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): internal fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false + val observed = observedMediaSegment(request.format) ?: return false + val requestEndMs = playbackSegmentEndMs(request.format, request.sequenceNumber) + if (requestEndMs < live.headTimeMs - LIVE_HISTORICAL_REQUEST_TOLERANCE_MS) return true lastServedSequence(request.format)?.let { lastServed -> if (request.sequenceNumber in lastServed..lastServed + LIVE_FUTURE_SEGMENT_TOLERANCE) return false } - val observed = observedMediaSegment(request.format) ?: return false - if (request.sequenceNumber < observed.header.sequenceNumber) return true - val requestEndMs = playbackSegmentEndMs(request.format, request.sequenceNumber) - return requestEndMs < live.headTimeMs - LIVE_HISTORICAL_REQUEST_TOLERANCE_MS + return request.sequenceNumber < observed.header.sequenceNumber } internal fun SabrSessionHolder.liveRetryAfterMs(blockedRequests: List = emptyList()): Long = diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt new file mode 100644 index 00000000..4486e836 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt @@ -0,0 +1,30 @@ +package dev.typetype.server.services + +internal class SabrPlaybackInfoResolver( + private val sessionStore: SabrSessionStore, + private val authenticatedInfoService: AuthenticatedSabrInfoService?, +) { + suspend fun initial( + userId: String?, + videoId: String, + startTimeMs: Long, + ): SabrPreparedInfo? = when (val authenticated = authenticatedInfoService?.fetch(userId, videoId)) { + is AuthenticatedSabrInfoResult.Ready -> authenticated.prepared + AuthenticatedSabrInfoResult.Failed -> null + AuthenticatedSabrInfoResult.NotConnected, null -> + sessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = true) + } + + suspend fun replacement(holder: SabrSessionHolder, startTimeMs: Long): SabrPreparedInfo? { + if (holder.source == SabrPreparedSource.PUBLIC) { + return sessionStore.fetchInfo(holder.key.videoId, startTimeMs, cachedFirst = true) + } + return when (val authenticated = authenticatedInfoService?.fetch(holder.key.userId, holder.key.videoId)) { + is AuthenticatedSabrInfoResult.Ready -> authenticated.prepared + AuthenticatedSabrInfoResult.Failed, + AuthenticatedSabrInfoResult.NotConnected, + null, + -> null + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index 2bd6732d..b3fb8190 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -29,16 +29,11 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS purpose = SabrSessionPurpose.PLAYBACK, audioOnly = audioOnly, initialGeneration = initialGeneration, + source = prepared.source, ) if (isLive || prepared.isLive) holder.markExpectedLive() if (holder.expectsLive()) { - holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) - holder.session.streamState.setSelectVideoFormatBeforeAudio(!audioOnly) - if (startTimeMs == 0L) { - holder.session.streamState.setPlayerTimeMs(OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS) - holder.session.streamState.setWriteTopLevelPlayerTimeMs(false) - } - sessionStore.ensureWarmed(holder, LIVE_INITIAL_PUMPS) + prepareLive(holder, startTimeMs, audioOnly) } else { val initialization = SabrPlaybackInitializationPreloader.preload( sessionStore, @@ -47,11 +42,16 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS INITIALIZATION_PRELOAD_TIMEOUT_MS, ) if (!initialization.isComplete(audioOnly)) { - val missing = initialization.missingTracks(audioOnly, video.itag, audio.itag) - holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) - holder.setPlayerTimeMs(startTimeMs) - holder.failTerminal(sabrRecoverableFailureMessage("SABR initialization unavailable for $missing")) - return SabrPlaybackPreparation(holder, startTimeMs, ready = false) + if (holder.livePlaybackSnapshot()?.active == true) { + holder.markExpectedLive() + prepareLive(holder, startTimeMs, audioOnly) + } else { + val missing = initialization.missingTracks(audioOnly, video.itag, audio.itag) + holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) + holder.setPlayerTimeMs(startTimeMs) + holder.failTerminal(sabrRecoverableFailureMessage("SABR initialization unavailable for $missing")) + return SabrPlaybackPreparation(holder, startTimeMs, ready = false) + } } } return SabrPlaybackStarter.start( @@ -63,6 +63,16 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS ) } + private suspend fun prepareLive(holder: SabrSessionHolder, startTimeMs: Long, audioOnly: Boolean) { + holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) + holder.session.streamState.setSelectVideoFormatBeforeAudio(!audioOnly) + if (startTimeMs == 0L) { + holder.session.streamState.setPlayerTimeMs(OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS) + holder.session.streamState.setWriteTopLevelPlayerTimeMs(false) + } + sessionStore.ensureWarmed(holder, LIVE_INITIAL_PUMPS) + } + suspend fun seek( source: SabrSessionHolder, prepared: SabrPreparedInfo, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt b/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt index 98de8cd2..23d59c87 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt @@ -7,6 +7,7 @@ internal class SabrPreparedInfo( val initialToken: SabrTokenBundle?, val isLive: Boolean = false, val isLiveContent: Boolean = false, + val source: SabrPreparedSource = SabrPreparedSource.PUBLIC, ) internal fun SabrPreparedInfo.hasAudioAndVideoFormats(): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt b/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt new file mode 100644 index 00000000..93b526e1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.services + +internal enum class SabrPreparedSource { + PUBLIC, + AUTHENTICATED_YOUTUBE, +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt index 3269b6dc..5e6ac047 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt @@ -16,6 +16,7 @@ internal class SabrSessionFactory( sessionToken: String, initialToken: SabrTokenBundle?, initialGeneration: Long, + source: SabrPreparedSource, ): SabrSessionHolder { val provider = TypetypeTokenSabrPoTokenProvider(tokenClient, initialToken) val sessionInfo = if (key.sourceId == null) info else SabrSessionIdentity.fresh(info) @@ -34,6 +35,7 @@ internal class SabrSessionFactory( Instant.now(), initialToken, initialGeneration = initialGeneration, + source = source, ).also { it.setPlayerTimeMs(key.startTimeMs) } } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt index 2d0d7ecb..626e6b2a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt @@ -23,6 +23,7 @@ internal class SabrSessionHolder( @Volatile var playerContextToken: SabrTokenBundle? = null, val pumpMutex: Mutex = Mutex(), initialGeneration: Long = 0L, + val source: SabrPreparedSource = SabrPreparedSource.PUBLIC, ) { private val readerPositions = ConcurrentHashMap() private val lastServedSequences = ConcurrentHashMap() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt index 75f8946d..11280335 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt @@ -9,7 +9,7 @@ import java.time.Instant internal class SabrSessionPump( private val segmentCache: SabrSegmentCache? = null, - refreshPoToken: (String) -> SabrTokenBundle? = { null }, + refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle? = { null }, ) { private val loop = SabrSessionPumpLoop( unauthorizedRecovery = SabrUnauthorizedResponseRecovery(refreshPoToken), @@ -22,7 +22,7 @@ internal class SabrSessionPump( var liveWarmupTarget: SabrLiveWarmupTarget? = null holder.setPlaybackState(SabrPlaybackState.PREPARING) while (pumps < maxPumps && - !isWarmEnough(holder) && + !isWarmEnough(holder, liveWarmupTarget) && (!holder.session.isComplete || holder.expectsLive()) ) { val currentLiveTarget = liveWarmupTarget @@ -56,10 +56,19 @@ internal class SabrSessionPump( holder.setPlaybackState(SabrPlaybackState.IDLE) } - private fun isWarmEnough(holder: SabrSessionHolder): Boolean { + private fun isWarmEnough(holder: SabrSessionHolder, liveTarget: SabrLiveWarmupTarget?): Boolean { val audioObserved = holder.observedMediaSegment(holder.audioFormat) != null val videoObserved = !holder.isVideoActive() || holder.observedMediaSegment(holder.videoFormat) != null - if (holder.expectsLive()) return audioObserved && videoObserved + if (holder.expectsLive()) { + val target = liveTarget ?: return false + val audioStartMs = holder.earliestObservedMediaStartMs(holder.audioFormat) ?: return false + val videoStartMs = if (holder.isVideoActive()) { + holder.earliestObservedMediaStartMs(holder.videoFormat) ?: return false + } else { + audioStartMs + } + return maxOf(audioStartMs, videoStartMs) <= target.timeMs + target.segmentDurationMs + } return bothFormatsKnown(holder) || holder.session.streamState.getMaxSegment(holder.audioFormat) > 0 && holder.session.streamState.getMaxSegment(holder.videoFormat) > 0 diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index a76ea88a..5f666549 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -27,8 +27,10 @@ internal class SabrSessionStore( ) { private val registry = SabrSessionRegistry() private val segmentCache = SabrSegmentCache() - private val pump = SabrSessionPump(segmentCache) { videoId -> - tokenClient.fetch(videoId, refreshVideo = true) + private val pump = SabrSessionPump(segmentCache) { holder -> + val binding = holder.playerContextToken?.sessionBinding + if (binding == null) tokenClient.fetch(holder.key.videoId, refreshVideo = true) + else tokenClient.fetchSession(holder.key.videoId, binding, refreshVideo = true) } private val warmer = SabrPlaybackWarmer() private val infoFetcher = SabrInfoFetcher(tokenClient, sessionClient, sharedCache = initCache) @@ -51,6 +53,7 @@ internal class SabrSessionStore( purpose: SabrSessionPurpose = SabrSessionPurpose.MANIFEST, audioOnly: Boolean = false, initialGeneration: Long = 0L, + source: SabrPreparedSource = SabrPreparedSource.PUBLIC, ): SabrSessionHolder { val sessionToken = SabrSessionTokenGenerator.newToken() val isolatedSourceId = sessionToken.takeIf { @@ -77,6 +80,7 @@ internal class SabrSessionStore( sessionToken, initialToken, initialGeneration, + source, ) val active = registry.put(key, holder) if (active !== holder) return active diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt b/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt index e2be83a9..001f71a0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services import org.json.JSONObject +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo import java.util.Base64 @@ -11,6 +12,8 @@ internal class SabrTokenBundle( val visitorData: String, val videoBoundPoToken: String, val videoBoundPoTokenBytes: ByteArray, + val sessionBinding: String? = null, + val sessionBoundPoToken: String? = null, ) { val visitorPoToken: String = visitorBoundPoToken val visitorPoTokenBytes: ByteArray = visitorBoundPoTokenBytes @@ -33,6 +36,26 @@ internal class SabrTokenBundle( ) }.getOrNull() + fun fromSessionResponse( + videoId: String, + sessionBinding: String, + json: JSONObject, + ): SabrTokenBundle? { + val base = fromResponse(videoId, json) ?: return null + val sessionBoundPoToken = json.optString("sessionBoundPoToken").takeIf(String::isNotBlank) + ?: return null + return SabrTokenBundle( + videoId = base.videoId, + visitorBoundPoToken = base.visitorBoundPoToken, + visitorBoundPoTokenBytes = base.visitorBoundPoTokenBytes, + visitorData = base.visitorData, + videoBoundPoToken = base.videoBoundPoToken, + videoBoundPoTokenBytes = base.videoBoundPoTokenBytes, + sessionBinding = sessionBinding, + sessionBoundPoToken = sessionBoundPoToken, + ) + } + private fun decodeBase64Url(value: String): ByteArray { val padded = value + "=".repeat((4 - value.length % 4) % 4) return Base64.getUrlDecoder().decode(padded) @@ -40,8 +63,14 @@ internal class SabrTokenBundle( } } +internal fun SabrTokenBundle.youtubeSessionPoToken(): YoutubeSessionPoToken = + YoutubeSessionPoToken(sessionBinding ?: visitorData, sessionBoundPoToken ?: visitorBoundPoToken) + internal fun SabrTokenBundle.streamingPoTokenBytesFor(info: YoutubeSabrInfo): ByteArray? = - takeIf { it.videoId == info.videoId && it.visitorData == info.visitorData } + takeIf { + it.videoId == info.videoId && + (it.visitorData == info.visitorData || it.sessionBinding == info.visitorData) + } ?.streamingPoTokenBytes ?.takeIf { it.isNotEmpty() } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt b/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt index 393988a8..8f3d0d12 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt @@ -3,12 +3,12 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException internal class SabrUnauthorizedResponseRecovery( - private val refreshPoToken: (String) -> SabrTokenBundle?, + private val refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle?, ) { fun verify(holder: SabrSessionHolder): Unit { val status = latestUnauthorizedStatus(holder.session.diagnosticTrace) ?: return if (!holder.markUnauthorizedRefreshAttempted()) throw unauthorized(status) - val refreshed = refreshPoToken(holder.key.videoId) ?: throw unauthorized(status) + val refreshed = refreshPoToken(holder) ?: throw unauthorized(status) val token = refreshed.streamingPoTokenBytesFor(holder.info) ?.takeUnless { holder.session.streamState.poToken?.contentEquals(it) == true } ?: throw unauthorized(status) diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index f40a0237..e2087004 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -25,6 +25,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( skipPlaylistAutoplayScreen = this[SettingsTable.skipPlaylistAutoplayScreen], volume = this[SettingsTable.volume], muted = this[SettingsTable.muted], + notificationPopupsEnabled = this[SettingsTable.notificationPopupsEnabled], subtitlesEnabled = this[SettingsTable.subtitlesEnabled], defaultSubtitleLanguage = this[SettingsTable.defaultSubtitleLanguage], defaultAudioLanguage = this[SettingsTable.defaultAudioLanguage], @@ -46,6 +47,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( hideComments = this[SettingsTable.hideComments], hideShorts = this[SettingsTable.hideShorts], hideSubscriptionLiveStreams = this[SettingsTable.hideSubscriptionLiveStreams], + hideMembersOnlyContent = this[SettingsTable.hideMembersOnlyContent], disableWatchHistory = this[SettingsTable.disableWatchHistory], deArrowEnabled = this[SettingsTable.deArrowEnabled], deArrowTitleMode = this[SettingsTable.deArrowTitleMode], @@ -63,6 +65,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.skipPlaylistAutoplayScreen] = settings.skipPlaylistAutoplayScreen this[SettingsTable.volume] = settings.volume this[SettingsTable.muted] = settings.muted + this[SettingsTable.notificationPopupsEnabled] = settings.notificationPopupsEnabled this[SettingsTable.subtitlesEnabled] = settings.subtitlesEnabled this[SettingsTable.defaultSubtitleLanguage] = settings.defaultSubtitleLanguage this[SettingsTable.defaultAudioLanguage] = settings.defaultAudioLanguage @@ -84,6 +87,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.hideComments] = settings.hideComments this[SettingsTable.hideShorts] = settings.hideShorts this[SettingsTable.hideSubscriptionLiveStreams] = settings.hideSubscriptionLiveStreams + this[SettingsTable.hideMembersOnlyContent] = settings.hideMembersOnlyContent this[SettingsTable.disableWatchHistory] = settings.disableWatchHistory this[SettingsTable.deArrowEnabled] = settings.deArrowEnabled this[SettingsTable.deArrowTitleMode] = settings.deArrowTitleMode diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt index 1dffe3c3..d609931b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt @@ -35,9 +35,13 @@ class SettingsService { ?.get(SettingsTable.disableWatchHistory) ?: false } - suspend fun hidesSubscriptionLiveStreams(userId: String): Boolean = DatabaseFactory.query { - SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull() - ?.get(SettingsTable.hideSubscriptionLiveStreams) ?: false + internal suspend fun subscriptionFeedVisibility(userId: String): SubscriptionFeedVisibility = DatabaseFactory.query { + SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull()?.let { + SubscriptionFeedVisibility( + hideLiveStreams = it[SettingsTable.hideSubscriptionLiveStreams], + hideMembersOnlyContent = it[SettingsTable.hideMembersOnlyContent], + ) + } ?: SubscriptionFeedVisibility() } suspend fun getAccessModePolicy(userId: String): AccessModePolicy = DatabaseFactory.query { @@ -50,4 +54,9 @@ class SettingsService { } } +internal data class SubscriptionFeedVisibility( + val hideLiveStreams: Boolean = false, + val hideMembersOnlyContent: Boolean = false, +) + data class AccessModePolicy(val accessMode: String, val adminManaged: Boolean) diff --git a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt b/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt index 2de55302..2be09e73 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt +++ b/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt @@ -31,8 +31,11 @@ internal object StreamExtractionErrorMapper { sanitize(error.message) ?: "This premiere has not started yet", "scheduled_premiere", ) + is AgeRestrictedContentException -> ExtractionResult.BadRequest( + sanitize(error.message) ?: "This video is age-restricted", + "age_restricted", + ) is GeographicRestrictionException, - is AgeRestrictedContentException, is PrivateContentException -> ExtractionResult.BadRequest(sanitize(error.message) ?: "Content not available") else -> ExtractionResult.Failure( sanitize(error.message) ?: fallback, diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index bae327dc..95beed0d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -22,13 +22,28 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic } catch (error: CancellationException) { throw error } catch (_: Throwable) { - SubscriptionSourceResult(emptyList(), successfulSources = 0, failedSources = 1) + SubscriptionSourceResult( + channelUrl = subscription.channelUrl, + videos = emptyList(), + successfulSources = 0, + failedSources = 1, + ) } } }.map { it.await() } - val videos = outcomes.flatMap { it.videos }.deduplicated() + val videosByKey = linkedMapOf() + val sourceChannelUrls = linkedMapOf>() + outcomes.forEach { outcome -> + outcome.videos.forEach { video -> + val key = video.subscriptionFeedKey() + val current = videosByKey[key] + if (current == null || video.isLiveContent && !current.isLiveContent) videosByKey[key] = video + sourceChannelUrls.getOrPut(key, ::linkedSetOf).add(outcome.channelUrl) + } + } SubscriptionFeedBuildResult( - videos = videos, + videos = videosByKey.values.toList(), + sourceChannelUrls = sourceChannelUrls.mapValues { it.value.toList() }, successfulSources = outcomes.sumOf { it.successfulSources }, failedSources = outcomes.sumOf { it.failedSources }, ) @@ -42,10 +57,11 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic val videos = if (liveResult == null) { channelResult.videos } else { - channelResult.videos.filterNot(VideoItem::isLive) + liveResult.videos + channelResult.videos.filterNot(VideoItem::isLive) + liveResult.videos.map { it.asLiveContent() } } val results = listOfNotNull(channelResult, liveResult) SubscriptionSourceResult( + channelUrl = channelUrl, videos = mergeVideos(videos), successfulSources = results.count { it.success }, failedSources = results.count { !it.success }, @@ -73,10 +89,12 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic this@deduplicated.forEach { video -> val key = video.subscriptionFeedKey() val current = get(key) - if (current == null || video.isLive && !current.isLive) put(key, video) + if (current == null || video.isLiveContent && !current.isLiveContent) put(key, video) } }.values.toList() + private fun VideoItem.asLiveContent(): VideoItem = if (isLiveContent) this else copy(isLiveContent = true) + private fun String.toLivestreamsTabUrl(): String { val uri = URI(this) val path = uri.path.trimEnd('/') @@ -91,6 +109,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic private data class SourceFetchResult(val videos: List, val success: Boolean) private data class SubscriptionSourceResult( + val channelUrl: String, val videos: List, val successfulSources: Int, val failedSources: Int, @@ -105,6 +124,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic internal data class SubscriptionFeedBuildResult( val videos: List, + val sourceChannelUrls: Map>, val successfulSources: Int, val failedSources: Int, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt index d305a5da..863bbba9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt @@ -9,6 +9,8 @@ object SubscriptionFeedCacheKeys { fun invalidation(userId: String): String = "feed:invalidation:${hash(userId)}" + fun selection(userId: String, slot: Int): String = "feed:selection:${hash(userId)}:$slot" + fun shorts(userId: String): String = "feed:shorts:${hash(userId)}" private fun hash(userId: String): String = MessageDigest diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt index c9cb2f66..a1dc2b9d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt @@ -10,13 +10,13 @@ internal class SubscriptionFeedOrderer { ): SubscriptionFeedOrdering { val previousByKey = previous?.videos.orEmpty().associateBy(VideoItem::subscriptionFeedKey) val promotedAt = buildMap { - videos.filter(VideoItem::isLive).forEach { video -> + videos.filter { it.isLiveOrUpcomingAt(refreshedAt) }.forEach { video -> val key = video.subscriptionFeedKey() val previousVideo = previousByKey[key] val promotion = when { - previousVideo?.isLive == true -> previous?.livePromotedAt?.get(key) - ?: previous?.generatedAt - ?: refreshedAt + previous == null || previousVideo == null -> refreshedAt + samePromotionPhase(video, previousVideo, previous) -> + previous.livePromotedAt[key] ?: previous.generatedAt else -> refreshedAt } put(key, promotion) @@ -30,6 +30,15 @@ internal class SubscriptionFeedOrderer { return SubscriptionFeedOrdering(ordered, promotedAt) } + private fun samePromotionPhase( + video: VideoItem, + previousVideo: VideoItem, + previous: SubscriptionFeedSnapshot, + ): Boolean = when { + video.isLive -> previousVideo.isLive + else -> previousVideo.isUpcomingAt(previous.generatedAt) + } + private fun VideoItem.feedTimestamp(): Long = when { uploaded >= 0L -> uploaded publishedAt != null && publishedAt >= 0L -> publishedAt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt new file mode 100644 index 00000000..b8bc1737 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt @@ -0,0 +1,104 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.cache.CacheService +import kotlinx.serialization.Serializable +import java.security.MessageDigest + +internal class SubscriptionFeedSelectionStore( + private val cache: CacheService, + private val subscriptions: SubscriptionsService, +) { + suspend fun resolve( + userId: String, + selection: SubscriptionSelection, + token: String?, + ): SubscriptionFeedSelectionSnapshot? { + if (selection == SubscriptionSelection.All) return SubscriptionFeedSelectionSnapshot(null, null) + if (token == null) { + val channelUrls = subscriptions.getChannelUrls(userId, selection) + return SubscriptionFeedSelectionSnapshot( + token = tokenFor(selection.cursorKey, channelUrls), + channelUrls = channelUrls, + ) + } + for (slot in 0 until MAX_SNAPSHOTS_PER_USER) { + val stored = read(userId, slot) ?: continue + if (stored.token != token) continue + if (stored.filterKey != selection.cursorKey) return null + return SubscriptionFeedSelectionSnapshot(token, stored.channelUrls.toSet()) + } + return null + } + + suspend fun persist( + userId: String, + selection: SubscriptionSelection, + snapshot: SubscriptionFeedSelectionSnapshot, + ): Boolean { + val token = snapshot.token ?: return true + val channelUrls = snapshot.channelUrls ?: return true + val stored = StoredSubscriptionFeedSelection( + token = token, + filterKey = selection.cursorKey, + channelUrls = channelUrls.sorted(), + ) + val encoded = CacheJson.encodeToString(StoredSubscriptionFeedSelection.serializer(), stored) + for (slot in 0 until MAX_SNAPSHOTS_PER_USER) { + val current = read(userId, slot) + if (current != null) { + if (current == stored) { + val key = SubscriptionFeedCacheKeys.selection(userId, slot) + if (cache.refreshIfValueMatches(key, encoded, SubscriptionFeedSnapshotStore.RETENTION_SECONDS)) { + return true + } + } + continue + } + val key = SubscriptionFeedCacheKeys.selection(userId, slot) + if (cache.setIfAbsent(key, encoded, SubscriptionFeedSnapshotStore.RETENTION_SECONDS)) return true + if (read(userId, slot) == stored) return true + } + return false + } + + private suspend fun read(userId: String, slot: Int): StoredSubscriptionFeedSelection? { + val raw = runCatching { cache.get(SubscriptionFeedCacheKeys.selection(userId, slot)) }.getOrNull() + ?: return null + return runCatching { + CacheJson.decodeFromString(StoredSubscriptionFeedSelection.serializer(), raw) + }.getOrNull() + } + + private fun tokenFor(filterKey: String, channelUrls: Set): String { + val identity = CacheJson.encodeToString( + SubscriptionFeedSelectionIdentity.serializer(), + SubscriptionFeedSelectionIdentity(filterKey, channelUrls.sorted()), + ) + return MessageDigest.getInstance("SHA-256") + .digest(identity.toByteArray()) + .joinToString("") { "%02x".format(it) } + } + + private companion object { + const val MAX_SNAPSHOTS_PER_USER = 8 + } +} + +@Serializable +private data class StoredSubscriptionFeedSelection( + val token: String, + val filterKey: String, + val channelUrls: List, +) + +@Serializable +private data class SubscriptionFeedSelectionIdentity( + val filterKey: String, + val channelUrls: List, +) + +internal data class SubscriptionFeedSelectionSnapshot( + val token: String?, + val channelUrls: Set?, +) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index f641e2ec..7581eb66 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -26,6 +26,7 @@ class SubscriptionFeedService( private val refreshScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), ) { private val store = SubscriptionFeedSnapshotStore(cache, clock) + private val selections = SubscriptionFeedSelectionStore(cache, subscriptionsService) private val builder = SubscriptionFeedBuilder(channelService) private val orderer = SubscriptionFeedOrderer() private val refreshJobs = ConcurrentHashMap() @@ -37,6 +38,8 @@ class SubscriptionFeedService( limit: Int, cursor: String?, hideLiveStreams: Boolean = false, + hideMembersOnlyContent: Boolean = false, + selection: SubscriptionSelection = SubscriptionSelection.All, requestId: String? = currentRequestId(), ): SubscriptionFeedPageResult { val current = store.current(userId) @@ -53,16 +56,40 @@ class SubscriptionFeedService( if (cursorState != null && cursorState.hideLiveStreams != hideLiveStreams) { return SubscriptionFeedPageResult.InvalidCursor } + if (cursorState != null && cursorState.hideMembersOnlyContent != hideMembersOnlyContent) { + return SubscriptionFeedPageResult.InvalidCursor + } + if (cursorState != null && cursorState.filterKey != selection.cursorKey) { + return SubscriptionFeedPageResult.InvalidCursor + } val snapshot = when { cursorState == null -> current cursorState.generation == current.generation -> current else -> store.previous(userId)?.takeIf { it.generation == cursorState.generation } ?: return SubscriptionFeedPageResult.StaleGeneration } + if (selection != SubscriptionSelection.All && !snapshot.hasCompleteSourceAttribution()) { + if (cursorState != null) return SubscriptionFeedPageResult.StaleGeneration + scheduleRefresh(userId, requestId) + return SubscriptionFeedPageResult.Preparing(PREPARING_RETRY_AFTER_MS) + } val offset = cursorState?.offset ?: page * limit - return SubscriptionFeedPageResult.Ready( - snapshot.page(offset, limit, isRefreshing(userId), hideLiveStreams), + val selected = selections.resolve(userId, selection, cursorState?.selectionToken) + ?: return SubscriptionFeedPageResult.StaleGeneration + val response = snapshot.page( + offset, + limit, + isRefreshing(userId), + hideLiveStreams, + hideMembersOnlyContent, + selection, + selected.channelUrls, + selected.token, ) + if (cursorState == null && response.nextpage != null && !selections.persist(userId, selection, selected)) { + return SubscriptionFeedPageResult.CursorCapacityReached + } + return SubscriptionFeedPageResult.Ready(response) } suspend fun getFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse = @@ -148,6 +175,7 @@ class SubscriptionFeedService( stale = false, videos = ordering.videos, livePromotedAt = ordering.livePromotedAt, + sourceChannelUrls = result.sourceChannelUrls, ) runCatching { store.publish(userId, snapshot) }.onFailure { logger.warn("subscription_feed event=publish_failed user={} error={}", userKey(userId), it.message) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index 2f7fd2ec..436d7948 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -13,6 +13,7 @@ internal data class SubscriptionFeedSnapshot( val stale: Boolean, val videos: List, val livePromotedAt: Map = emptyMap(), + val sourceChannelUrls: Map> = emptyMap(), ) @Serializable @@ -21,13 +22,26 @@ private data class SubscriptionFeedCursor( val offset: Int, val limit: Int, val hideLiveStreams: Boolean = false, + val hideMembersOnlyContent: Boolean = false, + val filterKey: String = SubscriptionSelection.All.cursorKey, + val selectionToken: String? = null, ) internal object SubscriptionFeedCursorCodec { - fun encode(generation: Long, offset: Int, limit: Int, hideLiveStreams: Boolean): String { + fun encode( + generation: Long, + offset: Int, + limit: Int, + hideLiveStreams: Boolean, + hideMembersOnlyContent: Boolean, + filterKey: String, + selectionToken: String?, + ): String { val payload = CacheJson.encodeToString( SubscriptionFeedCursor.serializer(), - SubscriptionFeedCursor(generation, offset, limit, hideLiveStreams), + SubscriptionFeedCursor( + generation, offset, limit, hideLiveStreams, hideMembersOnlyContent, filterKey, selectionToken, + ), ) return Base64.getUrlEncoder().withoutPadding().encodeToString(payload.toByteArray()) } @@ -35,9 +49,25 @@ internal object SubscriptionFeedCursorCodec { fun decode(value: String): SubscriptionFeedCursorState? = runCatching { val payload = String(Base64.getUrlDecoder().decode(value)) val cursor = CacheJson.decodeFromString(SubscriptionFeedCursor.serializer(), payload) - cursor.takeIf { it.generation > 0L && it.offset >= 0 && it.limit in 1..100 } - ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit, it.hideLiveStreams) } + cursor.takeIf { + it.generation > 0L && it.offset >= 0 && it.limit in 1..100 && + ((it.filterKey == SubscriptionSelection.All.cursorKey) == (it.selectionToken == null)) && + (it.selectionToken == null || SELECTION_TOKEN.matches(it.selectionToken)) + } + ?.let { + SubscriptionFeedCursorState( + it.generation, + it.offset, + it.limit, + it.hideLiveStreams, + it.hideMembersOnlyContent, + it.filterKey, + it.selectionToken, + ) + } }.getOrNull() + + private val SELECTION_TOKEN = Regex("[0-9a-f]{64}") } internal data class SubscriptionFeedCursorState( @@ -45,28 +75,45 @@ internal data class SubscriptionFeedCursorState( val offset: Int, val limit: Int, val hideLiveStreams: Boolean, + val hideMembersOnlyContent: Boolean, + val filterKey: String, + val selectionToken: String?, ) +internal fun SubscriptionFeedSnapshot.hasCompleteSourceAttribution(): Boolean = + videos.all { sourceChannelUrls.containsKey(it.subscriptionFeedKey()) } + internal fun SubscriptionFeedSnapshot.page( offset: Int, limit: Int, refreshing: Boolean, hideLiveStreams: Boolean = false, + hideMembersOnlyContent: Boolean = false, + selection: SubscriptionSelection = SubscriptionSelection.All, + selectedChannelUrls: Set? = null, + selectionToken: String? = null, ): SubscriptionFeedResponse { - val visibleVideos = if (hideLiveStreams) { - videos.filterNot { it.isLiveOrUpcomingAt(generatedAt) } - } else { - videos + val projectedVideos = projectedVideos(selection, selectedChannelUrls).filterNot { video -> + (hideLiveStreams && video.isLiveContentOrUpcomingAt(generatedAt)) || + (hideMembersOnlyContent && video.requiresMembership) } - val from = offset.coerceAtMost(visibleVideos.size) - val to = minOf(from + limit, visibleVideos.size) - val nextpage = if (to < visibleVideos.size) { - SubscriptionFeedCursorCodec.encode(generation, to, limit, hideLiveStreams) + val from = offset.coerceAtMost(projectedVideos.size) + val to = minOf(from + limit, projectedVideos.size) + val nextpage = if (to < projectedVideos.size) { + SubscriptionFeedCursorCodec.encode( + generation, + to, + limit, + hideLiveStreams, + hideMembersOnlyContent, + selection.cursorKey, + selectionToken, + ) } else { null } return SubscriptionFeedResponse( - videos = visibleVideos.subList(from, to), + videos = projectedVideos.subList(from, to), nextpage = nextpage, generation = generation, generatedAt = generatedAt, @@ -74,9 +121,27 @@ internal fun SubscriptionFeedSnapshot.page( ) } +private fun SubscriptionFeedSnapshot.projectedVideos( + selection: SubscriptionSelection, + selectedChannelUrls: Set?, +): List { + if (selection == SubscriptionSelection.All) return videos + val allowed = selectedChannelUrls.orEmpty() + if (allowed.isEmpty()) return emptyList() + return videos.filter { video -> + val sources = sourceChannelUrls[video.subscriptionFeedKey()] + if (sources != null) { + sources.any { ChannelUrlCanonicalizer.canonicalize(it) in allowed } + } else { + ChannelUrlCanonicalizer.canonicalize(video.uploaderUrl) in allowed + } + } +} + internal sealed interface SubscriptionFeedPageResult { data class Ready(val response: SubscriptionFeedResponse) : SubscriptionFeedPageResult data class Preparing(val retryAfterMs: Long) : SubscriptionFeedPageResult data object InvalidCursor : SubscriptionFeedPageResult data object StaleGeneration : SubscriptionFeedPageResult + data object CursorCapacityReached : SubscriptionFeedPageResult } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt new file mode 100644 index 00000000..3df60b25 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt @@ -0,0 +1,70 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.models.SubscriptionGroupBackupItem +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.selectAll +import java.util.Locale +import java.util.UUID + +internal object SubscriptionGroupBackupRepository { + suspend fun export( + userId: String, + subscriptionUrls: Set, + ): List = DatabaseFactory.query { + val channelsByGroup = SubscriptionGroupMembershipsTable.selectAll() + .where { SubscriptionGroupMembershipsTable.userId eq userId } + .orderBy(SubscriptionGroupMembershipsTable.addedAt to SortOrder.ASC) + .filter { + ChannelUrlCanonicalizer.canonicalize(it[SubscriptionGroupMembershipsTable.channelUrl]) in subscriptionUrls + } + .groupBy( + keySelector = { it[SubscriptionGroupMembershipsTable.groupId] }, + valueTransform = { it[SubscriptionGroupMembershipsTable.channelUrl] }, + ) + SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.ASC) + .map { row -> + SubscriptionGroupBackupItem( + name = row[SubscriptionGroupsTable.name], + channelUrls = channelsByGroup[row[SubscriptionGroupsTable.id]].orEmpty(), + createdAt = row[SubscriptionGroupsTable.createdAt], + updatedAt = row[SubscriptionGroupsTable.updatedAt], + ) + } + } + + fun restore(userId: String, items: List): Pair { + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } + SubscriptionGroupsTable.deleteWhere { SubscriptionGroupsTable.userId eq userId } + val groups = items.map { it to UUID.randomUUID().toString() } + if (groups.isNotEmpty()) { + SubscriptionGroupsTable.batchInsert(groups, shouldReturnGeneratedValues = false) { (item, id) -> + this[SubscriptionGroupsTable.id] = id + this[SubscriptionGroupsTable.userId] = userId + this[SubscriptionGroupsTable.name] = item.name + this[SubscriptionGroupsTable.normalizedName] = item.name.lowercase(Locale.ROOT) + this[SubscriptionGroupsTable.createdAt] = item.createdAt + this[SubscriptionGroupsTable.updatedAt] = item.updatedAt + } + } + val memberships = groups.flatMap { (item, groupId) -> + item.channelUrls.map { channelUrl -> groupId to ChannelUrlCanonicalizer.canonicalize(channelUrl) } + } + if (memberships.isNotEmpty()) { + SubscriptionGroupMembershipsTable.batchInsert(memberships, shouldReturnGeneratedValues = false) { (groupId, channelUrl) -> + this[SubscriptionGroupMembershipsTable.groupId] = groupId + this[SubscriptionGroupMembershipsTable.userId] = userId + this[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + this[SubscriptionGroupMembershipsTable.addedAt] = System.currentTimeMillis() + } + } + return groups.size to memberships.size + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt new file mode 100644 index 00000000..d78f7833 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt @@ -0,0 +1,19 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.notInList +import org.jetbrains.exposed.v1.jdbc.deleteWhere + +internal object SubscriptionGroupMembershipCleaner { + fun retain(userId: String, channelUrls: Collection) { + val retained = channelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) + SubscriptionGroupMembershipsTable.deleteWhere { + val ownedByUser = SubscriptionGroupMembershipsTable.userId eq userId + if (retained.isEmpty()) ownedByUser else { + ownedByUser and (SubscriptionGroupMembershipsTable.channelUrl notInList retained) + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt new file mode 100644 index 00000000..6c19f560 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.SubscriptionGroupItem + +sealed interface SubscriptionGroupWriteResult { + data class Success(val group: SubscriptionGroupItem) : SubscriptionGroupWriteResult + data object InvalidName : SubscriptionGroupWriteResult + data object DuplicateName : SubscriptionGroupWriteResult + data object NotFound : SubscriptionGroupWriteResult +} + +sealed interface SubscriptionGroupMembershipResult { + data object Success : SubscriptionGroupMembershipResult + data object GroupNotFound : SubscriptionGroupMembershipResult + data object SubscriptionNotFound : SubscriptionGroupMembershipResult + data object MembershipNotFound : SubscriptionGroupMembershipResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt new file mode 100644 index 00000000..9529ad45 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -0,0 +1,187 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionGroupItem +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update +import java.sql.SQLException +import java.util.Locale +import java.util.UUID + +class SubscriptionGroupsService { + suspend fun getAll(userId: String): List = DatabaseFactory.query { + val counts = SubscriptionGroupMembershipsTable.selectAll() + .where { SubscriptionGroupMembershipsTable.userId eq userId } + .groupingBy { it[SubscriptionGroupMembershipsTable.groupId] } + .eachCount() + SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.DESC) + .map { it.toItem(counts[it[SubscriptionGroupsTable.id]] ?: 0) } + } + + suspend fun exists(userId: String, groupId: String): Boolean = DatabaseFactory.query { + groupExists(userId, groupId) + } + + suspend fun create(userId: String, rawName: String): SubscriptionGroupWriteResult { + val name = normalizeDisplayName(rawName) ?: return SubscriptionGroupWriteResult.InvalidName + val normalizedName = normalizeUniqueName(name) + return DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) + if (nameExists(userId, normalizedName)) return@query SubscriptionGroupWriteResult.DuplicateName + val id = UUID.randomUUID().toString() + val now = System.currentTimeMillis() + val inserted = SubscriptionGroupsTable.insertIgnore { + it[SubscriptionGroupsTable.id] = id + it[SubscriptionGroupsTable.userId] = userId + it[SubscriptionGroupsTable.name] = name + it[SubscriptionGroupsTable.normalizedName] = normalizedName + it[createdAt] = now + it[updatedAt] = now + }.insertedCount + if (inserted == 0) SubscriptionGroupWriteResult.DuplicateName else { + SubscriptionGroupWriteResult.Success(SubscriptionGroupItem(id, name, 0, now, now)) + } + } + } + + suspend fun rename(userId: String, groupId: String, rawName: String): SubscriptionGroupWriteResult { + val name = normalizeDisplayName(rawName) ?: return SubscriptionGroupWriteResult.InvalidName + val normalizedName = normalizeUniqueName(name) + return try { + DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) + val current = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }.singleOrNull() ?: return@query SubscriptionGroupWriteResult.NotFound + val duplicate = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalizedName) + }.any { it[SubscriptionGroupsTable.id] != groupId } + if (duplicate) return@query SubscriptionGroupWriteResult.DuplicateName + val now = System.currentTimeMillis() + SubscriptionGroupsTable.update({ + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }) { + it[SubscriptionGroupsTable.name] = name + it[SubscriptionGroupsTable.normalizedName] = normalizedName + it[updatedAt] = now + } + val count = membershipCount(userId, groupId) + SubscriptionGroupWriteResult.Success( + SubscriptionGroupItem(groupId, name, count, current[SubscriptionGroupsTable.createdAt], now), + ) + } + } catch (error: Throwable) { + if (error.isUniqueConstraintViolation()) SubscriptionGroupWriteResult.DuplicateName else throw error + } + } + + suspend fun delete(userId: String, groupId: String): Boolean = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) + if (!groupExists(userId, groupId)) return@query false + SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) + } + SubscriptionGroupsTable.deleteWhere { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + } > 0 + } + + suspend fun addSubscription( + userId: String, + groupId: String, + rawChannelUrl: String, + ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) + if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + val subscriptionExists = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) + }.any() + if (!subscriptionExists) return@query SubscriptionGroupMembershipResult.SubscriptionNotFound + SubscriptionGroupMembershipsTable.insertIgnore { + it[SubscriptionGroupMembershipsTable.groupId] = groupId + it[SubscriptionGroupMembershipsTable.userId] = userId + it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + it[addedAt] = System.currentTimeMillis() + } + SubscriptionGroupMembershipResult.Success + } + + suspend fun removeSubscription( + userId: String, + groupId: String, + rawChannelUrl: String, + ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) + if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + val deleted = SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl eq channelUrl) + } + if (deleted > 0) SubscriptionGroupMembershipResult.Success else { + SubscriptionGroupMembershipResult.MembershipNotFound + } + } + + suspend fun getChannelUrls(userId: String, groupId: String): List = DatabaseFactory.query { + SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) + }.orderBy(SubscriptionGroupMembershipsTable.addedAt to SortOrder.DESC) + .map { it[SubscriptionGroupMembershipsTable.channelUrl] } + } + + private fun groupExists(userId: String, groupId: String): Boolean = + SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }.any() + + private fun nameExists(userId: String, normalizedName: String): Boolean = + SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalizedName) + }.any() + + private fun membershipCount(userId: String, groupId: String): Int = + SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.groupId eq groupId) + }.count().toInt() + + private fun ResultRow.toItem(channelCount: Int): SubscriptionGroupItem = SubscriptionGroupItem( + id = this[SubscriptionGroupsTable.id], + name = this[SubscriptionGroupsTable.name], + channelCount = channelCount, + createdAt = this[SubscriptionGroupsTable.createdAt], + updatedAt = this[SubscriptionGroupsTable.updatedAt], + ) + + private fun normalizeDisplayName(value: String): String? = + value.trim().takeIf { it.length in 1..MAX_GROUP_NAME_LENGTH } + + private fun normalizeUniqueName(value: String): String = value.lowercase(Locale.ROOT) + + private fun Throwable.isUniqueConstraintViolation(): Boolean = generateSequence(this) { it.cause } + .filterIsInstance() + .any { it.sqlState == UNIQUE_VIOLATION_SQL_STATE } + + companion object { + const val MAX_GROUP_NAME_LENGTH = 100 + private const val UNIQUE_VIOLATION_SQL_STATE = "23505" + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt new file mode 100644 index 00000000..8320b1ab --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt @@ -0,0 +1,15 @@ +package dev.typetype.server.services + +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager + +internal object SubscriptionMutationLock { + fun acquire(userId: String) { + val userKey = userId.hashCode() and Int.MAX_VALUE + TransactionManager.current().exec( + "SELECT pg_advisory_xact_lock($LOCK_NAMESPACE, $userKey)", + ) + } + + // Precomputed PostgreSQL hashtext('subscriptions'). + private const val LOCK_NAMESPACE = 720_815_616 +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt new file mode 100644 index 00000000..cbdf5d99 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.services + +sealed interface SubscriptionSelection { + val cursorKey: String + + data object All : SubscriptionSelection { + override val cursorKey: String = "all" + } + + data object Ungrouped : SubscriptionSelection { + override val cursorKey: String = "ungrouped" + } + + data class Group(val id: String) : SubscriptionSelection { + override val cursorKey: String = "group:$id" + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index 887a931e..6e90ab8f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.models.SubscriptionItem import org.jetbrains.exposed.v1.core.ResultRow @@ -13,14 +14,22 @@ import org.jetbrains.exposed.v1.jdbc.selectAll class SubscriptionsService { - suspend fun getAll(userId: String): List = DatabaseFactory.query { + suspend fun getAll( + userId: String, + selection: SubscriptionSelection = SubscriptionSelection.All, + ): List = DatabaseFactory.query { + val selectedUrls = selectedChannelUrls(userId, selection) val items = SubscriptionsTable.selectAll() .where { SubscriptionsTable.userId eq userId } .orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC) .map { it.toItem() } + .filter { selection == SubscriptionSelection.All || it.channelUrl in selectedUrls } SubscriptionAvatarRepairer.repair(userId = userId, items = items) } + suspend fun getChannelUrls(userId: String, selection: SubscriptionSelection): Set = + DatabaseFactory.query { selectedChannelUrls(userId, selection) } + suspend fun add(userId: String, item: SubscriptionItem): SubscriptionItem { val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(item.channelUrl) val now = System.currentTimeMillis() @@ -37,10 +46,36 @@ class SubscriptionsService { } suspend fun delete(userId: String, channelUrl: String): Boolean = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(channelUrl) + SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl eq canonicalUrl) + } SubscriptionsTable.deleteWhere { SubscriptionsTable.channelUrl eq canonicalUrl and (SubscriptionsTable.userId eq userId) } > 0 } + private fun selectedChannelUrls(userId: String, selection: SubscriptionSelection): Set { + val all = SubscriptionsTable.selectAll() + .where { SubscriptionsTable.userId eq userId } + .mapTo(linkedSetOf()) { ChannelUrlCanonicalizer.canonicalize(it[SubscriptionsTable.channelUrl]) } + if (selection == SubscriptionSelection.All) return all + val memberships = SubscriptionGroupMembershipsTable.selectAll().where { + when (selection) { + SubscriptionSelection.All -> SubscriptionGroupMembershipsTable.userId eq userId + SubscriptionSelection.Ungrouped -> SubscriptionGroupMembershipsTable.userId eq userId + is SubscriptionSelection.Group -> + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.groupId eq selection.id) + } + }.mapTo(mutableSetOf()) { it[SubscriptionGroupMembershipsTable.channelUrl] } + return when (selection) { + SubscriptionSelection.All -> all + SubscriptionSelection.Ungrouped -> all - memberships + is SubscriptionSelection.Group -> all intersect memberships + } + } + private fun ResultRow.toItem() = SubscriptionItem( channelUrl = ChannelUrlCanonicalizer.canonicalize(this[SubscriptionsTable.channelUrl]), name = this[SubscriptionsTable.name], diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt index f8f5b715..7106218c 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt @@ -22,6 +22,7 @@ internal object TypeTypeBackupCoreRestore { this[SubscriptionsTable.avatarUrl] = item.avatarUrl this[SubscriptionsTable.subscribedAt] = item.subscribedAt } + SubscriptionGroupMembershipCleaner.retain(userId, items.map(SubscriptionItem::channelUrl)) return items.size } diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt index ad10644a..b8209cb9 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt @@ -10,12 +10,18 @@ internal object TypeTypeBackupRestoreWriter { backup: TypeTypeBackupItem, categories: Set, ): TypeTypeRestoreSummary = DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) val restored = linkedMapOf() if (TypeTypeBackupCategory.SUBSCRIPTIONS in categories) { restored["subscriptions"] = TypeTypeBackupCoreRestore.subscriptions( userId, requireNotNull(backup.subscriptions), ) + backup.subscriptionGroups?.let { groups -> + val counts = SubscriptionGroupBackupRepository.restore(userId, groups) + restored["subscriptionGroups"] = counts.first + restored["subscriptionGroupMemberships"] = counts.second + } } if (TypeTypeBackupCategory.HISTORY in categories) { restored["history"] = TypeTypeBackupCoreRestore.history( diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt index 1c98ddfb..fb341ae2 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt @@ -5,6 +5,7 @@ import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION import dev.typetype.server.models.TypeTypeBackupItem import dev.typetype.server.models.TypeTypeContentFiltersBackup import dev.typetype.server.models.TypeTypeRestoreSummary +import java.util.Locale class TypeTypeBackupService( private val subscriptions: SubscriptionsService, @@ -30,10 +31,21 @@ class TypeTypeBackupService( } else { null } + val subscriptionItems = if (includes(TypeTypeBackupCategory.SUBSCRIPTIONS)) { + subscriptions.getAll(userId) + } else { + null + } return TypeTypeBackupItem( exportedAt = System.currentTimeMillis(), categories = categories.map(TypeTypeBackupCategory::wireName).sorted(), - subscriptions = if (includes(TypeTypeBackupCategory.SUBSCRIPTIONS)) subscriptions.getAll(userId) else null, + subscriptions = subscriptionItems, + subscriptionGroups = subscriptionItems?.let { items -> + val channelUrls = items.mapTo(hashSetOf()) { + ChannelUrlCanonicalizer.canonicalize(it.channelUrl) + } + SubscriptionGroupBackupRepository.export(userId, channelUrls) + }, history = if (includes(TypeTypeBackupCategory.HISTORY)) history.getAll(userId) else null, playlists = fullPlaylists, watchLater = if (includes(TypeTypeBackupCategory.WATCH_LATER)) watchLater.getAll(userId) else null, @@ -53,6 +65,7 @@ class TypeTypeBackupService( val categories = TypeTypeBackupCategory.parse(backup.categories.joinToString(",")) ?: throw IllegalArgumentException("Invalid backup categories") validateSections(backup, categories) + validateSubscriptionGroups(backup, categories) validateContentFilters(backup, categories) return TypeTypeBackupRestoreWriter.restore(userId, backup, categories) } @@ -87,6 +100,36 @@ private fun validateSections( require(missing.isEmpty()) { "Backup is missing selected data" } } +private fun validateSubscriptionGroups( + backup: TypeTypeBackupItem, + categories: Set, +) { + val groups = backup.subscriptionGroups ?: return + require(TypeTypeBackupCategory.SUBSCRIPTIONS in categories) { + "Subscription groups require the subscriptions category" + } + val normalizedNames = groups.map { group -> + require(group.name == group.name.trim() && group.name.length in 1..SubscriptionGroupsService.MAX_GROUP_NAME_LENGTH) { + "Subscription group names must contain 1 to 100 characters" + } + group.name.lowercase(Locale.ROOT) + } + require(normalizedNames.distinct().size == normalizedNames.size) { + "Backup contains duplicate subscription group names" + } + val subscriptions = requireNotNull(backup.subscriptions) + .mapTo(mutableSetOf()) { ChannelUrlCanonicalizer.canonicalize(it.channelUrl) } + groups.forEach { group -> + val channels = group.channelUrls.map(ChannelUrlCanonicalizer::canonicalize) + require(channels.distinct().size == channels.size) { + "Backup contains duplicate subscription group memberships" + } + require(channels.all { it in subscriptions }) { + "Subscription group membership references an unknown subscription" + } + } +} + private fun validateContentFilters( backup: TypeTypeBackupItem, categories: Set, diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt index a9447a42..6daab550 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt @@ -1,7 +1,9 @@ package dev.typetype.server.services import okhttp3.OkHttpClient +import okhttp3.MediaType.Companion.toMediaType import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody import org.json.JSONObject import java.net.URLEncoder import java.nio.charset.StandardCharsets @@ -18,6 +20,34 @@ internal class TypetypeTokenSabrTokenClient( fun fetchBoundToken(binding: String): String? = fetch(binding, forceRefresh = false, refreshVideo = false, logIdentifier = false)?.videoBoundPoToken + fun fetchSession( + videoId: String, + sessionBinding: String, + refreshVideo: Boolean = false, + ): SabrTokenBundle? { + val body = JSONObject() + .put("videoId", videoId) + .put("sessionBinding", sessionBinding) + .put("refreshVideo", refreshVideo) + .toString() + .toRequestBody(JSON_MEDIA_TYPE) + val request = Request.Builder() + .url("${tokenServiceUrl.trimEnd('/')}/potoken/session") + .post(body) + .build() + return try { + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) return null + SabrTokenBundle.fromSessionResponse(videoId, sessionBinding, JSONObject(response.body.string())) + } + } catch (error: Exception) { + System.err.println( + "[TypetypeTokenSabrTokenClient] authenticated token fetch failed: ${error.message}", + ) + null + } + } + private fun fetch( binding: String, forceRefresh: Boolean, @@ -50,4 +80,8 @@ internal class TypetypeTokenSabrTokenClient( (if (forceRefresh) "&refresh=true" else "") + (if (refreshVideo) "&refreshVideo=true" else "") } + + private companion object { + val JSON_MEDIA_TYPE = "application/json".toMediaType() + } } diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt index abb8ca67..15b63405 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt @@ -3,8 +3,6 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.ServiceList import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider import java.security.MessageDigest @@ -20,7 +18,7 @@ internal class TypetypeTokenYoutubeSessionPoTokenProvider( TypetypeTokenSabrTokenClient(tokenServiceUrl).let { client -> { binding -> client.fetchBoundToken(binding) } }, - ::fetchAuthenticatedVisitorData, + AuthenticatedYoutubeVisitorData::fetch, ) @Volatile private var cached: CachedToken? = null @@ -76,23 +74,5 @@ internal class TypetypeTokenYoutubeSessionPoTokenProvider( private companion object { const val TOKEN_TTL_MS = 6L * 60L * 60L * 1000L - fun fetchAuthenticatedVisitorData( - localization: Localization, - contentCountry: ContentCountry, - ): String { - val headers = HashMap>() - YoutubeParsingHelper.addYoutubeHeaders(headers) - headers["Content-Type"] = listOf("application/json") - YoutubeParsingHelper.addLoggedInHeaders(headers) - return YoutubeParsingHelper.getVisitorDataFromInnertube( - InnertubeClientRequestInfo.ofWebClient(), - localization, - contentCountry, - headers, - YoutubeParsingHelper.YOUTUBEI_V1_URL, - null, - false, - ) - } } } diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt index 72bff27b..74351ac3 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt @@ -7,11 +7,19 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvid internal object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenProvider { private val scopedToken = ThreadLocal() + @Volatile private var authenticatedProvider: YoutubeSessionPoTokenProvider? = null + + fun configureAuthenticatedProvider(provider: YoutubeSessionPoTokenProvider?): Unit { + authenticatedProvider = provider + } fun withToken(token: SabrTokenBundle, block: () -> T): T { + return withToken(token.youtubeSessionPoToken(), block) + } + + fun withToken(token: YoutubeSessionPoToken, block: () -> T): T { val previous = scopedToken.get() - val sessionToken = YoutubeSessionPoToken(token.visitorData, token.visitorBoundPoToken) - scopedToken.set(sessionToken) + scopedToken.set(token) return try { block() } finally { @@ -27,4 +35,12 @@ internal object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenPro contentCountry: ContentCountry, loggedIn: Boolean, ): YoutubeSessionPoToken? = scopedToken.get() + ?: authenticatedProvider?.getSessionPoToken( + clientName, + clientVersion, + userAgent, + localization, + contentCountry, + loggedIn, + ) } diff --git a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt index 977b3cd9..e8f1837f 100644 --- a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt +++ b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt @@ -3,6 +3,9 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem internal fun VideoItem.isUpcomingAt(now: Long): Boolean = - !isPostLive && duration < 0 && RssVideoMetadata.publishedAtMillis(this) > now + !isPostLive && RssVideoMetadata.publishedAtMillis(this) > now internal fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) + +internal fun VideoItem.isLiveContentOrUpcomingAt(now: Long): Boolean = + isLive || isPostLive || isLiveContent || isUpcomingAt(now) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt index b4ae2798..a5d6d429 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt @@ -45,7 +45,14 @@ class YoutubeRemoteBrowserService( if (!youtubeSessionService.isConfigured) return YoutubeRemoteBrowserCompleteResult.Unavailable val session = sessions.complete(request.sessionId, request.tokenSessionId) ?: return YoutubeRemoteBrowserCompleteResult.NotFound - return when (youtubeSessionService.completeRemote(session.userId, request.cookies, request.poToken)) { + return when ( + youtubeSessionService.completeRemote( + session.userId, + request.cookies, + request.poToken, + request.authUser, + ) + ) { YoutubeSessionCompleteResult.Completed -> YoutubeRemoteBrowserCompleteResult.Completed YoutubeSessionCompleteResult.InvalidCode, YoutubeSessionCompleteResult.ExpiredCode, diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt index 510200d5..111a1acc 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt @@ -5,4 +5,5 @@ data class YoutubeSessionCredentials( val fingerprint: String, val cookies: String, val poToken: String, + val authUser: Int = 0, ) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt new file mode 100644 index 00000000..ae313935 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -0,0 +1,32 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse + +internal const val YOUTUBE_SESSION_REQUIRED_CODE = "youtube_session_required" +internal const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access this video" + +internal class YoutubeSessionSabrStreamService( + private val metadataService: YoutubeSessionStreamService, + private val infoService: AuthenticatedSabrInfoService, +) { + suspend fun getStreamInfo(userId: String, url: String): ExtractionResult? { + val metadata = metadataService.getStreamInfo(userId, url) ?: return null + if (metadata !is ExtractionResult.Success) return metadata + val videoId = youtubeVideoId(url) ?: return ExtractionResult.BadRequest("Invalid YouTube URL") + return when (val info = infoService.fetch(userId, videoId)) { + is AuthenticatedSabrInfoResult.Ready -> + ExtractionResult.Success(metadata.data.withSabrFallback(videoId, info.prepared.info)) + AuthenticatedSabrInfoResult.Failed -> + ExtractionResult.Failure("Authenticated SABR playback unavailable") + AuthenticatedSabrInfoResult.NotConnected -> null + } + } +} + +internal fun ExtractionResult.requiresYoutubeSession(): Boolean = + when (this) { + is ExtractionResult.Success -> data.requiresMembership + is ExtractionResult.BadRequest -> code == "age_restricted" || code == "members_only" + is ExtractionResult.Failure -> false + } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt index 43dc4a8e..0575a407 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt @@ -20,28 +20,35 @@ class YoutubeSessionService( val cookies = YoutubeSessionCookieNormalizer.normalize(request.cookies) ?: return YoutubeSessionCompleteResult.InvalidCredentials val poToken = request.poToken.trim() - if (code.isBlank() || !YoutubeSessionCredentialValidator.isValid(cookies, poToken)) { + if (code.isBlank() || !validCredentials(cookies, poToken, request.authUser)) { return YoutubeSessionCompleteResult.InvalidCredentials } return store.complete( code = code, encryptedCookies = crypto.encrypt(cookies), encryptedPoToken = crypto.encrypt(poToken), + authUser = request.authUser, ) } - suspend fun completeRemote(userId: String, rawCookies: String, rawPoToken: String): YoutubeSessionCompleteResult { + suspend fun completeRemote( + userId: String, + rawCookies: String, + rawPoToken: String, + authUser: Int = 0, + ): YoutubeSessionCompleteResult { val crypto = crypto ?: return YoutubeSessionCompleteResult.Unavailable val cookies = YoutubeSessionCookieNormalizer.normalize(rawCookies) ?: return YoutubeSessionCompleteResult.InvalidCredentials val poToken = rawPoToken.trim() - if (!YoutubeSessionCredentialValidator.isValid(cookies, poToken)) { + if (!validCredentials(cookies, poToken, authUser)) { return YoutubeSessionCompleteResult.InvalidCredentials } store.completeForUser( userId = userId, encryptedCookies = crypto.encrypt(cookies), encryptedPoToken = crypto.encrypt(poToken), + authUser = authUser, ) return YoutubeSessionCompleteResult.Completed } @@ -57,9 +64,15 @@ class YoutubeSessionService( val credentials = runCatching { YoutubeSessionCredentials( userId = userId, - fingerprint = PublicCacheKey.of("youtube-session", encrypted.first, encrypted.second), - cookies = crypto.decrypt(encrypted.first), - poToken = crypto.decrypt(encrypted.second), + fingerprint = PublicCacheKey.of( + "youtube-session", + encrypted.cookies, + encrypted.poToken, + encrypted.authUser.toString(), + ), + cookies = crypto.decrypt(encrypted.cookies), + poToken = crypto.decrypt(encrypted.poToken), + authUser = encrypted.authUser, ) }.getOrNull() if (credentials == null) store.markNeedsReconnect(userId) @@ -69,4 +82,11 @@ class YoutubeSessionService( suspend fun markUsed(userId: String): Unit = store.markUsed(userId) suspend fun markNeedsReconnect(userId: String): Unit = store.markNeedsReconnect(userId) + + private fun validCredentials(cookies: String, poToken: String, authUser: Int): Boolean = + authUser in 0..MAX_AUTH_USER && YoutubeSessionCredentialValidator.isValid(cookies, poToken) + + private companion object { + const val MAX_AUTH_USER = 99 + } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt index 2d424534..8e7f7f32 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt @@ -13,7 +13,12 @@ import org.jetbrains.exposed.v1.jdbc.update class YoutubeSessionStore( private val nowMillis: () -> Long = System::currentTimeMillis, ) { - suspend fun complete(code: String, encryptedCookies: String, encryptedPoToken: String): YoutubeSessionCompleteResult { + suspend fun complete( + code: String, + encryptedCookies: String, + encryptedPoToken: String, + authUser: Int, + ): YoutubeSessionCompleteResult { val now = nowMillis() return DatabaseFactory.query { val pairing = YoutubeSessionPairingsTable.selectAll() @@ -23,7 +28,7 @@ class YoutubeSessionStore( YoutubeSessionPairingsTable.deleteWhere { YoutubeSessionPairingsTable.code eq code } return@query YoutubeSessionCompleteResult.ExpiredCode } - upsertSession(pairing[YoutubeSessionPairingsTable.userId], encryptedCookies, encryptedPoToken, now) + upsertSession(pairing[YoutubeSessionPairingsTable.userId], encryptedCookies, encryptedPoToken, authUser, now) YoutubeSessionPairingsTable.deleteWhere { YoutubeSessionPairingsTable.code eq code } YoutubeSessionCompleteResult.Completed } @@ -45,17 +50,28 @@ class YoutubeSessionStore( YoutubeSessionsTable.deleteWhere { YoutubeSessionsTable.userId eq userId } > 0 } - suspend fun completeForUser(userId: String, encryptedCookies: String, encryptedPoToken: String): Unit = + suspend fun completeForUser( + userId: String, + encryptedCookies: String, + encryptedPoToken: String, + authUser: Int, + ): Unit = DatabaseFactory.query { - upsertSession(userId, encryptedCookies, encryptedPoToken, nowMillis()) + upsertSession(userId, encryptedCookies, encryptedPoToken, authUser, nowMillis()) } - suspend fun connectedEncrypted(userId: String): Pair? = DatabaseFactory.query { + suspend fun connectedEncrypted(userId: String): EncryptedYoutubeSessionCredentials? = DatabaseFactory.query { YoutubeSessionsTable.selectAll() .where { YoutubeSessionsTable.userId eq userId } .singleOrNull() ?.takeIf { YoutubeSessionStatus.from(it[YoutubeSessionsTable.status]) == YoutubeSessionStatus.Connected } - ?.let { it[YoutubeSessionsTable.encryptedCookies] to it[YoutubeSessionsTable.encryptedPoToken] } + ?.let { + EncryptedYoutubeSessionCredentials( + cookies = it[YoutubeSessionsTable.encryptedCookies], + poToken = it[YoutubeSessionsTable.encryptedPoToken], + authUser = it[YoutubeSessionsTable.authUser], + ) + } } suspend fun markUsed(userId: String): Unit = DatabaseFactory.query { @@ -71,22 +87,36 @@ class YoutubeSessionStore( } } - private fun upsertSession(userId: String, encryptedCookies: String, encryptedPoToken: String, now: Long) { + private fun upsertSession( + userId: String, + encryptedCookies: String, + encryptedPoToken: String, + authUser: Int, + now: Long, + ) { val updated = YoutubeSessionsTable.update({ YoutubeSessionsTable.userId eq userId }) { it[YoutubeSessionsTable.encryptedCookies] = encryptedCookies it[YoutubeSessionsTable.encryptedPoToken] = encryptedPoToken + it[YoutubeSessionsTable.authUser] = authUser it[status] = YoutubeSessionStatus.Connected.value it[updatedAt] = now it[lastUsedAt] = 0 } - if (updated == 0) insertSession(userId, encryptedCookies, encryptedPoToken, now) + if (updated == 0) insertSession(userId, encryptedCookies, encryptedPoToken, authUser, now) } - private fun insertSession(userId: String, encryptedCookies: String, encryptedPoToken: String, now: Long) { + private fun insertSession( + userId: String, + encryptedCookies: String, + encryptedPoToken: String, + authUser: Int, + now: Long, + ) { YoutubeSessionsTable.insert { it[YoutubeSessionsTable.userId] = userId it[YoutubeSessionsTable.encryptedCookies] = encryptedCookies it[YoutubeSessionsTable.encryptedPoToken] = encryptedPoToken + it[YoutubeSessionsTable.authUser] = authUser it[status] = YoutubeSessionStatus.Connected.value it[createdAt] = now it[updatedAt] = now @@ -94,3 +124,9 @@ class YoutubeSessionStore( } } } + +data class EncryptedYoutubeSessionCredentials( + val cookies: String, + val poToken: String, + val authUser: Int, +) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt index 0c14ab8c..d9ca60b6 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt @@ -36,7 +36,10 @@ class YoutubeSessionStreamService( } if (requiresReconnect(result)) { youtubeSessionService.markNeedsReconnect(credentials.userId) - return ExtractionResult.BadRequest(YOUTUBE_SESSION_RECONNECT_ERROR) + return ExtractionResult.BadRequest( + YOUTUBE_SESSION_RECONNECT_ERROR, + YOUTUBE_SESSION_RECONNECT_CODE, + ) } youtubeSessionService.markUsed(credentials.userId) return result @@ -75,3 +78,5 @@ class YoutubeSessionStreamService( const val AUTHENTICATED_STREAM_MAX_TTL_SECONDS = 900L } } + +internal const val YOUTUBE_SESSION_RECONNECT_CODE = "youtube_session_needs_reconnect" diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt index b6f45dcf..5c0c3c6f 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt @@ -1,5 +1,6 @@ package dev.typetype.server.services +import dev.typetype.server.downloader.YoutubeAuthUserContext import org.schabi.newpipe.extractor.ServiceList import java.util.concurrent.Semaphore import kotlinx.coroutines.Dispatchers @@ -13,18 +14,21 @@ object YoutubeSessionTokenScope { withPermits(PUBLIC_PERMITS) { val youtube = ServiceList.YouTube try { + YoutubeAuthUserContext.set(credentials.authUser) youtube.setTokens(credentials.cookies) youtube.setAdditionalTokens(credentials.poToken) block() } finally { youtube.setTokens("") youtube.setAdditionalTokens("") + YoutubeAuthUserContext.set(null) } } suspend fun withoutCredentials(block: suspend () -> T): T = withPermits(1) { val youtube = ServiceList.YouTube + YoutubeAuthUserContext.set(null) youtube.setTokens("") youtube.setAdditionalTokens("") block() diff --git a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt index 487f90fb..56741b55 100644 --- a/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/AuthServiceCoreTest.kt @@ -4,6 +4,7 @@ import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.services.AuthService import dev.typetype.server.services.AuthSessionConfig +import kotlinx.coroutines.test.runTest import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction @@ -30,7 +31,7 @@ class AuthServiceCoreTest { } @Test - fun `register sets first admin and second user`() { + fun `register sets first admin and second user`() = runTest { val service = AuthService("test-secret") assertFalse(service.hasUsers()) assertFalse(service.hasAdmin()) @@ -63,7 +64,7 @@ class AuthServiceCoreTest { } @Test - fun `login and refresh token keep same user`() { + fun `login and refresh token keep same user`() = runTest { val service = AuthService("test-secret") val registered = service.register("login@test.local", "secret-1", "Login") val expectedUser = service.verify(registered.accessToken) @@ -79,7 +80,7 @@ class AuthServiceCoreTest { } @Test - fun `configured refresh lifetime is stored for new sessions`() { + fun `configured refresh lifetime is stored for new sessions`() = runTest { val before = System.currentTimeMillis() val service = AuthService( "test-secret", @@ -96,7 +97,7 @@ class AuthServiceCoreTest { } @Test - fun `login supports public username identifier`() { + fun `login supports public username identifier`() = runTest { val service = AuthService("test-secret") val session = service.register("username@test.local", "secret-1", "User") val userId = service.verify(session.accessToken) ?: error("missing user id") @@ -111,7 +112,7 @@ class AuthServiceCoreTest { } @Test - fun `guest token verifies and has user role`() { + fun `guest token verifies and has user role`() = runTest { val service = AuthService("test-secret") val guestToken = service.guestLogin() val guestId = service.verify(guestToken) diff --git a/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt b/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt new file mode 100644 index 00000000..d2bb7f4c --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/AuthServiceDispatcherTest.kt @@ -0,0 +1,74 @@ +package dev.typetype.server + +import dev.typetype.server.services.AuthService +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import kotlin.math.max + +class AuthServiceDispatcherTest { + @Test + fun `database work remains available when shared IO is saturated`() { + val ioParallelism = max(64, Runtime.getRuntime().availableProcessors()) + val entered = CountDownLatch(ioParallelism) + val release = CountDownLatch(1) + val service = AuthService("test-secret", hasUsersProbe = { true }) + + runBlocking { + val blockers = List(ioParallelism) { + async(Dispatchers.IO) { + entered.countDown() + release.await() + } + } + try { + assertTrue(entered.await(5, TimeUnit.SECONDS)) + assertTrue(withTimeout(1_000) { service.hasUsers() }) + } finally { + release.countDown() + blockers.awaitAll() + } + } + } + + @Test + fun `slow database work does not block the caller dispatcher`() { + val entered = CountDownLatch(1) + val release = CountDownLatch(1) + val callerDispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() + val service = AuthService("test-secret", hasUsersProbe = { + entered.countDown() + release.await() + false + }) + + try { + runBlocking { + val pending = async(callerDispatcher) { service.hasUsers() } + assertTrue(entered.await(2, TimeUnit.SECONDS)) + + val result = withTimeout(1_000) { + withContext(callerDispatcher) { "responsive" } + } + assertEquals("responsive", result) + + release.countDown() + assertFalse(pending.await()) + } + } finally { + release.countDown() + callerDispatcher.close() + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt b/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt index 292a8aec..0f4c544d 100644 --- a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt @@ -25,7 +25,8 @@ class DownloaderGatewayArtifactProxyTest { val requestedRange = AtomicReference() val upstream = HttpServer.create(InetSocketAddress(0), 0) upstream.createContext("/jobs/test/artifact") { exchange -> - exchange.responseHeaders.add(HttpHeaders.Location, "http://garage:${upstream.address.port}/object") + exchange.responseHeaders.add(HttpHeaders.Location, "http://typetype-garage:${upstream.address.port}/object") + exchange.responseHeaders.add("X-TypeType-Artifact-Proxy", "1") exchange.sendResponseHeaders(302, -1) exchange.close() } @@ -68,7 +69,29 @@ class DownloaderGatewayArtifactProxyTest { } } + @Test + fun `public artifact redirect remains external`() = testApplication { + val upstream = HttpServer.create(InetSocketAddress(0), 0) + upstream.createContext("/jobs/test/artifact") { exchange -> + exchange.responseHeaders.add(HttpHeaders.Location, "https://downloads.example.com/object") + exchange.sendResponseHeaders(302, -1) + exchange.close() + } + upstream.start() + val gateway = DownloaderGatewayService("http://127.0.0.1:${upstream.address.port}") + application { routing { downloaderGatewayRoutes(gateway) } } + val noRedirectClient = createClient { followRedirects = false } + + try { + val response = noRedirectClient.get("/downloader/jobs/test/artifact") + assertEquals(HttpStatusCode.Found, response.status) + assertEquals("https://downloads.example.com/object", response.headers[HttpHeaders.Location]) + } finally { + upstream.stop(0) + } + } + private fun testDns(): Dns = Dns { hostname -> - if (hostname == "garage") listOf(InetAddress.getByName("127.0.0.1")) else Dns.SYSTEM.lookup(hostname) + if (hostname == "typetype-garage") listOf(InetAddress.getByName("127.0.0.1")) else Dns.SYSTEM.lookup(hostname) } } diff --git a/src/test/kotlin/dev/typetype/server/FakeCacheService.kt b/src/test/kotlin/dev/typetype/server/FakeCacheService.kt index ac2319ec..d046ba93 100644 --- a/src/test/kotlin/dev/typetype/server/FakeCacheService.kt +++ b/src/test/kotlin/dev/typetype/server/FakeCacheService.kt @@ -12,6 +12,18 @@ class FakeCacheService : CacheService { values[key] = value } + override suspend fun setIfAbsent(key: String, value: String, ttlSeconds: Long): Boolean = + values.putIfAbsent(key, value) == null + + override suspend fun refreshIfValueMatches(key: String, value: String, ttlSeconds: Long): Boolean { + var matched = false + values.computeIfPresent(key) { _, current -> + matched = current == value + current + } + return matched + } + override suspend fun delete(key: String) { values.remove(key) } @@ -19,4 +31,6 @@ class FakeCacheService : CacheService { fun clear() { values.clear() } + + fun keys(): Set = values.keys.toSet() } diff --git a/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt b/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt index 38a6dca9..28df1516 100644 --- a/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt @@ -2,6 +2,7 @@ package dev.typetype.server import com.sun.net.httpserver.HttpServer import dev.typetype.server.downloader.OkHttpDownloader +import dev.typetype.server.downloader.YoutubeAuthUserContext import dev.typetype.server.downloader.normalizeExtractorUrl import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull @@ -16,6 +17,18 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit class OkHttpDownloaderCoreTest { + @Test + fun `YouTube auth user is limited to InnerTube requests`() { + YoutubeAuthUserContext.set(3) + try { + assertEquals("3", YoutubeAuthUserContext.headerFor("https://www.youtube.com/youtubei/v1/player")) + assertEquals(null, YoutubeAuthUserContext.headerFor("https://www.youtube.com/watch?v=test")) + assertEquals(null, YoutubeAuthUserContext.headerFor("https://example.com/youtubei/v1/player")) + } finally { + YoutubeAuthUserContext.set(null) + } + } + @Test fun `execute maps http response payload`() { val server = server(status = 200, body = "ok") diff --git a/src/test/kotlin/dev/typetype/server/PasswordResetServiceCoreTest.kt b/src/test/kotlin/dev/typetype/server/PasswordResetServiceCoreTest.kt index 98539eb1..8f0af090 100644 --- a/src/test/kotlin/dev/typetype/server/PasswordResetServiceCoreTest.kt +++ b/src/test/kotlin/dev/typetype/server/PasswordResetServiceCoreTest.kt @@ -3,6 +3,7 @@ package dev.typetype.server import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.services.AuthService import dev.typetype.server.services.PasswordResetService +import kotlinx.coroutines.test.runTest import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update @@ -27,7 +28,7 @@ class PasswordResetServiceCoreTest { } @Test - fun `reset password updates credentials and token cannot be reused`() { + fun `reset password updates credentials and token cannot be reused`() = runTest { val auth = AuthService("test-secret") val reset = PasswordResetService() val oldPassword = "secret-1" @@ -43,7 +44,7 @@ class PasswordResetServiceCoreTest { } @Test - fun `expired token is rejected`() { + fun `expired token is rejected`() = runTest { val auth = AuthService("test-secret") val reset = PasswordResetService() val userId = auth.verify(auth.register("expired@test.local", "secret-1", "Expired").accessToken) diff --git a/src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt b/src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt index 01390d34..6c4de0d2 100644 --- a/src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt +++ b/src/test/kotlin/dev/typetype/server/RegistrationSettingsTest.kt @@ -11,6 +11,7 @@ import io.ktor.client.request.post import io.ktor.client.request.setBody import io.ktor.client.statement.bodyAsText import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.contentType import io.ktor.serialization.kotlinx.json.json @@ -79,6 +80,7 @@ class RegistrationSettingsTest { } val response = client.get("/auth/register/status") assertEquals(HttpStatusCode.OK, response.status) + assertEquals("no-store, no-cache, must-revalidate, max-age=0", response.headers[HttpHeaders.CacheControl]) assertEquals("""{"allowRegistration":false,"bootstrapAvailable":true,"localLoginEnabled":true}""", response.bodyAsText()) } diff --git a/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt index cf6891bd..c91360e4 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt @@ -64,6 +64,7 @@ class SettingsPrivacyControlsRoutesTest { "\"hideComments\":false", "\"hideShorts\":false", "\"hideSubscriptionLiveStreams\":false", + "\"hideMembersOnlyContent\":false", ), ) } @@ -89,6 +90,7 @@ class SettingsPrivacyControlsRoutesTest { "\"hideComments\":true", "\"hideShorts\":true", "\"hideSubscriptionLiveStreams\":true", + "\"hideMembersOnlyContent\":true", ), ) } @@ -108,6 +110,6 @@ class SettingsPrivacyControlsRoutesTest { values.forEach { assertTrue(body.contains(it)) } private fun settingsBody(sponsorBlockMode: String = "mark_only"): String = """ - {"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"sponsorBlockMode":"$sponsorBlockMode","hideHomeRecommendations":true,"hideRelatedVideos":true,"hideComments":true,"hideShorts":true,"hideSubscriptionLiveStreams":true} + {"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"sponsorBlockMode":"$sponsorBlockMode","hideHomeRecommendations":true,"hideRelatedVideos":true,"hideComments":true,"hideShorts":true,"hideSubscriptionLiveStreams":true,"hideMembersOnlyContent":true} """.trimIndent() } diff --git a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt index 883e3808..f2fc67f3 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt @@ -59,6 +59,7 @@ class SettingsRoutesTest { val body = response.bodyAsText() assertTrue(body.contains("\"volume\":1.0")) assertTrue(body.contains("\"muted\":false")) + assertTrue(body.contains("\"notificationPopupsEnabled\":true")) assertTrue(body.contains("\"defaultLandingPage\":\"home\"")) assertTrue(body.contains("\"defaultPlaybackSpeed\":1.0")) } @@ -81,11 +82,12 @@ class SettingsRoutesTest { client.put("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - setBody("""{"defaultService":0,"defaultQuality":"720p","defaultLandingPage":"subscriptions","autoplay":false,"volume":0.5,"muted":true}""") + setBody("""{"defaultService":0,"defaultQuality":"720p","defaultLandingPage":"subscriptions","autoplay":false,"volume":0.5,"muted":true,"notificationPopupsEnabled":false}""") } val body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() assertTrue(body.contains("\"volume\":0.5")) assertTrue(body.contains("\"muted\":true")) + assertTrue(body.contains("\"notificationPopupsEnabled\":false")) assertTrue(body.contains("\"defaultQuality\":\"720p\"")) assertTrue(body.contains("\"defaultLandingPage\":\"subscriptions\"")) } diff --git a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt b/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt index dc8c27a3..25db135c 100644 --- a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt @@ -7,6 +7,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException @@ -70,6 +71,15 @@ class StreamExtractionErrorMapperTest { assertEquals(ExtractionResult.BadRequest("private video"), result) } + @Test + fun `maps age restrictions to a stable access code`() { + val result = StreamExtractionErrorMapper.map(AgeRestrictedContentException("Sign in to confirm your age")) + assertEquals( + ExtractionResult.BadRequest("Sign in is required to verify access to this video", "age_restricted"), + result, + ) + } + @Test fun `maps unknown exceptions to failure`() { val result = StreamExtractionErrorMapper.map(IllegalStateException("boom")) diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt index 69794325..d9d6cac6 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt @@ -71,6 +71,46 @@ class StreamRoutesTest { assertTrue(response.bodyAsText().contains("\"code\":\"paid_content\"")) } + @Test + fun `GET restricted YouTube streams asks guests to connect YouTube`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.BadRequest("Sign in to confirm your age", "age_restricted") + application { + install(ContentNegotiation) { json() } + routing { + streamRoutes( + streamService = streamService, + youtubeSessionSabrStreamInfo = { _, _ -> null }, + ) + } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=restricted") + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_required\"")) + } + + @Test + fun `GET members-only metadata asks guests to connect YouTube`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Success(sabrResponse().copy(requiresMembership = true)) + application { + install(ContentNegotiation) { json() } + routing { + streamRoutes( + streamService = streamService, + youtubeSessionSabrStreamInfo = { _, _ -> null }, + ) + } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=members") + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"youtube_session_required\"")) + } + @Test fun `GET sabr streams returns 422 when final response has no playable source`() = testApplication { coEvery { streamService.getStreamInfo(any()) } returns diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionBackupConsistencyTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionBackupConsistencyTest.kt new file mode 100644 index 00000000..02afdec5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionBackupConsistencyTest.kt @@ -0,0 +1,72 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.TypeTypeBackupCategory +import dev.typetype.server.services.TypeTypeBackupService +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionBackupConsistencyTest { + private val subscriptions = SubscriptionsService() + private val groups = SubscriptionGroupsService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `backup stays restorable when a subscription is added between section reads`() = runTest { + val group = (groups.create(SOURCE, "New") as SubscriptionGroupWriteResult.Success).group + val capturedSubscriptions = mockk() + coEvery { capturedSubscriptions.getAll(SOURCE, any()) } coAnswers { + subscriptions.add(SOURCE, SubscriptionItem(CHANNEL_URL, "Channel", "")) + assertEquals( + SubscriptionGroupMembershipResult.Success, + groups.addSubscription(SOURCE, group.id, CHANNEL_URL), + ) + emptyList() + } + val service = backupService(capturedSubscriptions) + + val backup = service.export(SOURCE, setOf(TypeTypeBackupCategory.SUBSCRIPTIONS)) + val restored = service.restore(TARGET, backup) + + assertEquals(emptyList(), backup.subscriptions) + assertEquals(emptyList(), backup.subscriptionGroups?.single()?.channelUrls) + assertEquals(1, restored.restored["subscriptionGroups"]) + assertEquals(0, restored.restored["subscriptionGroupMemberships"]) + } + + private fun backupService(subscriptions: SubscriptionsService) = TypeTypeBackupService( + subscriptions = subscriptions, + history = mockk(), + playlists = mockk(), + watchLater = mockk(), + favorites = mockk(), + progress = mockk(), + searchHistory = mockk(), + savedPlaylists = mockk(), + settings = mockk(), + blocked = mockk(), + allowedChannels = mockk(), + allowedPlaylists = mockk(), + ) +} + +private const val SOURCE = "concurrent-backup-source" +private const val TARGET = "concurrent-backup-target" +private const val CHANNEL_URL = "https://youtube.com/channel/concurrent" diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt index c92392f2..e4d94975 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt @@ -26,6 +26,7 @@ import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import io.mockk.coEvery import io.mockk.mockk +import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue @@ -49,15 +50,30 @@ class SubscriptionFeedLiveVisibilityRoutesTest { coEvery { channelService.getChannel(any(), null) } returns channel( video(4_000L, url = "https://youtube.com/watch?v=live", live = true), video(3_500L, url = "https://youtube.com/watch?v=scheduled").copy( - duration = -1L, + duration = 0L, publishedAt = System.currentTimeMillis() + 86_400_000L, ), + video(3_250L, url = "https://youtube.com/watch?v=replay").copy( + streamType = "post_live_stream", + isPostLive = true, + ), video(3_000L, url = "https://youtube.com/watch?v=normal-1"), video(2_000L, url = "https://youtube.com/watch?v=normal-2"), ) feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) } + @Test + fun `disabled setting keeps every live state and normal videos`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) + feedService.awaitRefresh(TEST_USER_ID) + + val videoIds = readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }.toSet() + assertEquals(setOf("live", "scheduled", "replay", "normal-1", "normal-2"), videoIds) + } + @Test fun `account setting hides live streams before pagination`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription(1)) @@ -87,15 +103,15 @@ class SubscriptionFeedLiveVisibilityRoutesTest { } @Test - fun `hidden live streams do not remove finished recordings`() = withApp { + fun `account setting hides finished live recordings`() = withApp { val channelService = mockk() coEvery { channelService.getChannel(any(), null) } returns channel( - video(4_000L, url = "https://youtube.com/watch?v=live", live = true), video(3_000L, url = "https://youtube.com/watch?v=replay").copy( streamType = "post_live_stream", isPostLive = true, - isLiveContent = true, ), + video(2_500L, url = "https://youtube.com/watch?v=live-content").copy(isLiveContent = true), + video(2_000L, url = "https://youtube.com/watch?v=normal"), ) feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) subscriptionsService.add(TEST_USER_ID, subscription(1)) @@ -104,7 +120,95 @@ class SubscriptionFeedLiveVisibilityRoutesTest { assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) feedService.awaitRefresh(TEST_USER_ID) - assertEquals(listOf("replay"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + + @Test + fun `account setting hides unclassified videos from streams tab`() = withApp { + val channelUrl = "https://www.youtube.com/channel/UC1" + val channelService = mockk() + coEvery { channelService.getChannel(channelUrl, null) } returns channel( + video(3_000L, url = "https://youtube.com/watch?v=normal"), + video(2_000L, url = "https://youtube.com/watch?v=replay"), + ) + coEvery { channelService.getChannel("$channelUrl/streams", null) } returns channel( + video(2_000L, url = "https://youtube.com/watch?v=replay"), + video(1_000L, url = "https://youtube.com/watch?v=scheduled"), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) + subscriptionsService.add(TEST_USER_ID, subscription(channelUrl, "Live channel")) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) + feedService.awaitRefresh(TEST_USER_ID) + + assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + + @Test + fun `cached snapshot keeps live filtering after service reconstruction`() = runBlocking { + val cache = FakeCacheService() + val channelService = mockk() + coEvery { channelService.getChannel(any(), null) } returns channel( + video(4_000L, url = "https://youtube.com/watch?v=live", live = true), + video(3_500L, url = "https://youtube.com/watch?v=scheduled").copy( + duration = 0L, + publishedAt = System.currentTimeMillis() + 86_400_000L, + ), + video(3_000L, url = "https://youtube.com/watch?v=replay").copy( + streamType = "post_live_stream", + isPostLive = true, + ), + video(2_000L, url = "https://youtube.com/watch?v=normal"), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, cache) + subscriptionsService.add(TEST_USER_ID, subscription(1)) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + withApp { + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) + feedService.awaitRefresh(TEST_USER_ID) + assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + + feedService = SubscriptionFeedService(subscriptionsService, channelService, cache) + withApp { + assertEquals(listOf("normal"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + } + + @Test + fun `account setting hides members only videos before pagination`() = withApp { + val channelService = mockk() + coEvery { channelService.getChannel(any(), null) } returns channel( + video(4_000L, url = "https://youtube.com/watch?v=members").copy(requiresMembership = true), + video(3_000L, url = "https://youtube.com/watch?v=public-1"), + video(2_000L, url = "https://youtube.com/watch?v=public-2"), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) + subscriptionsService.add(TEST_USER_ID, subscription(1)) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideMembersOnlyContent = true)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feedService.awaitRefresh(TEST_USER_ID) + val first = readPage(requestFeed(limit = 1)) + val second = readPage(requestFeed(limit = 1, cursor = requireNotNull(first.nextpage))) + + assertEquals(listOf("public-1"), first.videos.map { it.url.substringAfter("v=") }) + assertEquals(listOf("public-2"), second.videos.map { it.url.substringAfter("v=") }) + assertTrue(second.nextpage == null) + } + + @Test + fun `cursor is rejected after members only visibility changes`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feedService.awaitRefresh(TEST_USER_ID) + val cursor = requireNotNull(readPage(requestFeed(limit = 1)).nextpage) + + settingsService.upsert(TEST_USER_ID, SettingsItem(hideMembersOnlyContent = true)) + + assertEquals(HttpStatusCode.BadRequest, requestFeed(limit = 1, cursor = cursor).status) } private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt index 8dd8dd1c..77956590 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt @@ -10,20 +10,31 @@ class SubscriptionFeedOrdererTest { private val orderer = SubscriptionFeedOrderer() @Test - fun `scheduled livestream follows normal chronology`() { - val scheduled = video(2_000L, url = "scheduled") - val recent = video(3_000L, url = "recent") + fun `scheduled livestream is promoted once then newer uploads pass it`() { + val discoveredAt = 1_800_000_000_000L + val scheduled = video(discoveredAt + 86_400_000L, url = "scheduled").copy(duration = 0L) + val first = orderer.order( + listOf(scheduled, video(discoveredAt - 1_000L, url = "existing")), + previous = null, + refreshedAt = discoveredAt, + ) + assertEquals(listOf("scheduled", "existing"), first.videos.map { it.url }) + val previous = snapshot(discoveredAt, first.videos, first.livePromotedAt) - val result = orderer.order(listOf(scheduled, recent), previous = null, refreshedAt = 10_000L) + val result = orderer.order( + listOf(scheduled, video(discoveredAt + 1_000L, url = "recent")), + previous, + refreshedAt = discoveredAt + 2_000L, + ) assertEquals(listOf("recent", "scheduled"), result.videos.map { it.url }) - assertEquals(emptyMap(), result.livePromotedAt) + assertEquals(discoveredAt, result.livePromotedAt["scheduled"]) } @Test fun `scheduled to live transition is promoted once`() { - val scheduled = video(2_000L, url = "live") - val previous = snapshot(5_000L, listOf(scheduled)) + val scheduled = video(20_000L, url = "live").copy(duration = 0L) + val previous = snapshot(5_000L, listOf(scheduled), mapOf("live" to 5_000L)) val live = video(-1L, url = "live", live = true) val promoted = orderer.order(listOf(video(8_000L), live), previous, refreshedAt = 10_000L) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt new file mode 100644 index 00000000..d528297f --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt @@ -0,0 +1,73 @@ +package dev.typetype.server + +import dev.typetype.server.services.SubscriptionFeedSelectionSnapshot +import dev.typetype.server.services.SubscriptionFeedSelectionStore +import dev.typetype.server.services.SubscriptionSelection +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SubscriptionFeedSelectionStoreTest { + @Test + fun `a ninth distinct session cannot evict the first issued cursor`() = runTest { + val cache = FakeCacheService() + val store = SubscriptionFeedSelectionStore(cache, SubscriptionsService()) + val selections = (1..9).map { index -> + val selection = SubscriptionSelection.Group("group-$index") + val snapshot = SubscriptionFeedSelectionSnapshot( + token = index.toString().padStart(64, '0'), + channelUrls = setOf("https://example.com/channel/$index"), + ) + selection to snapshot + } + + selections.take(8).forEach { (selection, snapshot) -> + assertTrue(store.persist(TEST_USER_ID, selection, snapshot)) + } + val (ninthSelection, ninthSnapshot) = selections.last() + assertFalse(store.persist(TEST_USER_ID, ninthSelection, ninthSnapshot)) + + val (firstSelection, firstSnapshot) = selections.first() + val restored = store.resolve(TEST_USER_ID, firstSelection, firstSnapshot.token) + assertNotNull(restored) + assertEquals(firstSnapshot.channelUrls, restored?.channelUrls) + assertEquals(8, cache.keys().count { it.startsWith("feed:selection") }) + } + + @Test + fun `independent store instances cannot overwrite concurrently issued cursors`() = runTest { + val cache = FakeCacheService() + val stores = List(2) { SubscriptionFeedSelectionStore(cache, SubscriptionsService()) } + val selections = List(2) { index -> + val number = index + 1 + val selection = SubscriptionSelection.Group("group-$number") + val snapshot = SubscriptionFeedSelectionSnapshot( + token = number.toString().padStart(64, '0'), + channelUrls = setOf("https://example.com/channel/$number"), + ) + selection to snapshot + } + val start = CompletableDeferred() + val writes = stores.zip(selections).map { (store, pair) -> + async(Dispatchers.Default) { + start.await() + store.persist(TEST_USER_ID, pair.first, pair.second) + } + } + + start.complete(Unit) + + assertTrue(writes.awaitAll().all { it }) + selections.forEachIndexed { index, (selection, snapshot) -> + assertNotNull(stores[index].resolve(TEST_USER_ID, selection, snapshot.token)) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt new file mode 100644 index 00000000..7872dbb6 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt @@ -0,0 +1,218 @@ +package dev.typetype.server + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.SubscriptionFeedResponse +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.subscriptionFeedRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionFeedCacheKeys +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionFeedSnapshot +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupFeedRoutesTest { + private val subscriptions = SubscriptionsService() + private val groups = SubscriptionGroupsService() + private lateinit var feed: SubscriptionFeedService + private lateinit var cache: FakeCacheService + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + cache = FakeCacheService() + feed = SubscriptionFeedService(subscriptions, FakeChannelService(), cache) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionFeedRoutes(feed, auth, groupsService = groups) } + } + block() + } + + @Test + fun `group and ungrouped feeds project one shared global snapshot`() = withApp { + subscriptions.add(TEST_USER_ID, subscription("one")) + subscriptions.add(TEST_USER_ID, subscription("two")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + assertEquals( + SubscriptionGroupMembershipResult.Success, + groups.addSubscription(TEST_USER_ID, group.id, channel("one")), + ) + + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(listOf("${channel("one")}/video"), requestReadyFeed(groupId = group.id).videos.map { it.url }) + assertEquals(listOf("${channel("two")}/video"), requestReadyFeed(ungrouped = true).videos.map { it.url }) + assertEquals(2, requestReadyFeed().videos.size) + } + + @Test + fun `cursor keeps its original group membership across pages`() = withApp { + val channelService = mockk() + coEvery { channelService.getChannel(channel("one"), null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(3_000L, channel = "one", url = "video-one"), + ) + coEvery { channelService.getChannel(channel("two"), null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(2_000L, channel = "two", url = "video-two"), + ) + coEvery { channelService.getChannel(channel("three"), null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(1_000L, channel = "three", url = "video-three"), + ) + feed = SubscriptionFeedService(subscriptions, channelService, cache) + listOf("one", "two", "three").forEach { subscriptions.add(TEST_USER_ID, subscription(it)) } + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, channel("one")) + groups.addSubscription(TEST_USER_ID, group.id, channel("two")) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1, groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + val firstPage = requestReadyFeed(limit = 1, groupId = group.id) + assertEquals(listOf("video-one"), firstPage.videos.map { it.url }) + val repeatedFirstPage = requestReadyFeed(limit = 1, groupId = group.id) + assertEquals(firstPage.nextpage, repeatedFirstPage.nextpage) + assertEquals(1, cache.keys().count { it.startsWith("feed:selection") }) + + groups.removeSubscription(TEST_USER_ID, group.id, channel("two")) + groups.addSubscription(TEST_USER_ID, group.id, channel("three")) + val changedFirstPage = requestReadyFeed(limit = 1, groupId = group.id) + assertNotEquals(firstPage.nextpage, changedFirstPage.nextpage) + assertEquals(2, cache.keys().count { it.startsWith("feed:selection") }) + val secondPage = requestFeed(limit = 1, cursor = requireNotNull(firstPage.nextpage), groupId = group.id) + + assertEquals(HttpStatusCode.OK, secondPage.status) + assertEquals(listOf("video-two"), Json.decodeFromString(secondPage.bodyAsText()).videos.map { it.url }) + } + + @Test + fun `terminal filtered page does not retain a membership snapshot`() = withApp { + subscriptions.add(TEST_USER_ID, subscription("one")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, channel("one")) + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(1, requestReadyFeed(groupId = group.id).videos.size) + + assertTrue(cache.keys().none { it.startsWith("feed:selection") }) + } + + @Test + fun `cursor cannot be reused with another subscription filter`() = withApp { + subscriptions.add(TEST_USER_ID, subscription("one")) + subscriptions.add(TEST_USER_ID, subscription("two")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, channel("one")) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feed.awaitRefresh(TEST_USER_ID) + val cursor = requireNotNull(requestReadyFeed(limit = 1).nextpage) + + val response = requestFeed(limit = 1, cursor = cursor, groupId = group.id) + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("subscription_feed_invalid_cursor")) + } + + @Test + fun `group feed follows the fetched subscription source when uploader url differs`() = withApp { + val sourceUrl = channel("one") + subscriptions.add(TEST_USER_ID, subscription("one")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, sourceUrl) + val channelService = mockk() + coEvery { channelService.getChannel(sourceUrl, null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(1_000L, channel = "different-canonical-uploader"), + ) + feed = SubscriptionFeedService(subscriptions, channelService, FakeCacheService()) + + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(1, requestReadyFeed(groupId = group.id).videos.size) + } + + @Test + fun `group feed refreshes snapshots without source attribution`() = withApp { + val sourceUrl = channel("one") + val video = SubscriptionFeedTestFixtures.video(1_000L, channel = "different-canonical-uploader") + subscriptions.add(TEST_USER_ID, subscription("one")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, sourceUrl) + cache.set( + SubscriptionFeedCacheKeys.feed(TEST_USER_ID), + CacheJson.encodeToString( + SubscriptionFeedSnapshot.serializer(), + SubscriptionFeedSnapshot(1L, System.currentTimeMillis(), stale = false, videos = listOf(video)), + ), + 60, + ) + val channelService = mockk() + coEvery { channelService.getChannel(sourceUrl, null) } returns SubscriptionFeedTestFixtures.channel(video) + feed = SubscriptionFeedService(subscriptions, channelService, cache) + + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(1, requestReadyFeed(groupId = group.id).videos.size) + } + + private suspend fun ApplicationTestBuilder.requestReadyFeed( + limit: Int = 30, + groupId: String? = null, + ungrouped: Boolean = false, + ): SubscriptionFeedResponse { + val response = requestFeed(limit = limit, groupId = groupId, ungrouped = ungrouped) + assertEquals(HttpStatusCode.OK, response.status) + return Json.decodeFromString(response.bodyAsText()) + } + + private suspend fun ApplicationTestBuilder.requestFeed( + limit: Int = 30, + cursor: String? = null, + groupId: String? = null, + ungrouped: Boolean = false, + ): HttpResponse = client.get("/subscriptions/feed") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + parameter("limit", limit) + cursor?.let { parameter("cursor", it) } + groupId?.let { parameter("groupId", it) } + if (ungrouped) parameter("ungrouped", true) + } + + private fun subscription(id: String) = SubscriptionItem(channel(id), id, "") + + private fun channel(id: String) = "https://example.com/channel/$id" +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt new file mode 100644 index 00000000..70dd04b3 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt @@ -0,0 +1,161 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionGroupItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.subscriptionGroupsRoutes +import dev.typetype.server.routes.subscriptionsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.request.post +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupsRoutesTest { + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { + subscriptionGroupsRoutes(groups, auth) + subscriptionsRoutes(subscriptions, auth, groupsService = groups) + } + } + block() + } + + @Test + fun `group routes require authentication`() = withApp { + assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions/groups").status) + } + + @Test + fun `groups can be created listed renamed and deleted`() = withApp { + val create = client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"Work"}""") + } + assertEquals(HttpStatusCode.Created, create.status) + val group = Json.decodeFromString(create.bodyAsText()) + + assertTrue(authorizedGet("/subscriptions/groups").bodyAsText().contains("\"name\":\"Work\"")) + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}") { + authorizeJson() + setBody("""{"name":"Research"}""") + }.status) + assertTrue(authorizedGet("/subscriptions/groups").bodyAsText().contains("\"name\":\"Research\"")) + assertEquals(HttpStatusCode.NoContent, client.delete("/subscriptions/groups/${group.id}") { authorize() }.status) + assertEquals("[]", authorizedGet("/subscriptions/groups").bodyAsText()) + } + + @Test + fun `blank and duplicate group names are rejected`() = withApp { + assertEquals(HttpStatusCode.BadRequest, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":" "}""") + }.status) + assertEquals(HttpStatusCode.Created, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"Work"}""") + }.status) + assertEquals(HttpStatusCode.Conflict, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"work"}""") + }.status) + } + + @Test + fun `membership drives grouped and ungrouped subscription projections`() = withApp { + subscriptions.add(TEST_USER_ID, SubscriptionItem(channel("one"), "One", "")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(channel("two"), "Two", "")) + val group = createGroup("Work") + + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrl":"${channel("one")}"}""") + }.status) + + val grouped = authorizedGet("/subscriptions") { parameter("groupId", group.id) } + assertTrue(grouped.bodyAsText().contains(channel("one"))) + assertTrue(!grouped.bodyAsText().contains(channel("two"))) + val ungrouped = authorizedGet("/subscriptions") { parameter("ungrouped", true) } + assertTrue(!ungrouped.bodyAsText().contains(channel("one"))) + assertTrue(ungrouped.bodyAsText().contains(channel("two"))) + + assertEquals(HttpStatusCode.NoContent, client.delete("/subscriptions/groups/${group.id}/channels") { + authorize() + parameter("url", channel("one")) + }.status) + assertTrue(authorizedGet("/subscriptions") { parameter("ungrouped", true) }.bodyAsText().contains(channel("one"))) + } + + @Test + fun `invalid or inaccessible filters fail explicitly`() = withApp { + assertEquals(HttpStatusCode.BadRequest, authorizedGet("/subscriptions") { + parameter("groupId", "group") + parameter("ungrouped", true) + }.status) + assertEquals(HttpStatusCode.NotFound, authorizedGet("/subscriptions") { + parameter("groupId", "missing") + }.status) + } + + private suspend fun ApplicationTestBuilder.createGroup(name: String): SubscriptionGroupItem { + val response = client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"$name"}""") + } + return Json.decodeFromString(response.bodyAsText()) + } + + private suspend fun ApplicationTestBuilder.authorizedGet( + path: String, + configure: io.ktor.client.request.HttpRequestBuilder.() -> Unit = {}, + ) = client.get(path) { + authorize() + configure() + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() { + header(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorizeJson() { + authorize() + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + } + + private fun channel(id: String) = "https://yt.com/channel/$id" +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt new file mode 100644 index 00000000..4233deae --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -0,0 +1,258 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.TypeTypeBackupItem +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.SubscriptionGroupMembershipCleaner +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection +import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.TypeTypeBackupCategory +import dev.typetype.server.services.TypeTypeBackupRestoreWriter +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.coroutines.yield +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class SubscriptionGroupsServiceTest { + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `group names are normalized unique and account scoped`() = runTest { + val group = groups.create("user-a", " Work ").createdGroup() + + assertEquals("Work", group.name) + assertEquals(SubscriptionGroupWriteResult.DuplicateName, groups.create("user-a", "work")) + assertTrue(groups.create("user-b", "work") is SubscriptionGroupWriteResult.Success) + assertFalse(groups.exists("user-b", group.id)) + assertEquals( + SubscriptionGroupWriteResult.NotFound, + groups.rename("user-b", group.id, "Other"), + ) + groups.create("user-a", "Other") + assertEquals(SubscriptionGroupWriteResult.DuplicateName, groups.rename("user-a", group.id, "OTHER")) + } + + @Test + fun `a subscription can belong to multiple groups while ungrouped stays distinct`() = runTest { + subscriptions.add("user", subscription("one")) + subscriptions.add("user", subscription("two")) + subscriptions.add("user", subscription("three")) + val first = groups.create("user", "First").createdGroup() + val second = groups.create("user", "Second").createdGroup() + + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", first.id, channel("one"))) + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", second.id, channel("one"))) + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", second.id, channel("two"))) + assertEquals(1, groups.getAll("user").first { it.id == first.id }.channelCount) + + assertEquals( + listOf(channel("one")), + subscriptions.getAll("user", SubscriptionSelection.Group(first.id)).map { it.channelUrl }, + ) + assertEquals( + setOf(channel("one"), channel("two")), + subscriptions.getAll("user", SubscriptionSelection.Group(second.id)).map { it.channelUrl }.toSet(), + ) + assertEquals( + listOf(channel("three")), + subscriptions.getAll("user", SubscriptionSelection.Ungrouped).map { it.channelUrl }, + ) + } + + @Test + fun `membership requires both the users group and subscription`() = runTest { + val group = groups.create("user-a", "A").createdGroup() + subscriptions.add("user-b", subscription("shared")) + + assertEquals( + SubscriptionGroupMembershipResult.SubscriptionNotFound, + groups.addSubscription("user-a", group.id, channel("shared")), + ) + assertEquals( + SubscriptionGroupMembershipResult.GroupNotFound, + groups.addSubscription("user-b", group.id, channel("shared")), + ) + } + + @Test + fun `deleting a subscription removes its memberships`() = runTest { + val group = groups.create("user", "Group").createdGroup() + subscriptions.add("user", subscription("one")) + groups.addSubscription("user", group.id, channel("one")) + + assertTrue(subscriptions.delete("user", channel("one"))) + + assertEquals(emptyList(), groups.getChannelUrls("user", group.id)) + } + + @Test + fun `membership assignment deletion and replacement share a user lock`() = runTest { + val userId = "concurrent-user" + val group = groups.create(userId, "Group").createdGroup() + subscriptions.add(userId, subscription("one")) + val lockHeld = CountDownLatch(1) + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + TransactionManager.current().exec(subscriptionLockSql(userId)) + lockHeld.countDown() + check(releaseLock.await(5, TimeUnit.SECONDS)) + } + } + assertTrue(lockHeld.await(5, TimeUnit.SECONDS)) + + val assignment = async(Dispatchers.IO) { + groups.addSubscription(userId, group.id, channel("one")) + } + val deletion = async(Dispatchers.IO) { subscriptions.delete(userId, channel("one")) } + val replacement = async(Dispatchers.IO) { + TypeTypeBackupRestoreWriter.restore( + userId = userId, + backup = TypeTypeBackupItem( + exportedAt = 1, + categories = listOf(TypeTypeBackupCategory.SUBSCRIPTIONS.wireName), + subscriptions = listOf(subscription("one").copy(subscribedAt = 1)), + ), + categories = setOf(TypeTypeBackupCategory.SUBSCRIPTIONS), + ) + } + val allWaited = try { + withContext(Dispatchers.IO) { + withTimeoutOrNull(2_000L) { + var waiting = false + while (!waiting && !(assignment.isCompleted && deletion.isCompleted && replacement.isCompleted)) { + waiting = waitingSubscriptionLocks(userId) >= 3 + if (!waiting) yield() + } + waiting + } ?: false + } + } finally { + releaseLock.countDown() + } + + holder.await() + assignment.await() + assertTrue(deletion.await()) + replacement.await() + assertTrue(allWaited, "all mutations must wait for the same account-scoped lock") + val subscriptionUrls = subscriptions.getAll(userId).mapTo(hashSetOf(), SubscriptionItem::channelUrl) + assertTrue(groups.getChannelUrls(userId, group.id).all { it in subscriptionUrls }) + } + + @Test + fun `group mutations share the account subscription lock`() = runTest { + val userId = "concurrent-group-user" + subscriptions.add(userId, subscription("one")) + val renamedGroup = groups.create(userId, "Rename").createdGroup() + val deletedGroup = groups.create(userId, "Delete").createdGroup() + val membershipGroup = groups.create(userId, "Membership").createdGroup() + groups.addSubscription(userId, membershipGroup.id, channel("one")) + val lockHeld = CountDownLatch(1) + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + TransactionManager.current().exec(subscriptionLockSql(userId)) + lockHeld.countDown() + check(releaseLock.await(5, TimeUnit.SECONDS)) + } + } + assertTrue(lockHeld.await(5, TimeUnit.SECONDS)) + + val creation = async(Dispatchers.IO) { groups.create(userId, "Created") } + val rename = async(Dispatchers.IO) { groups.rename(userId, renamedGroup.id, "Renamed") } + val deletion = async(Dispatchers.IO) { groups.delete(userId, deletedGroup.id) } + val removal = async(Dispatchers.IO) { + groups.removeSubscription(userId, membershipGroup.id, channel("one")) + } + val allWaited = try { + withContext(Dispatchers.IO) { + withTimeoutOrNull(2_000L) { + var waiting = false + while (!waiting && !(creation.isCompleted && rename.isCompleted && deletion.isCompleted && removal.isCompleted)) { + waiting = waitingSubscriptionLocks(userId) >= 4 + if (!waiting) yield() + } + waiting + } ?: false + } + } finally { + releaseLock.countDown() + } + + holder.await() + assertTrue(creation.await() is SubscriptionGroupWriteResult.Success) + assertTrue(rename.await() is SubscriptionGroupWriteResult.Success) + assertTrue(deletion.await()) + assertEquals(SubscriptionGroupMembershipResult.Success, removal.await()) + assertTrue(allWaited, "all group mutations must wait for the account-scoped lock") + } + + @Test + fun `replacement imports retain only memberships for subscriptions still present`() = runTest { + val group = groups.create("user", "Group").createdGroup() + subscriptions.add("user", subscription("one")) + subscriptions.add("user", subscription("two")) + groups.addSubscription("user", group.id, channel("one")) + groups.addSubscription("user", group.id, channel("two")) + + DatabaseFactory.query { SubscriptionGroupMembershipCleaner.retain("user", listOf(channel("one"))) } + + assertEquals(listOf(channel("one")), groups.getChannelUrls("user", group.id)) + } + + private fun SubscriptionGroupWriteResult.createdGroup() = + (this as SubscriptionGroupWriteResult.Success).group + + private fun subscription(id: String) = SubscriptionItem(channel(id), id, "") + + private fun channel(id: String) = "https://yt.com/channel/$id" + + private fun subscriptionLockSql(userId: String): String = + "SELECT pg_advisory_xact_lock($SUBSCRIPTION_LOCK_NAMESPACE, ${subscriptionLockKey(userId)})" + + private suspend fun waitingSubscriptionLocks(userId: String): Int = DatabaseFactory.query { + TransactionManager.current().exec( + """ + SELECT count(*) + FROM pg_locks + WHERE locktype = 'advisory' + AND classid = $SUBSCRIPTION_LOCK_NAMESPACE + AND objid = ${subscriptionLockKey(userId)} + AND NOT granted + """.trimIndent(), + ) { result -> + result.next() + result.getInt(1) + } ?: 0 + } + + private fun subscriptionLockKey(userId: String): Int = userId.hashCode() and Int.MAX_VALUE +} + +// Precomputed PostgreSQL hashtext('subscriptions'); must match SubscriptionMutationLock. +private const val SUBSCRIPTION_LOCK_NAMESPACE = 720_815_616 diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt index d3075356..96097f40 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt @@ -19,6 +19,7 @@ import io.ktor.server.plugins.contentnegotiation.ContentNegotiation import io.ktor.server.routing.routing import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll @@ -41,7 +42,7 @@ class SubscriptionsRoutesTest { private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { application { - install(ContentNegotiation) { json() } + install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } routing { subscriptionsRoutes(service, auth) } } block() @@ -62,14 +63,31 @@ class SubscriptionsRoutesTest { } @Test - fun `POST subscriptions returns 201 and persists item`() = withApp { + fun `POST subscriptions generates and persists the server timestamp`() = withApp { val response = client.post("/subscriptions") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) setBody(itemBody) } assertEquals(HttpStatusCode.Created, response.status) - assertTrue(response.bodyAsText().contains("\"channelUrl\":\"https://yt.com/channel/1\"")) + val created = Json.decodeFromString(response.bodyAsText()) + assertEquals("https://yt.com/channel/1", created.channelUrl) + assertTrue(created.subscribedAt > 1) + assertEquals(created.subscribedAt, service.getAll(TEST_USER_ID).single().subscribedAt) + } + + @Test + fun `POST subscriptions ignores the obsolete client timestamp`() = withApp { + val response = client.post("/subscriptions") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody(itemBody.dropLast(1) + ",\"subscribedAt\":1}") + } + + assertEquals(HttpStatusCode.Created, response.status) + val created = Json.decodeFromString(response.bodyAsText()) + assertTrue(created.subscribedAt > 1) + assertEquals(created.subscribedAt, service.getAll(TEST_USER_ID).single().subscribedAt) } @Test diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index 1a22cc72..be1dc356 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -24,6 +24,8 @@ import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SettingsTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable import dev.typetype.server.db.tables.YoutubeSessionPairingsTable @@ -102,6 +104,8 @@ object TestDatabase { HistoryTable.deleteAll() FavoritesTable.deleteAll() SettingsTable.deleteAll() + SubscriptionGroupMembershipsTable.deleteAll() + SubscriptionGroupsTable.deleteAll() SubscriptionsTable.deleteAll() WatchLaterTable.deleteAll() ProgressTable.deleteAll() diff --git a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt b/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt index 8cce3567..1d9a7d8f 100644 --- a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt @@ -23,6 +23,9 @@ import dev.typetype.server.services.ProgressService import dev.typetype.server.services.SavedPlaylistService import dev.typetype.server.services.SearchHistoryService import dev.typetype.server.services.SettingsService +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionsService import dev.typetype.server.services.TypeTypeBackupCategory import dev.typetype.server.services.TypeTypeBackupService @@ -37,6 +40,7 @@ import org.junit.jupiter.api.Test class TypeTypeBackupServiceTest { private val subscriptions = SubscriptionsService() + private val subscriptionGroups = SubscriptionGroupsService() private val history = HistoryService() private val playlists = PlaylistService() private val watchLater = WatchLaterService() @@ -78,6 +82,13 @@ class TypeTypeBackupServiceTest { @Test fun `full backup restores every user data category`() = runTest { subscriptions.add(SOURCE, SubscriptionItem("https://youtube.com/channel/source", "Source", "avatar")) + val subscriptionGroup = ( + subscriptionGroups.create(SOURCE, "Favorites") as SubscriptionGroupWriteResult.Success + ).group + assertEquals( + SubscriptionGroupMembershipResult.Success, + subscriptionGroups.addSubscription(SOURCE, subscriptionGroup.id, "https://youtube.com/channel/source"), + ) history.addImported(SOURCE, videoHistory()) val playlist = playlists.create(SOURCE, PlaylistItem(name = "Saved videos")) playlists.addVideo(SOURCE, playlist.id, playlistVideo()) @@ -107,6 +118,14 @@ class TypeTypeBackupServiceTest { val result = service.restore(TARGET, backup) assertEquals(1, result.restored["subscriptions"]) + assertEquals(1, result.restored["subscriptionGroups"]) + assertEquals(1, result.restored["subscriptionGroupMemberships"]) + val restoredGroup = subscriptionGroups.getAll(TARGET).single() + assertEquals("Favorites", restoredGroup.name) + assertEquals( + listOf("https://youtube.com/channel/source"), + subscriptionGroups.getChannelUrls(TARGET, restoredGroup.id), + ) assertEquals(1, result.restored["history"]) assertEquals(1, result.restored["playlists"]) assertEquals(1, result.restored["playlistVideos"]) @@ -133,6 +152,31 @@ class TypeTypeBackupServiceTest { assertTrue(backup.history == null) } + @Test + fun `legacy subscription backup without groups preserves compatible memberships`() = runTest { + subscriptions.add(TARGET, SubscriptionItem("https://youtube.com/channel/keep", "Keep", "")) + subscriptions.add(TARGET, SubscriptionItem("https://youtube.com/channel/drop", "Drop", "")) + val group = (subscriptionGroups.create(TARGET, "Existing") as SubscriptionGroupWriteResult.Success).group + subscriptionGroups.addSubscription(TARGET, group.id, "https://youtube.com/channel/keep") + subscriptionGroups.addSubscription(TARGET, group.id, "https://youtube.com/channel/drop") + val legacyBackup = TypeTypeBackupItem( + exportedAt = 1, + categories = listOf(TypeTypeBackupCategory.SUBSCRIPTIONS.wireName), + subscriptions = listOf( + SubscriptionItem("https://youtube.com/channel/keep", "Keep", "", subscribedAt = 1), + ), + ) + + service.restore(TARGET, legacyBackup) + + val preservedGroup = subscriptionGroups.getAll(TARGET).single() + assertEquals(group.id, preservedGroup.id) + assertEquals( + listOf("https://youtube.com/channel/keep"), + subscriptionGroups.getChannelUrls(TARGET, preservedGroup.id), + ) + } + @Test fun `restore rejects empty normalized blocked keywords`() = runTest { val backup = TypeTypeBackupItem( diff --git a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt index 0f01f486..e74fd18f 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt @@ -11,7 +11,7 @@ import dev.typetype.server.services.TypetypeTokenSabrTokenClient import dev.typetype.server.services.TypetypeTokenYoutubeSessionClient import dev.typetype.server.services.YouTubeSubtitleService import dev.typetype.server.services.YoutubePlayerClient -import dev.typetype.server.services.YoutubePlayerClientFallbackStreamService +import dev.typetype.server.services.YoutubePlayerClientStreamService import dev.typetype.server.services.YoutubeSessionCookieNormalizer import dev.typetype.server.services.YoutubeSessionCredentials import dev.typetype.server.services.YoutubeSessionTokenScope @@ -45,15 +45,15 @@ class YoutubeAuthenticatedExtractionProbeTest { fun `authenticated classic extraction receives a session bound player token`() = runBlocking { NewPipeInitializer.init(tokenServiceUrl) val cookies = readCookies() - val credentials = YoutubeSessionCredentials("probe", "probe", cookies, "probe-token") + val credentials = YoutubeSessionCredentials("probe", "probe", cookies, "probe-token", authUser = 1) val pipePipe = PipePipeStreamService( ProbeCache, YouTubeSubtitleService(OkHttpClient(), tokenServiceUrl), BilibiliRelatedService(), ) - val service = YoutubePlayerClientFallbackStreamService( + val service = YoutubePlayerClientStreamService( pipePipe, - listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.VISIONOS), + YoutubePlayerClient.MWEB, ) val result = YoutubeSessionTokenScope.withCredentials(credentials) { diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt index 8e55fda6..82f5413f 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt @@ -102,14 +102,21 @@ class YoutubeRemoteBrowserCompleteRoutesTest { } private fun completeBody(sessionId: String): String = - """{"sessionId":"$sessionId","tokenSessionId":"token-session","status":"completed","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value","capturedAt":123}""" + """{"sessionId":"$sessionId","tokenSessionId":"token-session","status":"completed","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value","authUser":2,"capturedAt":123}""" private suspend fun assertCredentialsAreEncrypted() { val encrypted = DatabaseFactory.query { YoutubeSessionsTable.selectAll().where { YoutubeSessionsTable.userId eq TEST_USER_ID }.single() - .let { it[YoutubeSessionsTable.encryptedCookies] to it[YoutubeSessionsTable.encryptedPoToken] } + .let { + Triple( + it[YoutubeSessionsTable.encryptedCookies], + it[YoutubeSessionsTable.encryptedPoToken], + it[YoutubeSessionsTable.authUser], + ) + } } assertFalse(encrypted.first.contains("secret-cookie")) assertFalse(encrypted.second.contains("secret-pot")) + assertEquals(2, encrypted.third) } } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt index 0f679437..1299da41 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt @@ -67,10 +67,10 @@ class YoutubeSessionRoutesTest { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText()).code - private suspend fun ApplicationTestBuilder.completeSession(code: String) = client.post("/youtube-session/complete") { + private suspend fun ApplicationTestBuilder.completeSession(code: String, authUser: Int = 2) = client.post("/youtube-session/complete") { headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) setBody( - """{"code":"$code","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value"}""", + """{"code":"$code","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value","authUser":$authUser}""", ) } @@ -110,12 +110,22 @@ class YoutubeSessionRoutesTest { assertEquals(HttpStatusCode.Gone, completeSession(code).status) } + @Test + fun `complete rejects an invalid Google account index`() = withApp { + assertEquals(HttpStatusCode.BadRequest, completeSession(pairingCode(), authUser = 100).status) + } + private suspend fun assertCredentialsAreEncrypted() { val encrypted = DatabaseFactory.query { val row = YoutubeSessionsTable.selectAll().where { YoutubeSessionsTable.userId eq TEST_USER_ID }.single() - row[YoutubeSessionsTable.encryptedCookies] to row[YoutubeSessionsTable.encryptedPoToken] + Triple( + row[YoutubeSessionsTable.encryptedCookies], + row[YoutubeSessionsTable.encryptedPoToken], + row[YoutubeSessionsTable.authUser], + ) } assertFalse(encrypted.first.contains("secret-cookie")) assertFalse(encrypted.second.contains("secret-pot")) + assertEquals(2, encrypted.third) } } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt index bcaa17d8..9049f2ff 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt @@ -11,6 +11,7 @@ import dev.typetype.server.services.YoutubeSessionCompleteResult import dev.typetype.server.services.YoutubeSessionCrypto import dev.typetype.server.services.YoutubeSessionService import dev.typetype.server.services.YoutubeSessionStreamService +import dev.typetype.server.services.YOUTUBE_SESSION_RECONNECT_CODE import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull @@ -70,6 +71,7 @@ class YoutubeSessionStreamServiceTest { val result = service.getStreamInfo(TEST_USER_ID, "https://youtube.com/watch?v=test") assertTrue(result is ExtractionResult.BadRequest) + assertEquals(YOUTUBE_SESSION_RECONNECT_CODE, (result as ExtractionResult.BadRequest).code) assertEquals("needs_reconnect", youtubeSessionService.status(TEST_USER_ID).status) } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt index 10da62ed..a7e06bc0 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt @@ -1,5 +1,6 @@ package dev.typetype.server +import dev.typetype.server.downloader.YoutubeAuthUserContext import dev.typetype.server.services.YoutubeSessionCredentials import dev.typetype.server.services.YoutubeSessionTokenScope import kotlinx.coroutines.runBlocking @@ -19,14 +20,21 @@ class YoutubeSessionTokenScopeTest { fingerprint = "session-fingerprint", cookies = "SID=session-cookie", poToken = "session-pot-value", + authUser = 2, ) ) { - youtube.tokens to youtube.additionalTokens + Triple( + youtube.tokens, + youtube.additionalTokens, + YoutubeAuthUserContext.headerFor("https://www.youtube.com/youtubei/v1/player"), + ) } assertEquals("SID=session-cookie", observed.first) assertEquals("session-pot-value", observed.second) + assertEquals("2", observed.third) assertEquals("", youtube.tokens.orEmpty()) assertEquals("", youtube.additionalTokens.orEmpty()) + assertEquals(null, YoutubeAuthUserContext.headerFor("https://www.youtube.com/youtubei/v1/player")) } @Test diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt new file mode 100644 index 00000000..dd1e0683 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.services.StreamService +import dev.typetype.server.testStreamResponse +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class SabrPlaybackAccessValidatorTest { + @Test + fun `uses linked YouTube session even when public metadata is accessible`() = runBlocking { + val authenticated = ExtractionResult.Success(testStreamResponse().copy(title = "Authenticated")) + val validator = validator( + publicResult = ExtractionResult.Success(testStreamResponse().copy(title = "Public")), + authenticatedResult = authenticated, + ) + + assertEquals(authenticated, validator.resolve("user-id", "video-id")) + } + + @Test + fun `uses linked YouTube session for age-restricted playback`() = runBlocking { + val authenticated = ExtractionResult.Success(testStreamResponse()) + val validator = validator( + publicResult = ExtractionResult.BadRequest("Confirm your age", "age_restricted"), + authenticatedResult = authenticated, + ) + + assertEquals(authenticated, validator.resolve("user-id", "video-id")) + } + + @Test + fun `asks for YouTube connection when restricted playback has no session`() = runBlocking { + val validator = validator( + publicResult = ExtractionResult.BadRequest("Sign in", "members_only"), + authenticatedResult = null, + ) + + assertEquals( + ExtractionResult.BadRequest("Connect YouTube to access this video", "youtube_session_required"), + validator.resolve(null, "video-id"), + ) + } + + @Test + fun `keeps membership error when linked account lacks access`() = runBlocking { + val membersOnly = ExtractionResult.BadRequest("Join this channel", "members_only") + val validator = validator( + publicResult = ExtractionResult.Success(testStreamResponse().copy(requiresMembership = true)), + authenticatedResult = membersOnly, + ) + + assertEquals(membersOnly, validator.resolve("user-id", "video-id")) + } + + private fun validator( + publicResult: ExtractionResult, + authenticatedResult: ExtractionResult?, + ): SabrPlaybackAccessValidator = SabrPlaybackAccessValidator( + publicStreamService = object : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult = publicResult + }, + youtubeSessionStreamInfo = { _, _ -> authenticatedResult }, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt new file mode 100644 index 00000000..9b6e3449 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt @@ -0,0 +1,163 @@ +package dev.typetype.server.services + +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo + +class AuthenticatedSabrInfoServiceTest { + @Test + fun `connected account uses one authenticated token pair`() = runTest { + val sessions = mockk() + val tokenClient = mockk() + val probe = mockk() + val token = sessionToken() + val info = playableInfo() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) + coEvery { sessions.markUsed(USER_ID) } returns Unit + every { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } returns token + every { probe.fetch(VIDEO_ID, any()) } answers { + val supplied = secondArg() + assertEquals(SESSION_BINDING, supplied.visitorData) + assertEquals(SESSION_PO_TOKEN, supplied.poToken) + info + } + val service = AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = { SESSION_BINDING }, + probe = probe, + ) + + val result = service.fetch(USER_ID, VIDEO_ID) as AuthenticatedSabrInfoResult.Ready + + assertSame(info, result.prepared.info) + assertSame(token, result.prepared.initialToken) + assertEquals(SabrPreparedSource.AUTHENTICATED_YOUTUBE, result.prepared.source) + coVerify(exactly = 1) { sessions.markUsed(USER_ID) } + } + + @Test + fun `reuses authenticated info for the following playback request`() = runTest { + val sessions = mockk() + val tokenClient = mockk() + val probe = mockk() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) + coEvery { sessions.markUsed(USER_ID) } returns Unit + every { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } returns sessionToken() + every { probe.fetch(VIDEO_ID, any()) } returns playableInfo() + val service = AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = { SESSION_BINDING }, + probe = probe, + ) + + val metadata = service.fetch(USER_ID, VIDEO_ID) + val playback = service.fetch(USER_ID, VIDEO_ID) + + assertSame((metadata as AuthenticatedSabrInfoResult.Ready).prepared, (playback as AuthenticatedSabrInfoResult.Ready).prepared) + verify(exactly = 1) { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } + verify(exactly = 1) { probe.fetch(VIDEO_ID, any()) } + coVerify(exactly = 1) { sessions.markUsed(USER_ID) } + } + + @Test + fun `guest playback does not inspect connected credentials`() = runTest { + val sessions = mockk() + val tokenClient = mockk() + val service = AuthenticatedSabrInfoService(sessions, tokenClient) + + val result = service.fetch("guest:anonymous", VIDEO_ID) + + assertEquals(AuthenticatedSabrInfoResult.NotConnected, result) + coVerify(exactly = 0) { sessions.connectedCredentials(any()) } + verify(exactly = 0) { tokenClient.fetchSession(any(), any(), any()) } + } + + @Test + fun `authenticated probe failure is typed and does not mark session used`() = runTest { + val sessions = mockk() + val tokenClient = mockk() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) + every { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } returns null + val service = AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = { SESSION_BINDING }, + ) + + val result = service.fetch(USER_ID, VIDEO_ID) + + assertEquals(AuthenticatedSabrInfoResult.Failed, result) + coVerify(exactly = 0) { sessions.markUsed(any()) } + } + + @Test + fun `cancellation remains observable`() { + val sessions = mockk() + val tokenClient = mockk() + val probe = mockk() + coEvery { sessions.connectedCredentials(USER_ID) } returns credentials(USER_ID) + every { tokenClient.fetchSession(VIDEO_ID, SESSION_BINDING, false) } returns sessionToken() + every { probe.fetch(VIDEO_ID, any()) } throws CancellationException("cancelled") + val service = AuthenticatedSabrInfoService( + sessions, + tokenClient, + visitorDataFetcher = { SESSION_BINDING }, + probe = probe, + ) + + assertThrows(CancellationException::class.java) { + runTest { service.fetch(USER_ID, VIDEO_ID) } + } + } + + private fun playableInfo(): YoutubeSabrInfo = mockk { + every { formats } returns listOf( + mockk { + every { isAudio } returns true + every { isVideo } returns false + }, + mockk { + every { isAudio } returns false + every { isVideo } returns true + }, + ) + } + + private fun credentials(userId: String) = YoutubeSessionCredentials( + userId = userId, + fingerprint = "fingerprint-$userId", + cookies = "SID=session-cookie", + poToken = "session-player-token", + ) + + private fun sessionToken() = SabrTokenBundle( + videoId = VIDEO_ID, + visitorBoundPoToken = "public-session-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = "public-visitor", + videoBoundPoToken = "video-token", + videoBoundPoTokenBytes = byteArrayOf(2), + sessionBinding = SESSION_BINDING, + sessionBoundPoToken = SESSION_PO_TOKEN, + ) + + private companion object { + const val USER_ID = "user-id" + const val VIDEO_ID = "video-id" + const val SESSION_BINDING = "connected-visitor" + const val SESSION_PO_TOKEN = "connected-session-token" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt index 479d317f..aa9c30c9 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt @@ -17,10 +17,12 @@ import java.time.Instant class SabrLiveContinuationRequestTest { @Test - fun `continuation advertises the exact observed live range`() { + fun `continuation advertises only media served to the player`() { val fixture = fixture() fixture.holder.observeMediaSegment(segment(fixture.audio.itag, 10_396, 10_395_000L, -1L)) fixture.holder.observeMediaSegment(segment(fixture.video.itag, 10_396, 10_395_000L, -1L)) + fixture.holder.setLastServedSequence(fixture.audio.itag, 10_392) + fixture.holder.setLastServedSequence(fixture.video.itag, 10_392) fixture.holder.setPlayerTimeMs(10_390_500L) val result = withLiveContinuationRequestShape(fixture.holder) { "pumped" } @@ -28,8 +30,8 @@ class SabrLiveContinuationRequestTest { assertEquals("pumped", result) assertEquals( listOf( - "itag=140:seq=1-10396:time=0+10396000:timescale=1000", - "itag=299:seq=1-10396:time=0+10396000:timescale=1000", + "itag=140:seq=1-10392:time=0+10392000:timescale=1000", + "itag=299:seq=1-10392:time=0+10392000:timescale=1000", ), requireNotNull(fixture.rangeOverrides.first()).map(SabrBufferedRange::summarize), ) @@ -37,6 +39,24 @@ class SabrLiveContinuationRequestTest { verify { fixture.state.setPlayerTimeMs(10_390_500L) } } + @Test + fun `continuation does not advertise an unserved live edge`() { + val fixture = fixture() + fixture.holder.observeMediaSegment(segment(fixture.audio.itag, 10_396, 10_395_000L, -1L)) + fixture.holder.observeMediaSegment(segment(fixture.video.itag, 10_396, 10_395_000L, -1L)) + fixture.holder.setPlayerTimeMs(10_390_500L) + + withLiveContinuationRequestShape(fixture.holder) { Unit } + + assertEquals( + listOf( + "itag=140:seq=1-10390:time=0+10390000:timescale=1000", + "itag=299:seq=1-10390:time=0+10390000:timescale=1000", + ), + requireNotNull(fixture.rangeOverrides.first()).map(SabrBufferedRange::summarize), + ) + } + @Test fun `continuation leaves request state unchanged before media is observed`() { val fixture = fixture() diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt new file mode 100644 index 00000000..9151234e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt @@ -0,0 +1,88 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrLiveFutureRequestTest { + @Test + fun `segment behind the reported live head is not treated as future`() { + val audio = format(140) + val video = format(299) + val state = mockk(relaxed = true) + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { session.liveHeadSequenceNumber } returns 230L + every { session.getCachedSegment(any()) } returns null + every { state.isLive } returns true + every { state.liveHeadSequenceNumber } returns 230L + every { state.liveHeadTimeMs } returns 1_060_000L + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + assertFalse(holder.isFutureLiveRequest(SabrSegmentRequest.media(video, 201))) + } + + @Test + fun `audio segment behind the live time is not treated as future`() { + val audio = format(140) + val video = format(299) + val state = mockk(relaxed = true) + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { session.liveHeadSequenceNumber } returns 3_920L + every { session.getCachedSegment(any()) } returns null + every { state.isLive } returns true + every { state.liveHeadSequenceNumber } returns 3_920L + every { state.liveHeadTimeMs } returns 7_842_000L + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + holder.observeMediaSegment(segment(audio.itag, 3_889, 7_776_000L)) + holder.setLastServedSequence(audio.itag, 3_889) + val request = SabrSegmentRequest.media(audio, 3_890) + + assertFalse(holder.isFutureLiveRequest(request)) + assertTrue(holder.isHistoricalLiveRequest(request)) + } + + private fun format(itag: Int): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + } + + private fun segment(itag: Int, sequence: Int, startMs: Long): SabrMediaSegment { + val header = mockk { + every { isInitSegment } returns false + every { this@mockk.itag } returns itag + every { sequenceNumber } returns sequence + every { this@mockk.startMs } returns startMs + every { durationMs } returns 2_000L + } + return mockk { every { this@mockk.header } returns header } + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt index be3caec6..30e2aa12 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt @@ -17,6 +17,44 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState import java.time.Instant class SabrLiveSessionWarmupTest { + @Test + fun `warmup continues when the first media pair is too close to the live head`() = runTest { + val audio = format(140, audio = true, "audio/mp4") + val video = format(299, audio = false, "video/mp4") + val streamState = mockk(relaxed = true) + val session = mockk() + var pumps = 0 + every { session.streamState } returns streamState + every { session.isComplete } returns false + every { session.isLive } returns true + every { session.liveHeadSequenceNumber } returns 1_000L + every { streamState.isLive } returns true + every { streamState.isPostLiveDvr } returns false + every { streamState.liveHeadSequenceNumber } returns 1_000L + every { streamState.liveHeadTimeMs } returns 2_000_000L + val audioInit = mp4Box("ftyp", byteArrayOf(1)) + mp4Box("moov", byteArrayOf(2)) + val videoInit = mp4Box("ftyp", byteArrayOf(3)) + mp4Box("moov", byteArrayOf(4)) + every { session.pumpOnce(any()) } answers { + pumps++ + val atTarget = pumps > 1 + val sequence = if (atTarget) 990 else 1_000 + val startMs = if (atTarget) 1_980_000L else 2_000_000L + listOf( + segment(140, sequence, startMs, 2_000L, audioInit + mediaFragment(5)), + segment(299, sequence, startMs, 2_000L, videoInit + mediaFragment(6)), + ) + } + val holder = holder(session, audio, video) + holder.markExpectedLive() + + SabrSessionPump(SabrSegmentCache()).ensureWarmed(holder, maxPumps = 8) + + assertEquals(2, pumps) + assertEquals(1_980_000L, holder.earliestObservedMediaStartMs(audio)) + assertEquals(1_980_000L, holder.earliestObservedMediaStartMs(video)) + assertEquals(1_980_000L, holder.resolvePlaybackStartMs(0L)) + } + @Test fun `warmup keeps bootstrap initialization and requests real live media`() = runTest { val audio = format(140, audio = true, "audio/mp4") diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt new file mode 100644 index 00000000..251c237a --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt @@ -0,0 +1,87 @@ +package dev.typetype.server.services + +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class SabrPlaybackInfoResolverTest { + @Test + fun `connected account uses authenticated playback info`() = runTest { + val store = mockk() + val authenticated = mockk() + val prepared = mockk() + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.Ready(prepared) + + val result = SabrPlaybackInfoResolver(store, authenticated).initial(USER_ID, VIDEO_ID, 0L) + + assertSame(prepared, result) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any(), any()) } + } + + @Test + fun `authenticated failure never falls back to public playback info`() = runTest { + val store = mockk() + val authenticated = mockk() + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.Failed + + val result = SabrPlaybackInfoResolver(store, authenticated).initial(USER_ID, VIDEO_ID, 0L) + + assertNull(result) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any(), any()) } + } + + @Test + fun `account without YouTube connection uses public playback info`() = runTest { + val store = mockk() + val authenticated = mockk() + val prepared = mockk() + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.NotConnected + coEvery { store.fetchInfo(VIDEO_ID, 5_000L, true, false) } returns prepared + + val result = SabrPlaybackInfoResolver(store, authenticated).initial(USER_ID, VIDEO_ID, 5_000L) + + assertSame(prepared, result) + } + + @Test + fun `authenticated replacement remains authenticated`() = runTest { + val store = mockk() + val authenticated = mockk() + val prepared = mockk() + val holder = holder(SabrPreparedSource.AUTHENTICATED_YOUTUBE) + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.Ready(prepared) + + val result = SabrPlaybackInfoResolver(store, authenticated).replacement(holder, 60_000L) + + assertSame(prepared, result) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any(), any()) } + } + + @Test + fun `authenticated replacement failure never changes source`() = runTest { + val store = mockk() + val authenticated = mockk() + val holder = holder(SabrPreparedSource.AUTHENTICATED_YOUTUBE) + coEvery { authenticated.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.Failed + + val result = SabrPlaybackInfoResolver(store, authenticated).replacement(holder, 60_000L) + + assertNull(result) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any(), any()) } + } + + private fun holder(source: SabrPreparedSource): SabrSessionHolder = mockk { + every { this@mockk.source } returns source + every { key } returns SabrSessionKey(VIDEO_ID, USER_ID, 140, null, 137, 0L) + } + + private companion object { + const val USER_ID = "user-id" + const val VIDEO_ID = "video-id" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt index ca8ae740..f07ad191 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt @@ -196,6 +196,7 @@ class SabrSeekRepositionPumpTest { every { session.pumpOnceStreamingForDemand(any(), request) } returns mockk(relaxed = true) val holder = holder(session, audio, video) holder.observeMediaSegment(mediaSegment(video.itag, sequence = 3_076)) + holder.setLastServedSequence(video.itag, 3_076) holder.requestSegmentDemand(request) assertFalse(holder.isFutureLiveRequest(request)) assertEquals(DEFAULT_PLAYBACK_RETRY_MS, holder.liveRetryAfterMs(listOf(request))) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt new file mode 100644 index 00000000..0cdbfa61 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt @@ -0,0 +1,92 @@ +package dev.typetype.server.services + +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrTransitioningLivePlaybackTest { + @Test + fun `live protocol response overrides stale ended metadata`() = runTest { + val audio = format(140, isAudio = true) + val video = format(299, isAudio = false) + val info = mockk() + val prepared = SabrPreparedInfo(info, token(), isLive = false, isLiveContent = true) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { session.liveHeadSequenceNumber } returns 5_536L + every { state.isLive } returns true + every { state.isPostLiveDvr } returns false + every { state.liveHeadTimeMs } returns 11_070_200L + every { state.liveHeadSequenceNumber } returns 5_536L + val holder = SabrSessionHolder( + session = session, + info = info, + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + val store = mockk() + every { + store.getOrCreate( + "video", + "user", + info, + audio, + video, + prepared.initialToken, + 0L, + false, + SabrSessionPurpose.PLAYBACK, + false, + 0L, + ) + } returns holder + coEvery { store.fetchInitializationData(holder, video) } returns null + coEvery { store.fetchInitializationData(holder, audio) } returns null + coEvery { store.ensureWarmed(holder, 8) } returns Unit + every { store.startPump(holder) } returns Unit + + val result = SabrPlaybackSessionService(store).prepare("video", "user", prepared, audio, video, 0L) + + assertTrue(holder.expectsLive()) + assertTrue(result.startTimeMs > 0L) + assertNull(holder.terminalFailure()) + assertEquals(11_050_200L, result.startTimeMs) + coVerify(exactly = 1) { store.ensureWarmed(holder, 8) } + verify(exactly = 1) { state.setPlayerTimeMs(9_007_199_254_740_991L) } + verify(exactly = 1) { store.startPump(holder) } + } + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { this@mockk.isAudio } returns isAudio + every { audioTrackId } returns null + every { mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + every { bitrate } returns if (isAudio) 128_000 else 5_000_000 + } + + private fun token(): SabrTokenBundle = SabrTokenBundle( + videoId = "video", + visitorBoundPoToken = "visitor-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = "visitor-data", + videoBoundPoToken = "video-token", + videoBoundPoTokenBytes = byteArrayOf(2), + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt index 54b3a704..6022321e 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt @@ -5,8 +5,10 @@ import io.mockk.mockk import okhttp3.Interceptor import okhttp3.OkHttpClient import okhttp3.Protocol +import okhttp3.Request import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody +import org.json.JSONObject import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -79,16 +81,36 @@ class TypetypeTokenSabrTokenClientTest { assertNull(url.queryParameter("refreshVideo")) } + @Test + fun sessionFetchPostsOneExplicitlyBoundTokenPair(): Unit { + val recorder = PotokenRequestRecorder(SESSION_TOKEN_JSON) + val client = TypetypeTokenSabrTokenClient("https://token.example", recorder.client) + + val token = client.fetchSession("video", "connected-visitor", refreshVideo = true) + + assertNotNull(token) + val request = recorder.requests.single() + assertEquals("POST", request.method) + assertEquals("/potoken/session", request.url.encodedPath) + val body = JSONObject(request.body!!.let { body -> okio.Buffer().also(body::writeTo).readUtf8() }) + assertEquals("video", body.getString("videoId")) + assertEquals("connected-visitor", body.getString("sessionBinding")) + assertEquals(true, body.getBoolean("refreshVideo")) + assertArrayEquals(byteArrayOf(2), token!!.streamingPoTokenBytesFor(info("connected-visitor"))) + assertNull(token.streamingPoTokenBytesFor(info("different-visitor"))) + } + private fun info(expectedVisitorData: String): YoutubeSabrInfo = mockk { every { videoId } returns "video" every { visitorData } returns expectedVisitorData } private class PotokenRequestRecorder(tokenJson: String = TOKEN_JSON) { - val urls = mutableListOf() + val requests = mutableListOf() + val urls: List get() = requests.map { it.url } val client: OkHttpClient = OkHttpClient.Builder() .addInterceptor(Interceptor { chain -> - urls += chain.request().url + requests += chain.request() Response.Builder() .request(chain.request()) .protocol(Protocol.HTTP_1_1) @@ -105,5 +127,7 @@ class TypetypeTokenSabrTokenClientTest { """{"visitorBoundPoToken":"AQ","visitorData":"visitor","videoBoundPoToken":"Ag"}""" const val MISMATCHED_TOKEN_JSON = """{"visitorBoundPoToken":"AQ","visitorData":"other-visitor","videoBoundPoToken":"Ag"}""" + const val SESSION_TOKEN_JSON = + """{"visitorBoundPoToken":"AQ","visitorData":"public","videoBoundPoToken":"Ag","sessionBoundPoToken":"Aw"}""" } } diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt index 7d80673c..3a6a12e4 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt @@ -2,11 +2,18 @@ package dev.typetype.server.services import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.localization.ContentCountry import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider class TypetypeYoutubeSessionPoTokenProviderTest { + @AfterEach + fun clearAuthenticatedProvider(): Unit = + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(null) + @Test fun `exposes the session token only inside its scope`() { TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { @@ -40,6 +47,24 @@ class TypetypeYoutubeSessionPoTokenProviderTest { assertNull(currentToken()) } + @Test + fun `uses the authenticated provider outside a SABR scope`() { + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) + + assertEquals("auth", currentToken()?.visitorData) + assertEquals("auth-token", currentToken()?.poToken) + } + + @Test + fun `prefers the SABR token over the authenticated provider`() { + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) + + TypetypeYoutubeSessionPoTokenProvider.withToken(token("sabr", "sabr-token")) { + assertEquals("sabr", currentToken()?.visitorData) + assertEquals("sabr-token", currentToken()?.poToken) + } + } + private fun currentToken() = TypetypeYoutubeSessionPoTokenProvider.getSessionPoToken( "MWEB", "2.20260801.00.00", @@ -57,4 +82,15 @@ class TypetypeYoutubeSessionPoTokenProviderTest { videoBoundPoToken = "video-token", videoBoundPoTokenBytes = byteArrayOf(2), ) + + private fun provider(visitorData: String, poToken: String) = object : YoutubeSessionPoTokenProvider { + override fun getSessionPoToken( + clientName: String, + clientVersion: String, + userAgent: String?, + localization: Localization, + contentCountry: ContentCountry, + loggedIn: Boolean, + ) = YoutubeSessionPoToken(visitorData, poToken) + } }