diff --git a/app/src/androidTest/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManagerTest.kt b/app/src/androidTest/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManagerTest.kt index b40bbfb5d84..23bdc42f5f0 100644 --- a/app/src/androidTest/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManagerTest.kt +++ b/app/src/androidTest/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManagerTest.kt @@ -5,11 +5,15 @@ import androidx.test.platform.app.InstrumentationRegistry import au.com.shiftyjelly.pocketcasts.analytics.testing.TestEventSink import au.com.shiftyjelly.pocketcasts.models.db.AppDatabase import au.com.shiftyjelly.pocketcasts.models.db.dao.EpisodeDao +import au.com.shiftyjelly.pocketcasts.models.db.dao.TranscriptDao import au.com.shiftyjelly.pocketcasts.models.di.ModelModule import au.com.shiftyjelly.pocketcasts.models.di.addTypeConverters import au.com.shiftyjelly.pocketcasts.models.entity.PodcastEpisode import au.com.shiftyjelly.pocketcasts.models.type.SyncStatus import au.com.shiftyjelly.pocketcasts.preferences.model.BookmarksSortTypeDefault +import au.com.shiftyjelly.pocketcasts.repositories.sync.SyncManager +import au.com.shiftyjelly.pocketcasts.repositories.transcript.TranscriptWindowExtractor +import au.com.shiftyjelly.pocketcasts.servers.podcast.TranscriptService import com.automattic.eventhorizon.BookmarkSourceType import com.automattic.eventhorizon.EventHorizon import com.squareup.moshi.Moshi @@ -25,6 +29,7 @@ import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Before import org.junit.Test +import org.mockito.kotlin.mock class BookmarkManagerTest { private lateinit var appDatabase: AppDatabase @@ -40,6 +45,11 @@ class BookmarkManagerTest { bookmarkManager = BookmarkManagerImpl( appDatabase = appDatabase, eventHorizon = EventHorizon(TestEventSink()), + syncManager = mock(), + transcriptWindowExtractor = TranscriptWindowExtractor( + transcriptDao = mock(), + transcriptService = mock(), + ), ) episodeDao = appDatabase.episodeDao() } diff --git a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManager.kt b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManager.kt index a0c331df44a..9874f409ac7 100644 --- a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManager.kt +++ b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManager.kt @@ -40,6 +40,7 @@ interface BookmarkManager { sortType: BookmarksSortTypeForProfile, ): Flow> fun hasBookmarksFlow(episodeUuid: String): Flow + suspend fun enrichBookmark(bookmark: Bookmark) var sourceView: SourceView } diff --git a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManagerImpl.kt b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManagerImpl.kt index 23409f0b08f..13cce74ff12 100644 --- a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManagerImpl.kt +++ b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/bookmark/BookmarkManagerImpl.kt @@ -9,6 +9,8 @@ import au.com.shiftyjelly.pocketcasts.models.type.SyncStatus import au.com.shiftyjelly.pocketcasts.preferences.model.BookmarksSortTypeDefault import au.com.shiftyjelly.pocketcasts.preferences.model.BookmarksSortTypeForPodcast import au.com.shiftyjelly.pocketcasts.preferences.model.BookmarksSortTypeForProfile +import au.com.shiftyjelly.pocketcasts.repositories.sync.SyncManager +import au.com.shiftyjelly.pocketcasts.repositories.transcript.TranscriptWindowExtractor import com.automattic.eventhorizon.BookmarkCreatedEvent import com.automattic.eventhorizon.BookmarkSourceType import com.automattic.eventhorizon.BookmarkUpdateTitleEvent @@ -18,16 +20,20 @@ import java.util.Date import java.util.UUID import javax.inject.Inject import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.cancellation.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.flowOf +import timber.log.Timber class BookmarkManagerImpl @Inject constructor( appDatabase: AppDatabase, private val eventHorizon: EventHorizon, + private val syncManager: SyncManager, + private val transcriptWindowExtractor: TranscriptWindowExtractor, ) : BookmarkManager, CoroutineScope { @@ -226,4 +232,35 @@ class BookmarkManagerImpl @Inject constructor( override fun hasBookmarksFlow(episodeUuid: String): Flow { return bookmarkDao.hasBookmarksFlow(episodeUuid) } + + override suspend fun enrichBookmark(bookmark: Bookmark) { + try { + val snippet = transcriptWindowExtractor.extractWindow( + episodeUuid = bookmark.episodeUuid, + timeSecs = bookmark.timeSecs, + ) ?: return + + val response = syncManager.enrichBookmark(transcriptSnippet = snippet) + if (response.error != null) { + Timber.w("Smart bookmark enrichment returned error for ${bookmark.uuid}: ${response.error}") + } + val title = response.title + val summary = response.summary + if (title != null && summary != null) { + val now = System.currentTimeMillis() + bookmarkDao.updateAiData( + bookmarkUuid = bookmark.uuid, + aiTitle = title, + aiSummary = summary, + aiTitleModified = now, + aiSummaryModified = now, + syncStatus = SyncStatus.NOT_SYNCED, + ) + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Timber.e(e, "Smart bookmark enrichment failed for ${bookmark.uuid}") + } + } } diff --git a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/sync/SyncManager.kt b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/sync/SyncManager.kt index 0d87b0db2eb..6083b518168 100644 --- a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/sync/SyncManager.kt +++ b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/sync/SyncManager.kt @@ -23,6 +23,7 @@ import au.com.shiftyjelly.pocketcasts.servers.sync.SubscriptionStatusResponse import au.com.shiftyjelly.pocketcasts.servers.sync.UpNextSyncRequest import au.com.shiftyjelly.pocketcasts.servers.sync.UpNextSyncResponse import au.com.shiftyjelly.pocketcasts.servers.sync.UserChangeResponse +import au.com.shiftyjelly.pocketcasts.servers.sync.bookmark.BookmarkEnrichResponse import au.com.shiftyjelly.pocketcasts.servers.sync.history.HistoryYearResponse import au.com.shiftyjelly.pocketcasts.servers.sync.login.ExchangeSonosResponse import au.com.shiftyjelly.pocketcasts.utils.Optional @@ -124,6 +125,7 @@ interface SyncManager : NamedSettingsCaller { suspend fun upNextSync(request: UpNextSyncRequest): UpNextSyncResponse suspend fun upNextSyncProtobuf(request: UpNextSyncRequestProtobuf): UpNextResponse suspend fun getBookmarks(): List + suspend fun enrichBookmark(transcriptSnippet: String): BookmarkEnrichResponse suspend fun sendAnonymousFeedback(subject: String, inbox: String, message: String): Response suspend fun sendFeedback(subject: String, inbox: String, message: String): Response diff --git a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/sync/SyncManagerImpl.kt b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/sync/SyncManagerImpl.kt index c262f27cd08..0a20b437ac9 100644 --- a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/sync/SyncManagerImpl.kt +++ b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/sync/SyncManagerImpl.kt @@ -36,6 +36,8 @@ import au.com.shiftyjelly.pocketcasts.servers.sync.SyncServiceManager import au.com.shiftyjelly.pocketcasts.servers.sync.UpNextSyncRequest import au.com.shiftyjelly.pocketcasts.servers.sync.UpNextSyncResponse import au.com.shiftyjelly.pocketcasts.servers.sync.UserChangeResponse +import au.com.shiftyjelly.pocketcasts.servers.sync.bookmark.BookmarkEnrichRequest +import au.com.shiftyjelly.pocketcasts.servers.sync.bookmark.BookmarkEnrichResponse import au.com.shiftyjelly.pocketcasts.servers.sync.bookmark.toBookmark import au.com.shiftyjelly.pocketcasts.servers.sync.exception.RefreshTokenExpiredException import au.com.shiftyjelly.pocketcasts.servers.sync.history.HistoryYearResponse @@ -447,6 +449,15 @@ class SyncManagerImpl @Inject constructor( } } + override suspend fun enrichBookmark( + transcriptSnippet: String, + ): BookmarkEnrichResponse = getCacheTokenOrLogin { token -> + syncServiceManager.enrichBookmark( + request = BookmarkEnrichRequest(transcriptSnippet = transcriptSnippet), + token = token, + ) + } + override suspend fun sendAnonymousFeedback(subject: String, inbox: String, message: String): Response { return syncServiceManager.sendAnonymousFeedback(subject, inbox, message) } diff --git a/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/transcript/TranscriptWindowExtractor.kt b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/transcript/TranscriptWindowExtractor.kt new file mode 100644 index 00000000000..043c8ce01ab --- /dev/null +++ b/modules/services/repositories/src/main/java/au/com/shiftyjelly/pocketcasts/repositories/transcript/TranscriptWindowExtractor.kt @@ -0,0 +1,91 @@ +package au.com.shiftyjelly.pocketcasts.repositories.transcript + +import au.com.shiftyjelly.pocketcasts.models.db.dao.TranscriptDao +import au.com.shiftyjelly.pocketcasts.servers.podcast.TranscriptService +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.coroutines.cancellation.CancellationException +import kotlin.time.Duration.Companion.minutes +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.withTimeoutOrNull +import okhttp3.CacheControl +import timber.log.Timber + +@Singleton +class TranscriptWindowExtractor @Inject constructor( + private val transcriptDao: TranscriptDao, + private val transcriptService: TranscriptService, +) { + suspend fun extractWindow(episodeUuid: String, timeSecs: Int, windowSecs: Int = 30): String? { + return try { + val transcripts = withTimeoutOrNull(1.minutes) { + transcriptDao.observeTranscripts(episodeUuid) + .filter { it.isNotEmpty() } + .firstOrNull() + } + val generated = transcripts?.firstOrNull { it.isGenerated } ?: return null + + val body = runCatching { transcriptService.getTranscriptOrThrow(generated.url, CacheControl.FORCE_CACHE) } + .getOrNull() ?: return null + + val vttContent = body.use { it.string() } + parseVttWindow(vttContent, timeSecs, windowSecs) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Timber.e(e, "Failed to extract transcript window for episode $episodeUuid") + null + } + } + + companion object { + private val TIMESTAMP_REGEX = + """(?:(\d{2}):)?(\d{2}):(\d{2})\.\d{3}\s*-->\s*(?:(\d{2}):)?(\d{2}):(\d{2})\.\d{3}""".toRegex() + private val HTML_TAG_REGEX = """<[^>]+>""".toRegex() + + internal fun parseVttWindow(content: String, timeSecs: Int, windowSecs: Int): String? { + val windowStart = (timeSecs - windowSecs).coerceAtLeast(0) + val windowEnd = timeSecs + windowSecs + val lines = content.lines() + val texts = mutableListOf() + var i = 0 + + while (i < lines.size) { + val match = TIMESTAMP_REGEX.find(lines[i]) + if (match != null) { + val sh = match.groupValues[1].takeIf { it.isNotEmpty() }?.toInt() ?: 0 + val sm = match.groupValues[2].toInt() + val ss = match.groupValues[3].toInt() + val eh = match.groupValues[4].takeIf { it.isNotEmpty() }?.toInt() ?: 0 + val em = match.groupValues[5].toInt() + val es = match.groupValues[6].toInt() + val start = sh * 3600 + sm * 60 + ss + val end = eh * 3600 + em * 60 + es + + if (start < windowEnd && end > windowStart) { + i++ + val cueLines = mutableListOf() + while (i < lines.size && lines[i].isNotBlank()) { + cueLines.add(lines[i].replace(HTML_TAG_REGEX, "").trim()) + i++ + } + val text = cueLines.joinToString(" ").trim() + if (text.isNotEmpty()) { + texts.add(text) + } + } else { + i++ + while (i < lines.size && lines[i].isNotBlank()) { + i++ + } + } + } + i++ + } + + val result = texts.joinToString(" ") + return result.takeIf { it.split("\\s+".toRegex()).size >= 10 } + } + } +} diff --git a/modules/services/repositories/src/test/java/au/com/shiftyjelly/pocketcasts/repositories/transcript/TranscriptWindowExtractorTest.kt b/modules/services/repositories/src/test/java/au/com/shiftyjelly/pocketcasts/repositories/transcript/TranscriptWindowExtractorTest.kt new file mode 100644 index 00000000000..850bfb39183 --- /dev/null +++ b/modules/services/repositories/src/test/java/au/com/shiftyjelly/pocketcasts/repositories/transcript/TranscriptWindowExtractorTest.kt @@ -0,0 +1,124 @@ +package au.com.shiftyjelly.pocketcasts.repositories.transcript + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class TranscriptWindowExtractorTest { + + private val sampleVtt = """ + |WEBVTT + | + |00:00:00.000 --> 00:00:05.000 + |Welcome to the show everyone. + | + |00:00:05.000 --> 00:00:10.000 + |Today we are going to discuss artificial intelligence. + | + |00:00:10.000 --> 00:00:20.000 + |Let me start by defining what AI actually means in practice. + | + |00:00:20.000 --> 00:00:30.000 + |AI is a broad field that includes machine learning, deep learning, and more. + | + |00:00:30.000 --> 00:00:40.000 + |The recent advances have been truly remarkable for the industry. + | + |00:00:40.000 --> 00:00:50.000 + |Companies are investing billions of dollars into AI research. + | + |00:01:00.000 --> 00:01:10.000 + |This is a much later segment about totally different things. + """.trimMargin() + + @Test + fun `extract window around middle of transcript`() { + val result = TranscriptWindowExtractor.parseVttWindow(sampleVtt, timeSecs = 25, windowSecs = 15) + + assertEquals( + "Let me start by defining what AI actually means in practice. " + + "AI is a broad field that includes machine learning, deep learning, and more. " + + "The recent advances have been truly remarkable for the industry.", + result, + ) + } + + @Test + fun `extract window at start of transcript`() { + val result = TranscriptWindowExtractor.parseVttWindow(sampleVtt, timeSecs = 0, windowSecs = 10) + + assertEquals( + "Welcome to the show everyone. Today we are going to discuss artificial intelligence.", + result, + ) + } + + @Test + fun `return null when window has too few words`() { + val shortVtt = """ + |WEBVTT + | + |00:00:00.000 --> 00:00:05.000 + |Just a few words. + """.trimMargin() + + val result = TranscriptWindowExtractor.parseVttWindow(shortVtt, timeSecs = 2, windowSecs = 30) + + assertNull(result) + } + + @Test + fun `return null when no cues in window`() { + val result = TranscriptWindowExtractor.parseVttWindow(sampleVtt, timeSecs = 300, windowSecs = 10) + + assertNull(result) + } + + @Test + fun `extract window from mm-ss-mmm timestamps`() { + val vtt = """ + |WEBVTT + | + |00:00.000 --> 00:05.000 + |Welcome to the show everyone. + | + |00:05.000 --> 00:10.000 + |Today we are going to discuss artificial intelligence. + | + |00:10.000 --> 00:20.000 + |Let me start by defining what AI actually means in practice. + | + |00:20.000 --> 00:30.000 + |AI is a broad field that includes machine learning, deep learning, and more. + | + |00:30.000 --> 00:40.000 + |The recent advances have been truly remarkable for the industry. + """.trimMargin() + + val result = TranscriptWindowExtractor.parseVttWindow(vtt, timeSecs = 15, windowSecs = 10) + + assertEquals( + "Today we are going to discuss artificial intelligence. " + + "Let me start by defining what AI actually means in practice. " + + "AI is a broad field that includes machine learning, deep learning, and more.", + result, + ) + } + + @Test + fun `strip html tags from cue text`() { + val vttWithTags = """ + |WEBVTT + | + |00:00:00.000 --> 00:00:10.000 + |This is a sentence with enough words to pass the minimum threshold for extraction. + """.trimMargin() + + val result = TranscriptWindowExtractor.parseVttWindow(vttWithTags, timeSecs = 5, windowSecs = 10) + + assertEquals( + "This is a sentence with enough words to pass the minimum threshold for extraction.", + result, + ) + } +} diff --git a/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/SyncService.kt b/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/SyncService.kt index 39cb2ffe31b..00fa5858105 100644 --- a/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/SyncService.kt +++ b/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/SyncService.kt @@ -2,6 +2,8 @@ package au.com.shiftyjelly.pocketcasts.servers.sync import au.com.shiftyjelly.pocketcasts.models.to.HistorySyncRequest import au.com.shiftyjelly.pocketcasts.models.to.HistorySyncResponse +import au.com.shiftyjelly.pocketcasts.servers.sync.bookmark.BookmarkEnrichRequest +import au.com.shiftyjelly.pocketcasts.servers.sync.bookmark.BookmarkEnrichResponse import au.com.shiftyjelly.pocketcasts.servers.sync.forgotpassword.ForgotPasswordRequest import au.com.shiftyjelly.pocketcasts.servers.sync.forgotpassword.ForgotPasswordResponse import au.com.shiftyjelly.pocketcasts.servers.sync.history.HistoryYearResponse @@ -180,6 +182,9 @@ interface SyncService { @POST("/user/bookmark/list") suspend fun getBookmarkList(@Header("Authorization") authorization: String, @Body request: BookmarkRequest): BookmarksResponse + @POST("/user/bookmark/enrich") + suspend fun enrichBookmark(@Header("Authorization") authorization: String, @Body request: BookmarkEnrichRequest): BookmarkEnrichResponse + @Headers("Content-Type: application/octet-stream") @POST("/user/podcast_rating/add") suspend fun addPodcastRating(@Header("Authorization") authorization: String, @Body request: PodcastRatingAddRequest): PodcastRatingResponse diff --git a/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/SyncServiceManager.kt b/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/SyncServiceManager.kt index aa3a13d0651..6cb4a37cc48 100644 --- a/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/SyncServiceManager.kt +++ b/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/SyncServiceManager.kt @@ -10,6 +10,8 @@ import au.com.shiftyjelly.pocketcasts.preferences.AccessToken import au.com.shiftyjelly.pocketcasts.preferences.RefreshToken import au.com.shiftyjelly.pocketcasts.preferences.Settings import au.com.shiftyjelly.pocketcasts.servers.di.Cached +import au.com.shiftyjelly.pocketcasts.servers.sync.bookmark.BookmarkEnrichRequest +import au.com.shiftyjelly.pocketcasts.servers.sync.bookmark.BookmarkEnrichResponse import au.com.shiftyjelly.pocketcasts.servers.sync.forgotpassword.ForgotPasswordRequest import au.com.shiftyjelly.pocketcasts.servers.sync.forgotpassword.ForgotPasswordResponse import au.com.shiftyjelly.pocketcasts.servers.sync.history.HistoryYearResponse @@ -178,6 +180,10 @@ open class SyncServiceManager @Inject constructor( return service.getBookmarkList(addBearer(token), bookmarkRequest {}) } + suspend fun enrichBookmark(request: BookmarkEnrichRequest, token: AccessToken): BookmarkEnrichResponse { + return service.enrichBookmark(addBearer(token), request) + } + suspend fun getEpisodes(request: PodcastsEpisodesRequest, token: AccessToken): EpisodesResponse { return service.getEpisodes(addBearer(token), request) } diff --git a/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/bookmark/BookmarkEnrichModels.kt b/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/bookmark/BookmarkEnrichModels.kt new file mode 100644 index 00000000000..c5e24577fd2 --- /dev/null +++ b/modules/services/servers/src/main/java/au/com/shiftyjelly/pocketcasts/servers/sync/bookmark/BookmarkEnrichModels.kt @@ -0,0 +1,16 @@ +package au.com.shiftyjelly.pocketcasts.servers.sync.bookmark + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass + +@JsonClass(generateAdapter = true) +data class BookmarkEnrichRequest( + @Json(name = "transcript_snippet") val transcriptSnippet: String, +) + +@JsonClass(generateAdapter = true) +data class BookmarkEnrichResponse( + @Json(name = "title") val title: String? = null, + @Json(name = "summary") val summary: String? = null, + @Json(name = "error") val error: String? = null, +) diff --git a/modules/services/utils/src/main/java/au/com/shiftyjelly/pocketcasts/utils/featureflag/Feature.kt b/modules/services/utils/src/main/java/au/com/shiftyjelly/pocketcasts/utils/featureflag/Feature.kt index 22ce5a8136a..a3ca58e1e6d 100644 --- a/modules/services/utils/src/main/java/au/com/shiftyjelly/pocketcasts/utils/featureflag/Feature.kt +++ b/modules/services/utils/src/main/java/au/com/shiftyjelly/pocketcasts/utils/featureflag/Feature.kt @@ -272,6 +272,14 @@ enum class Feature( hasFirebaseRemoteFlag = false, hasDevToggle = true, ), + SMART_BOOKMARKS( + key = "smart_bookmarks", + title = "AI-enriched bookmarks with title and summary", + defaultValue = isDebugOrPrototypeBuild, + tier = FeatureTier.Plus(), + hasFirebaseRemoteFlag = true, + hasDevToggle = true, + ), EPISODE_CHAT( key = "episode_chat", title = "Episode Chat",