diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 689cf6e53d2..275e1240207 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -237,9 +237,40 @@ kt_jvm_binary( ], ) -# Note that this is intentionally not test-only since it's used by the app build pipeline. Also, -# this apparently needs to be a java_binary to set up runfiles correctly when executed within a -# Starlark rule as a tool. +java_binary( + name = "download_lesson_list", + testonly = True, + # Hide warnings that come from https://github.com/square/retrofit/issues/3341. + jvm_flags = [ + "--add-opens", + "java.base/java.lang.invoke=ALL-UNNAMED", + ], + main_class = "org.oppia.android.scripts.assets.DownloadLessonListKt", + visibility = ["//visibility:public"], # TODO: Revert this when the script no longer needs to be referenced on develop. + runtime_deps = [ + "//scripts/src/java/org/oppia/android/scripts/assets:download_lesson_list_lib", + ], +) + +java_binary( + name = "download_lessons", + testonly = True, + # Hide warnings that come from https://github.com/square/retrofit/issues/3341. + jvm_flags = [ + "--add-opens", + "java.base/java.lang.invoke=ALL-UNNAMED", + "-Xmx16g", # Image conversion can require a lot of RAM. + ], + main_class = "org.oppia.android.scripts.assets.DownloadLessonsKt", + visibility = ["//visibility:public"], # TODO: Revert this when the script no longer needs to be referenced on develop. + runtime_deps = [ + "//scripts/src/java/org/oppia/android/scripts/assets:download_lessons_lib", + ], +) + +# Note that this & the other binaries below are intentionally not test-only since they're used by +# the app build pipeline. Also, this apparently needs to be a java_binary to set up runfiles +# correctly when executed within a Starlark rule as a tool. java_binary( name = "transform_android_manifest", main_class = "org.oppia.android.scripts.build.TransformAndroidManifestKt", diff --git a/scripts/src/java/org/oppia/android/scripts/assets/BUILD.bazel b/scripts/src/java/org/oppia/android/scripts/assets/BUILD.bazel new file mode 100644 index 00000000000..93509089c53 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/assets/BUILD.bazel @@ -0,0 +1,55 @@ +""" +Libraries corresponding to asset transformation & download scripts. +""" + +load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") + +kt_jvm_library( + name = "download_lesson_list_lib", + testonly = True, + srcs = ["DownloadLessonList.kt"], + visibility = ["//scripts:oppia_script_binary_visibility"], + deps = [ + "//scripts/src/java/org/oppia/android/scripts/common:script_background_coroutine_dispatcher", + "//scripts/src/java/org/oppia/android/scripts/gae", + "//scripts/src/java/org/oppia/android/scripts/gae:gae_json_impl", + "//scripts/src/java/org/oppia/android/scripts/gae/proto:proto_version_provider", + "//scripts/src/java/org/oppia/android/scripts/proto:download_list_versions_java_proto", + ], +) + +kt_jvm_library( + name = "download_lessons_lib", + testonly = True, + srcs = ["DownloadLessons.kt"], + visibility = ["//scripts:oppia_script_binary_visibility"], + deps = [ + ":dto_proto_to_legacy_proto_converter", + ":image_repairer", + "//scripts/src/java/org/oppia/android/scripts/gae", + "//scripts/src/java/org/oppia/android/scripts/gae:gae_json_impl", + "//scripts/src/java/org/oppia/android/scripts/gae/proto:proto_version_provider", + ], +) + +kt_jvm_library( + name = "dto_proto_to_legacy_proto_converter", + testonly = True, + srcs = ["DtoProtoToLegacyProtoConverter.kt"], + deps = [ + "//model/src/main/proto:exploration_java_proto", + "//model/src/main/proto:interaction_object_java_proto", + "//model/src/main/proto:languages_java_proto", + "//model/src/main/proto:math_java_proto", + "//model/src/main/proto:topic_java_proto", + "//model/src/main/proto:translation_java_proto", + "//third_party:oppia_proto_api_java_protos", + ], +) + +kt_jvm_library( + name = "image_repairer", + testonly = True, + srcs = ["ImageRepairer.kt"], + deps = ["//third_party:com_github_weisj_jsvg"], +) diff --git a/scripts/src/java/org/oppia/android/scripts/assets/DownloadLessonList.kt b/scripts/src/java/org/oppia/android/scripts/assets/DownloadLessonList.kt new file mode 100644 index 00000000000..c6a2c7dfe11 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/assets/DownloadLessonList.kt @@ -0,0 +1,246 @@ +package org.oppia.android.scripts.assets + +import com.google.protobuf.TextFormat +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import org.oppia.android.scripts.common.ScriptBackgroundCoroutineDispatcher +import org.oppia.android.scripts.gae.GaeAndroidEndpoint +import org.oppia.android.scripts.gae.GaeAndroidEndpointJsonImpl +import org.oppia.android.scripts.gae.gcs.GcsService +import org.oppia.android.scripts.gae.proto.ImageDownloader +import org.oppia.android.scripts.gae.proto.ProtoVersionProvider +import org.oppia.android.scripts.proto.DownloadListVersions +import org.oppia.android.scripts.proto.DownloadListVersions.ChapterInfo +import org.oppia.android.scripts.proto.DownloadListVersions.SkillInfo +import org.oppia.android.scripts.proto.DownloadListVersions.StoryInfo +import org.oppia.android.scripts.proto.DownloadListVersions.SubtopicInfo +import org.oppia.android.scripts.proto.DownloadListVersions.TopicInfo +import org.oppia.proto.v1.api.AndroidClientContextDto +import org.oppia.proto.v1.api.TopicListRequestDto +import org.oppia.proto.v1.api.TopicListResponseDto +import org.oppia.proto.v1.api.TopicListResponseDto.AvailableTopicDto.AvailabilityTypeCase.DOWNLOADABLE_TOPIC +import org.oppia.proto.v1.structure.ChapterSummaryDto +import org.oppia.proto.v1.structure.DownloadableTopicSummaryDto +import org.oppia.proto.v1.structure.LanguageType +import org.oppia.proto.v1.structure.StorySummaryDto +import org.oppia.proto.v1.structure.SubtopicSummaryDto +import java.io.File + +// TODO: hook up to language configs for prod/dev language restrictions. +// TODO: Consider using better argument parser so that dev env vals can be defaulted. +// TODO: verify that images aren't changed after upload, but this needs to be confirmed (that is, if they need to be changed a new image is added to GCS, instead). +fun main(vararg args: String) { + check(args.size == 6) { + "Expected use: bazel run //scripts:download_lesson_list " + + " " + + " " + } + + val baseUrl = args[0] + val gcsBaseUrl = args[1] + val gcsBucket = args[2] + val apiSecretPath = args[3] + val outputFilePath = args[4] + val apiDebugPath = args[5] + val apiSecretFile = File(apiSecretPath).absoluteFile.normalize().also { + check(it.exists() && it.isFile) { "Expected API secret file to exist: $apiSecretPath." } + } + val outputFile = File(outputFilePath).absoluteFile.normalize() + val apiDebugDir = File(apiDebugPath).absoluteFile.normalize().also { + check(if (!it.exists()) it.mkdirs() else it.isDirectory) { + "Expected API debug directory to exist or to be creatable: $apiDebugPath." + } + } + + val apiSecret = apiSecretFile.readText().trim() + + ScriptBackgroundCoroutineDispatcher().use { scriptBgDispatcher -> + val downloader = LessonListDownloader( + baseUrl, gcsBaseUrl, gcsBucket, apiSecret, apiDebugDir, scriptBgDispatcher + ) + runBlocking { downloader.downloadLessonListAsync(outputFile).await() } + } +} + +class LessonListDownloader( + gaeBaseUrl: String, + gcsBaseUrl: String, + gcsBucket: String, + apiSecret: String, + private val apiDebugDir: File, + private val scriptBgDispatcher: ScriptBackgroundCoroutineDispatcher +) { + private val gcsService by lazy { GcsService(gcsBaseUrl, gcsBucket) } + private val imageDownloader by lazy { ImageDownloader(gcsService, scriptBgDispatcher) } + private val androidEndpoint: GaeAndroidEndpoint by lazy { + GaeAndroidEndpointJsonImpl( + apiSecret, + gaeBaseUrl, + apiDebugDir, + forceCacheLoad = false, + scriptBgDispatcher, + imageDownloader, + forcedVersions = null // Always load latest when creating the pin versions list. + ) + } + + fun downloadLessonListAsync(lessonListOutputFile: File): Deferred { + return CoroutineScope(scriptBgDispatcher).async { + println("Config: Using ${apiDebugDir.path}/ for storing API responses (for debugging).") + + val listResponse = downloadTopicListResponseDto() + println() + + println("Writing captured lesson structure versions to:") + println(lessonListOutputFile.path) + withContext(Dispatchers.IO) { + lessonListOutputFile.outputStream().bufferedWriter().use { + TextFormat.printer().print(listResponse.captureVersions(), it) + } + } + } + } + + private suspend fun downloadTopicListResponseDto(): TopicListResponseDto { + val defaultLanguage = LanguageType.ENGLISH + val requestedLanguages = setOf( + LanguageType.ARABIC, + LanguageType.BRAZILIAN_PORTUGUESE, + LanguageType.NIGERIAN_PIDGIN + ) + val listRequest = TopicListRequestDto.newBuilder().apply { + protoVersion = ProtoVersionProvider.createLatestTopicListProtoVersion() + clientContext = CLIENT_CONTEXT + compatibilityContext = ProtoVersionProvider.createCompatibilityContext() + // No structures are considered already downloaded. TODO: Integrate with local files cache? + requestedDefaultLanguage = defaultLanguage +// addAllRequiredAdditionalLanguages(requestedLanguages) + addAllSupportedAdditionalLanguages(requestedLanguages) + }.build() + + println() + val listContentMessage = "Sending topic list download request" + val extraDotsThatCanFitForList = CONSOLE_COLUMN_COUNT - listContentMessage.length + var lastDotCount = 0 + print(listContentMessage) + val listResponse = + androidEndpoint.fetchTopicListAsync(listRequest) { finishCount, totalCount -> + val dotCount = (extraDotsThatCanFitForList * finishCount) / totalCount + val dotsToAdd = dotCount - lastDotCount + if (dotsToAdd > 0) { + print(".".repeat(dotsToAdd)) + lastDotCount = dotCount + } + }.await() + println() + + return listResponse + } + + private companion object { + private val CLIENT_CONTEXT = AndroidClientContextDto.newBuilder().apply { + appVersionName = checkNotNull(LessonListDownloader::class.qualifiedName) + appVersionCode = 0 + }.build() + private const val CONSOLE_COLUMN_COUNT = 80 + + private const val PLACE_VALUES_ID = "iX9kYCjnouWN" + private const val ADDITION_AND_SUBTRACTION_ID = "sWBXKH4PZcK6" + private const val MULTIPLICATION_ID = "C4fqwrvqWpRm" + private const val DIVISION_ID = "qW12maD4hiA8" + private const val EXPRESSIONS_AND_EQUATIONS_ID = "dLmjjMDbCcrf" + private const val FRACTIONS_ID = "0abdeaJhmfPm" + private const val RATIOS_ID = "5g0nxGUmx5J5" + + private val fractionsDependencies by lazy { + setOf(ADDITION_AND_SUBTRACTION_ID, MULTIPLICATION_ID, DIVISION_ID) + } + private val ratiosDependencies by lazy { + setOf(ADDITION_AND_SUBTRACTION_ID, MULTIPLICATION_ID, DIVISION_ID) + } + private val additionAndSubtractionDependencies by lazy { setOf(PLACE_VALUES_ID) } + private val multiplicationDependencies by lazy { setOf(ADDITION_AND_SUBTRACTION_ID) } + private val divisionDependencies by lazy { setOf(MULTIPLICATION_ID) } + private val placeValuesDependencies by lazy { setOf() } + private val expressionsAndEquationsDependencies by lazy { + setOf(ADDITION_AND_SUBTRACTION_ID, MULTIPLICATION_ID, DIVISION_ID) + } + + // TODO: Migrate deps over to the data coming from GAE (since it *is* present). + private val topicDependenciesTable by lazy { + mapOf( + FRACTIONS_ID to fractionsDependencies, + RATIOS_ID to ratiosDependencies, + ADDITION_AND_SUBTRACTION_ID to additionAndSubtractionDependencies, + MULTIPLICATION_ID to multiplicationDependencies, + DIVISION_ID to divisionDependencies, + PLACE_VALUES_ID to placeValuesDependencies, + EXPRESSIONS_AND_EQUATIONS_ID to expressionsAndEquationsDependencies, + ) + } + + private fun TopicListResponseDto.captureVersions(): DownloadListVersions { + val downloadableTopics = availableTopicsList.filter { availableTopic -> + availableTopic.availabilityTypeCase == DOWNLOADABLE_TOPIC + }.map { it.downloadableTopic.topicSummary } + val topicInfos = downloadableTopics.map { it.captureVersions() } + + // Ensure that duplicate skill structures are actually the same for a given ID. + val allReferencedSkills = + downloadableTopics.flatMap { it.referencedSkillsList }.groupBy { it.id } + val uniqueReferencedSkills = allReferencedSkills.mapValues { (skillId, dupedSkills) -> + val distinctSkills = dupedSkills.distinct() + check(distinctSkills.size == 1) { + "Expected all references to skill $skillId to be the same skill structure." + } + return@mapValues distinctSkills.single() + } + + val skillInfos = uniqueReferencedSkills.map { (skillId, skillSummary) -> + SkillInfo.newBuilder().apply { + this.id = skillId + this.contentVersion = skillSummary.contentVersion + }.build() + } + return DownloadListVersions.newBuilder().apply { + addAllTrackedTopicInfo(topicInfos) + addAllTrackedSkillInfo(skillInfos) + }.build() + } + + private fun DownloadableTopicSummaryDto.captureVersions(): TopicInfo { + return TopicInfo.newBuilder().apply { + this.id = this@captureVersions.id + this.contentVersion = this@captureVersions.contentVersion + addAllStoryInfo(this@captureVersions.storySummariesList.map { it.captureVersions() }) + addAllSubtopicInfo(this@captureVersions.subtopicSummariesList.map { it.captureVersion() }) + }.build() + } + + private fun StorySummaryDto.captureVersions(): StoryInfo { + return StoryInfo.newBuilder().apply { + this.id = this@captureVersions.id + this.contentVersion = this@captureVersions.contentVersion + addAllChapterInfo(this@captureVersions.chaptersList.map { it.captureVersion() }) + }.build() + } + + private fun ChapterSummaryDto.captureVersion(): ChapterInfo { + return ChapterInfo.newBuilder().apply { + this.explorationId = this@captureVersion.explorationId + this.explorationContentVersion = this@captureVersion.contentVersion + }.build() + } + + private fun SubtopicSummaryDto.captureVersion(): SubtopicInfo { + return SubtopicInfo.newBuilder().apply { + this.index = this@captureVersion.index + this.contentVersion = this@captureVersion.contentVersion + }.build() + } + } +} diff --git a/scripts/src/java/org/oppia/android/scripts/assets/DownloadLessons.kt b/scripts/src/java/org/oppia/android/scripts/assets/DownloadLessons.kt new file mode 100644 index 00000000000..f58b3d896fa --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/assets/DownloadLessons.kt @@ -0,0 +1,2508 @@ +package org.oppia.android.scripts.assets + +import com.google.protobuf.Message +import com.google.protobuf.TextFormat +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.SendChannel +import kotlinx.coroutines.flow.consumeAsFlow +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.withIndex +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToClassroomList +import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToConceptCardList +import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToExploration +import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToStoryRecord +import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToSubtopicRecord +import org.oppia.android.scripts.assets.DtoProtoToLegacyProtoConverter.convertToTopicRecord +import org.oppia.android.scripts.gae.GaeAndroidEndpoint +import org.oppia.android.scripts.gae.GaeAndroidEndpointJsonImpl +import org.oppia.android.scripts.gae.gcs.GcsService +import org.oppia.android.scripts.gae.gcs.GcsService.ImageContainerType +import org.oppia.android.scripts.gae.gcs.GcsService.ImageType +import org.oppia.android.scripts.gae.proto.ImageDownloader +import org.oppia.android.scripts.gae.proto.ProtoVersionProvider +import org.oppia.android.scripts.proto.DownloadListVersions +import org.oppia.proto.v1.api.AndroidClientContextDto +import org.oppia.proto.v1.api.DownloadRequestStructureIdentifierDto +import org.oppia.proto.v1.api.DownloadRequestStructureIdentifierDto.StructureTypeCase +import org.oppia.proto.v1.api.TopicContentRequestDto +import org.oppia.proto.v1.api.TopicContentResponseDto +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.CONCEPT_CARD +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.CONCEPT_CARD_LANGUAGE_PACK +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.EXPLORATION +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.EXPLORATION_LANGUAGE_PACK +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.QUESTION +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.QUESTION_ID_LIST +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.QUESTION_LANGUAGE_PACK +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.RESULTTYPE_NOT_SET +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.REVISION_CARD +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.REVISION_CARD_LANGUAGE_PACK +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.SKIPPED_DOES_NOT_EXIST +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.SKIPPED_FROM_FAILURE +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.SKIPPED_SHOULD_RETRY +import org.oppia.proto.v1.api.TopicContentResponseDto.DownloadResultDto.ResultTypeCase.TOPIC_SUMMARY +import org.oppia.proto.v1.api.TopicListRequestDto +import org.oppia.proto.v1.api.TopicListResponseDto.AvailableTopicDto.AvailabilityTypeCase.DOWNLOADABLE_TOPIC +import org.oppia.proto.v1.structure.BaseAnswerGroupDto +import org.oppia.proto.v1.structure.BaseSolutionDto +import org.oppia.proto.v1.structure.ChapterSummaryDto +import org.oppia.proto.v1.structure.ConceptCardDto +import org.oppia.proto.v1.structure.ConceptCardDto.WorkedExampleDto +import org.oppia.proto.v1.structure.ConceptCardLanguagePackDto +import org.oppia.proto.v1.structure.ContentLocalizationDto +import org.oppia.proto.v1.structure.ContentLocalizationsDto +import org.oppia.proto.v1.structure.DownloadableTopicSummaryDto +import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto.RuleTypeCase.IS_EQUAL_TO_ORDERING_WITH_ONE_ITEM_AT_INCORRECT_POSITION +import org.oppia.proto.v1.structure.ExplorationDto +import org.oppia.proto.v1.structure.ExplorationLanguagePackDto +import org.oppia.proto.v1.structure.HintDto +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase +import org.oppia.proto.v1.structure.LanguageType +import org.oppia.proto.v1.structure.ListOfSetsOfTranslatableHtmlContentIdsDto +import org.oppia.proto.v1.structure.LocalizableTextDto +import org.oppia.proto.v1.structure.LocalizedConceptCardIdDto +import org.oppia.proto.v1.structure.LocalizedExplorationIdDto +import org.oppia.proto.v1.structure.LocalizedRevisionCardIdDto +import org.oppia.proto.v1.structure.OutcomeDto +import org.oppia.proto.v1.structure.QuestionDto +import org.oppia.proto.v1.structure.QuestionLanguagePackDto +import org.oppia.proto.v1.structure.ReferencedImageDto +import org.oppia.proto.v1.structure.ReferencedImageListDto +import org.oppia.proto.v1.structure.RevisionCardDto +import org.oppia.proto.v1.structure.RevisionCardLanguagePackDto +import org.oppia.proto.v1.structure.SetOfTranslatableHtmlContentIdsDto +import org.oppia.proto.v1.structure.SkillSummaryDto +import org.oppia.proto.v1.structure.StateDto +import org.oppia.proto.v1.structure.StorySummaryDto +import org.oppia.proto.v1.structure.SubtopicPageIdDto +import org.oppia.proto.v1.structure.SubtopicSummaryDto +import org.oppia.proto.v1.structure.TextInputInstanceDto +import org.oppia.proto.v1.structure.ThumbnailDto +import org.oppia.proto.v1.structure.TranslatableHtmlContentIdDto +import org.oppia.proto.v1.structure.TranslatableSetOfNormalizedStringDto +import org.oppia.proto.v1.structure.UpcomingTopicSummaryDto +import java.io.File +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import org.oppia.proto.v1.api.DownloadRequestStructureIdentifierDto.Builder as DownloadReqStructIdDtoBuilder +import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto as DragDropSortRuleSpecDto +import org.oppia.proto.v1.structure.ItemSelectionInputInstanceDto.RuleSpecDto as ItemSelRuleSpecDto + +/* + TODO: Thoughts for improving the script: + - Introduce args to allow enabling/disabling specific checks. + - Change output structure such that just the top-level asset dir is provided and then the following are always generated: + - prod_assets/{images,*.pb} + - pipeline/ + - images_{conversions,renames}/ + - protov{1,2}/{textproto,binary}/ + - proto_conversion_info.log + - classrooms_cache/ (full picture isn't presented, but the idea is to show all versions downloaded & analyses from them, plus data hierarchy) + - {separate_classroom_name}_topics/ + - classroom_{name}_download_info.log + - {topic_id}_v{version_name}/ + - topic_{id}_summary_download_info.log + - topic_summary.json + ... + - Change cache_mode to just be lazy/force (with lazy being default). 'none' isn't necessary. + - Generate download_info.log files for the cache that include: + - The specific URL used to download each file. + - Any specific irrecoverable failures encountered during import (prior to conversion to proto). + - All optional & non-optional failures from json to protov2. + - Generate a proto_conversion_info.log that includes: + - Any issues or warnings from protov2 to protov1. + - Broadly, reduce all output such that the script is leaning much more heavily on files (where it also should output much more diagnostic information than done today). This keeps the script output lean to pass/fail & progress indicators, and the generated output logs to enough context to investigate both recoverable and irrecoverable failures. + - NOTE: Enabling addAllRequiredAdditionalLanguages below forces large batches of version requests (which leads to significant performance issues on Oppia web). Things to do here: + - 1. Introduce a batch fetcher for merging different structure versions & IDs (and maybe types) since the endpoint can handle this. It's fewer items to pass back and forth. + - 2. Make web more fault tolerant; it's fine if it can't fetch everything. Provide enough context to the client to retry failing requests (though we need to pass *why* it fails). Consider also avoiding using memcache in some cases since it doesn't need to cache old versions of structures. + - 3. Ensure the fetcher utilizes the retry functionality from (2). + - 4. Once the fetch succeeds again, look into fixing the compatibility problems that lead to the version explosion. Make sure on-disk caching works correctly. + - Consider looking into Firebase auth now vs. later since we need to make backend changes, anyway. + */ + +// TODO: hook up to language configs for prod/dev language restrictions. +// TODO: Consider using better argument parser so that dev env vals can be defaulted. +fun main(vararg args: String) { + check(args.size == 8) { + "Expected use: bazel run //scripts:download_lessons " + + " " + + " " + } + + val baseUrl = args[0] + val gcsBaseUrl = args[1] + val gcsBucket = args[2] + val apiSecretPath = args[3] + val outputDirPath = args[4] + val cacheModeLine = args[5] + val forceCacheLoad = when (val cacheMode = cacheModeLine.removePrefix("cache_mode=")) { + "none" -> false + "lazy" -> false + "force" -> true + else -> error("Invalid cache_mode: $cacheMode.") + } + val cacheDirPath = args[6] + val downloadListVersionsPath = args[7] + val cacheDir = File(cacheDirPath).absoluteFile.normalize().also { + check(if (!it.exists()) it.mkdirs() else it.isDirectory) { + "Expected cache directory to exist or to be creatable: $cacheDirPath." + } + } + val outputDir = File(outputDirPath).absoluteFile.normalize().also { + check(if (!it.exists()) it.mkdirs() else it.isDirectory) { + "Expected output directory to exist or to be creatable: $outputDirPath." + } + } + + val apiSecretFile = File(apiSecretPath).absoluteFile.normalize().also { + check(it.exists() && it.isFile) { "Expected API secret file to exist: $apiSecretPath." } + } + val downloadListVersionsFile = File(downloadListVersionsPath).absoluteFile.normalize().also { + check(it.exists() && it.isFile) { + "Expected versions proto file to exist: $downloadListVersionsPath." + } + } + val apiSecret = apiSecretFile.readText().trim() + val downloadListVersions = when (downloadListVersionsFile.extension) { + "pb" -> downloadListVersionsFile.inputStream().buffered().use(DownloadListVersions::parseFrom) + "textproto" -> // TODO: Force pb to be used. + TextFormat.parse(downloadListVersionsFile.readText(), DownloadListVersions::class.java) + else -> error("Invalid extension for versions proto file: $downloadListVersionsPath.") + } + + println("Using $downloadListVersionsPath to force structure versions for determinism.") + val downloader = + LessonDownloader( + baseUrl, gcsBaseUrl, gcsBucket, apiSecret, cacheDir, forceCacheLoad, downloadListVersions + ) + downloader.downloadLessons(outputDir) +} + +class LessonDownloader( + gaeBaseUrl: String, + gcsBaseUrl: String, + gcsBucket: String, + apiSecret: String, + private val cacheDir: File?, + private val forceCacheLoad: Boolean, + downloadListVersions: DownloadListVersions? +) { + private val threadPool by lazy { + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()) + } + private val coroutineDispatcher by lazy { threadPool.asCoroutineDispatcher() } + private val blockingDispatcher by lazy { + Executors.newSingleThreadExecutor().asCoroutineDispatcher() + } + private val gcsService by lazy { GcsService(gcsBaseUrl, gcsBucket) } + private val imageDownloader by lazy { ImageDownloader(gcsService, coroutineDispatcher) } + private val androidEndpoint: GaeAndroidEndpoint by lazy { + GaeAndroidEndpointJsonImpl( + apiSecret, + gaeBaseUrl, + cacheDir, + forceCacheLoad, + coroutineDispatcher, + imageDownloader, + forcedVersions = downloadListVersions + ) + } + private val textFormat by lazy { TextFormat.printer() } + private val imageRepairer by lazy { ImageRepairer() } + // TODO: Convert ByteArray to DownloadedImage for better analysis? + private val memoizedLoadedImageData by lazy { ConcurrentHashMap() } + + fun downloadLessons(outputDir: File) { + val downloadJob = CoroutineScope(coroutineDispatcher).launch { downloadAllLessons(outputDir) } + runBlocking { + downloadJob.invokeOnCompletion { exception -> + exception?.printStackTrace() + shutdownBlocking() + } + } + } + + private suspend fun downloadAllLessons(outputDir: File) { + when { + cacheDir == null -> println("Config: Not using a local disk directory for asset caching.") + !forceCacheLoad -> println("Config: Using ${cacheDir.path}/ for caching assets across runs.") + else -> { + println( + "Config: Using ${cacheDir.path}/ for caching assets across runs, with no latest updating." + ) + } + } + + val defaultLanguage = LanguageType.ENGLISH + val requestedLanguages = setOf( + LanguageType.ARABIC, + LanguageType.BRAZILIAN_PORTUGUESE, + LanguageType.NIGERIAN_PIDGIN + ) + val listRequest = TopicListRequestDto.newBuilder().apply { + protoVersion = ProtoVersionProvider.createLatestTopicListProtoVersion() + clientContext = CLIENT_CONTEXT + compatibilityContext = ProtoVersionProvider.createCompatibilityContext() + // No structures are considered already downloaded. TODO: Integrate with local files cache? + requestedDefaultLanguage = defaultLanguage + // addAllRequiredAdditionalLanguages(requestedLanguages) + addAllSupportedAdditionalLanguages(requestedLanguages) + }.build() + + println() + val listContentMessage = "Sending topic list download request" + val extraDotsThatCanFitForList = CONSOLE_COLUMN_COUNT - listContentMessage.length + var lastDotCount = 0 + print(listContentMessage) + val listResponse = + androidEndpoint.fetchTopicListAsync(listRequest) { finishCount, totalCount -> + val dotCount = (extraDotsThatCanFitForList * finishCount) / totalCount + val dotsToAdd = dotCount - lastDotCount + if (dotsToAdd > 0) { + print(".".repeat(dotsToAdd)) + lastDotCount = dotCount + } + }.await() + println() + + val downloadableTopics = listResponse.availableTopicsList.filter { availableTopic -> + availableTopic.availabilityTypeCase == DOWNLOADABLE_TOPIC + }.map { it.downloadableTopic.topicSummary } + val upcomingTopics = listResponse.futureTopicsList.map { it.topicSummary } + val downloadableTopicIds = downloadableTopics.map { it.id } + val futureTopicIds = listResponse.futureTopicsList.map { it.topicId } + println( + "Downloaded topic results: ${listResponse.availableTopicsCount} topics are available," + + " ${downloadableTopics.size} are downloadable, IDs: $downloadableTopicIds." + + " ${futureTopicIds.size} topics will later be available, IDs: $futureTopicIds." + ) + println("${listResponse.classroomsCount} classrooms are available:") + listResponse.classroomsList.forEach { classroom -> + val defaultLocalization = classroom.localizations.defaultMapping + val nameContentId = classroom.name.contentId + val textMapping = defaultLocalization.localizableTextContentMappingMap[nameContentId] + val classroomName = + textMapping?.singleLocalizableText?.text?.takeIf { it.isNotEmpty() } ?: "" + println("- ${classroom.id}: $classroomName (has ${classroom.topicIdsCount} topics)") + } + + println() + val contentRequest = + createDownloadContentRequest(downloadableTopics, defaultLanguage, requestedLanguages.toList()) + val contentMessage = "Requesting to download ${contentRequest.identifiersCount} content items" + val extraDotsThatCanFitForContent = CONSOLE_COLUMN_COUNT - contentMessage.length + lastDotCount = 0 + print(contentMessage) + val contentResponse = + androidEndpoint.fetchTopicContentAsync(contentRequest) { finishCount, totalCount -> + val dotCount = (extraDotsThatCanFitForContent * finishCount) / totalCount + val dotsToAdd = dotCount - lastDotCount + if (dotsToAdd > 0) { + print(".".repeat(dotsToAdd)) + lastDotCount = dotCount + } + }.await() + println() + + val revisionCardPackRequestResults = + contentResponse.downloadResultsList.filter { + it.identifier.structureTypeCase == StructureTypeCase.REVISION_CARD_LANGUAGE_PACK + } + val conceptCardPackRequestResults = + contentResponse.downloadResultsList.filter { + it.identifier.structureTypeCase == StructureTypeCase.CONCEPT_CARD_LANGUAGE_PACK + } + val explorationPackRequestResults = + contentResponse.downloadResultsList.filter { + it.identifier.structureTypeCase == StructureTypeCase.EXPLORATION_LANGUAGE_PACK + } + val revisionCardPackRequestResultsByLanguage = + revisionCardPackRequestResults.groupBy { it.identifier.revisionCardLanguagePack.language } + val conceptCardPackRequestResultsByLanguage = + conceptCardPackRequestResults.groupBy { it.identifier.conceptCardLanguagePack.language } + val explorationPackRequestResultsByLanguage = + explorationPackRequestResults.groupBy { it.identifier.explorationLanguagePack.language } + + val topicRequestCount = + contentResponse.downloadResultsList.count { + it.identifier.structureTypeCase == StructureTypeCase.TOPIC_SUMMARY_ID + } + val revisionCardRequestCount = + contentResponse.downloadResultsList.count { + it.identifier.structureTypeCase == StructureTypeCase.REVISION_CARD + } + val conceptCardRequestCount = + contentResponse.downloadResultsList.count { + it.identifier.structureTypeCase == StructureTypeCase.CONCEPT_CARD + } + val explorationRequestCount = + contentResponse.downloadResultsList.count { + it.identifier.structureTypeCase == StructureTypeCase.EXPLORATION + } + val revisionCardPackRequestCount = revisionCardPackRequestResults.size + val conceptCardPackRequestCount = conceptCardPackRequestResults.size + val explorationPackRequestCount = explorationPackRequestResults.size + + // Print diagnostics about the download results. + val successfulResults = contentResponse.downloadResultsList.filter { + it.resultTypeCase != SKIPPED_FROM_FAILURE && it.resultTypeCase != SKIPPED_SHOULD_RETRY + } + val revisionCardPackSuccessResults = + successfulResults.filter { it.resultTypeCase == REVISION_CARD_LANGUAGE_PACK } + val conceptCardPackSuccessResults = + successfulResults.filter { it.resultTypeCase == CONCEPT_CARD_LANGUAGE_PACK } + val explorationPackSuccessResults = + successfulResults.filter { it.resultTypeCase == EXPLORATION_LANGUAGE_PACK } + + val topicSuccessCount = successfulResults.count { it.resultTypeCase == TOPIC_SUMMARY } + val revisionCardSuccessCount = successfulResults.count { it.resultTypeCase == REVISION_CARD } + val conceptCardSuccessCount = successfulResults.count { it.resultTypeCase == CONCEPT_CARD } + val explorationSuccessCount = successfulResults.count { it.resultTypeCase == EXPLORATION } + val revisionCardPackSuccessCount = revisionCardPackSuccessResults.size + val conceptCardPackSuccessCount = conceptCardPackSuccessResults.size + val explorationPackSuccessCount = explorationPackSuccessResults.size + + println("Download results:") + println("- $topicSuccessCount/$topicRequestCount topics succeeded") + println("- $revisionCardSuccessCount/$revisionCardRequestCount revision cards succeeded") + println("- $conceptCardSuccessCount/$conceptCardRequestCount concept cards succeeded") + println("- $explorationSuccessCount/$explorationRequestCount explorations succeeded") + println( + "- $revisionCardPackSuccessCount/$revisionCardPackRequestCount revision card language" + + " packs succeeded" + ) + requestedLanguages.forEach { languageType -> + val results = revisionCardPackRequestResultsByLanguage.getValue(languageType) + val successCount = results.count { + it.resultTypeCase != SKIPPED_FROM_FAILURE && it.resultTypeCase != SKIPPED_SHOULD_RETRY + } + println(" - ${languageType.name}: $successCount/${results.size} succeeded") + } + println( + "- $conceptCardPackSuccessCount/$conceptCardPackRequestCount concept card language packs" + + " succeeded" + ) + requestedLanguages.forEach { languageType -> + val results = conceptCardPackRequestResultsByLanguage.getValue(languageType) + val successCount = results.count { + it.resultTypeCase != SKIPPED_FROM_FAILURE && it.resultTypeCase != SKIPPED_SHOULD_RETRY + } + println(" - ${languageType.name}: $successCount/${results.size} succeeded") + } + println( + "- $explorationPackSuccessCount/$explorationPackRequestCount exploration language packs" + + " succeeded" + ) + requestedLanguages.forEach { languageType -> + val results = explorationPackRequestResultsByLanguage.getValue(languageType) + val successCount = results.count { + it.resultTypeCase != SKIPPED_FROM_FAILURE && it.resultTypeCase != SKIPPED_SHOULD_RETRY + } + println(" - ${languageType.name}: $successCount/${results.size} succeeded") + } + + println() + println("Writing successful results to: ${outputDir.path}/...") + val protoV1Dir = File(outputDir, "protov1").also { it.mkdir() } + val protoV2Dir = File(outputDir, "protov2").also { it.mkdir() } + val textProtoV2Dir = File(protoV2Dir, "textproto").also { it.mkdir() } + val binaryProtoV2Dir = File(protoV2Dir, "binary").also { it.mkdir() } + val textProtoV1Dir = File(protoV1Dir, "textproto").also { it.mkdir() } + val binaryProtoV1Dir = File(protoV1Dir, "binary").also { it.mkdir() } + // NOTE: The 'protov2' values written here are not exactly the app's protov2 definitions (since + // those haven't been defined yet). They're just exact copies of the emulated server's + // responses. + val writeProtoV2AsyncResults = successfulResults.mapNotNull { result -> + when (result.resultTypeCase) { + TOPIC_SUMMARY -> writeProtosAsync(protoV2Dir, result.topicSummary.id, result.topicSummary) + REVISION_CARD -> + writeProtosAsync(protoV2Dir, result.revisionCard.id.collapse(), result.revisionCard) + CONCEPT_CARD -> writeProtosAsync(protoV2Dir, result.conceptCard.skillId, result.conceptCard) + EXPLORATION -> writeProtosAsync(protoV2Dir, result.exploration.id, result.exploration) + REVISION_CARD_LANGUAGE_PACK -> { + writeProtosAsync( + protoV2Dir, + result.revisionCardLanguagePack.id.collapse(), + result.revisionCardLanguagePack + ) + } + CONCEPT_CARD_LANGUAGE_PACK -> { + writeProtosAsync( + protoV2Dir, result.conceptCardLanguagePack.id.collapse(), result.conceptCardLanguagePack + ) + } + EXPLORATION_LANGUAGE_PACK -> { + writeProtosAsync( + protoV2Dir, result.explorationLanguagePack.id.collapse(), result.explorationLanguagePack + ) + } + // The result was a success, but the corresponding ID doesn't correspond to a structure. + SKIPPED_DOES_NOT_EXIST -> null + QUESTION_ID_LIST, QUESTION, QUESTION_LANGUAGE_PACK -> + error("Questions aren't yet supported.") + SKIPPED_SHOULD_RETRY, SKIPPED_FROM_FAILURE, RESULTTYPE_NOT_SET, null -> + error("Encountered unexpected result: $result.") + } + } + + val topicSummaries = successfulResults.filter { + it.resultTypeCase == TOPIC_SUMMARY + }.associate { it.topicSummary.id to it.topicSummary } + val storySummaries = topicSummaries.values.flatMap { it.storySummariesList } + + val revisionCards = successfulResults.filter { + it.resultTypeCase == REVISION_CARD + }.map { it.revisionCard } + val revisionCardPacks = successfulResults.filter { + it.resultTypeCase == REVISION_CARD_LANGUAGE_PACK + }.groupBy( + keySelector = { it.revisionCardLanguagePack.id.id }, + valueTransform = { it.revisionCardLanguagePack } + ) + + val conceptCards = successfulResults.filter { + it.resultTypeCase == CONCEPT_CARD + }.map { it.conceptCard } + val conceptCardPacks = successfulResults.filter { + it.resultTypeCase == CONCEPT_CARD_LANGUAGE_PACK + }.groupBy( + keySelector = { it.conceptCardLanguagePack.id.skillId }, + valueTransform = { it.conceptCardLanguagePack } + ) + + val explorations = successfulResults.filter { + it.resultTypeCase == EXPLORATION + }.map { it.exploration } + val explorationPacks = successfulResults.filter { + it.resultTypeCase == EXPLORATION_LANGUAGE_PACK + }.groupBy( + keySelector = { it.explorationLanguagePack.id.explorationId }, + valueTransform = { it.explorationLanguagePack } + ) + + println() + val imagesDir = File(outputDir, "images").also { it.mkdir() } + val imageReferences = contentResponse.collectImageReferences().distinct() + val baseImageMessage = "Downloading ${imageReferences.size} images" + val extraDotsThatCanFitForImages = CONSOLE_COLUMN_COUNT - baseImageMessage.length + lastDotCount = 0 + print(baseImageMessage) + val images = imageReferences.downloadAllAsync(imagesDir) { finishCount, totalCount -> + val dotCount = (extraDotsThatCanFitForImages * finishCount) / totalCount + val dotsToAdd = dotCount - lastDotCount + if (dotsToAdd > 0) { + print(".".repeat(dotsToAdd)) + lastDotCount = dotCount + } + }.await() + println() + println("Images downloaded to: ${imagesDir.path}/.") + println() + + val imageSuccessCount = images.values.count { it is DownloadedImage.Succeeded } + val imageDuplicationCount = images.values.count { it is DownloadedImage.Duplicated } + val renamedImages = images.values.filterIsInstance() + val convertedImages = images.values.filterIsInstance() + println("$imageSuccessCount/${images.size} images successfully downloaded.") + println("$imageDuplicationCount/${images.size} images were de-duplicated.") + println("${renamedImages.size}/${images.size} images required renaming due to conflicts.") + println("${convertedImages.size}/${images.size} images required repairing from SVG to PNG.") + println() + + if (renamedImages.isNotEmpty()) { + println("Please manually verify the following renamed images:") + val destDir by lazy { File(outputDir, "image_renames").also { it.mkdir() } } + renamedImages.forEach { renamedImage -> + val oldFilename = renamedImage.oldFilename + val newFilename = renamedImage.newFilename + val resolutionDir = File(destDir, oldFilename.substringBeforeLast('.')) + val beforeDir = File(resolutionDir, "before").also { it.mkdirs() } + val afterDir = File(resolutionDir, "after").also { it.mkdirs() } + val newFile = File(imagesDir, newFilename) + val beforeFile = File(beforeDir, oldFilename) + val afterFile = File(afterDir, newFilename) + val oldFileData = renamedImage.readOriginalFileData() + beforeFile.writeBytes(oldFileData) + newFile.copyTo(afterFile, overwrite = true) + val imageUrl = + imageDownloader.computeImageUrl( + renamedImage.imageRef.container.imageContainerType, + renamedImage.imageRef.imageType, + renamedImage.imageRef.container.entityId, + renamedImage.imageRef.filename + ) + println("- Image $oldFilename required repairing via renaming:") + println(" - Before: ${beforeFile.path} (${oldFileData.size} bytes)") + println(" - After: ${afterFile.path} (${newFile.length()} bytes)") + println(" - Image URL: $imageUrl") + println(" - Language: ${renamedImage.imageRef.container.language}") + } + println() + } + + if (convertedImages.isNotEmpty()) { + println("Please manually verify the following converted images:") + val destDir by lazy { File(outputDir, "image_conversions").also { it.mkdir() } } + convertedImages.forEach { convertedImage -> + val oldFilename = convertedImage.imageRef.filename + val newFilename = convertedImage.newFilename + val resolutionDir = File(destDir, oldFilename.substringBeforeLast('.')) + val beforeDir = File(resolutionDir, "before").also { it.mkdirs() } + val afterDir = File(resolutionDir, "after").also { it.mkdirs() } + val beforeImageData = convertedImage.downloadedImageData.toByteArray() + val afterImageData = convertedImage.convertedImageData.toByteArray() + val beforeFile = File(beforeDir, oldFilename).also { it.writeBytes(beforeImageData) } + val afterFile = File(afterDir, newFilename).also { it.writeBytes(afterImageData) } + val imageUrl = + imageDownloader.computeImageUrl( + convertedImage.imageRef.container.imageContainerType, + convertedImage.imageRef.imageType, + convertedImage.imageRef.container.entityId, + convertedImage.imageRef.filename + ) + println("- Image $oldFilename required repairing via conversion:") + println(" - Before: ${beforeFile.path} (${beforeImageData.size} bytes)") + println(" - After: ${afterFile.path} (${afterImageData.size} bytes)") + println(" - Image URL: $imageUrl") + println(" - Language: ${convertedImage.imageRef.container.language}") + println(" - Rendered resolution: ${convertedImage.width}x${convertedImage.height}") + } + println() + } + + val conceptCardImageReplacements = conceptCards.associate { dto -> + dto.skillId to images.computeReplacements(ImageContainerType.SKILL, dto.skillId) + } + val classroomImageReplacements = listResponse.classroomsList.associate { classroom -> + classroom.id to images.computeReplacements(ImageContainerType.CLASSROOM, classroom.id) + } + val writeProtoV1AsyncResults = topicSummaries.map { (topicId, topicSummary) -> + val imageReplacements = images.computeReplacements(ImageContainerType.TOPIC, topicId) + writeProtosAsync(protoV1Dir, topicId, topicSummary.convertToTopicRecord(imageReplacements)) + } + upcomingTopics.map { upcomingTopic -> + val imageReplacements = images.computeReplacements(ImageContainerType.TOPIC, upcomingTopic.id) + writeProtosAsync( + protoV1Dir, upcomingTopic.id, upcomingTopic.convertToTopicRecord(imageReplacements) + ) + } + storySummaries.map { storySummary -> + val imageReplacements = images.computeReplacements(ImageContainerType.STORY, storySummary.id) + writeProtosAsync( + protoV1Dir, storySummary.id, storySummary.convertToStoryRecord(imageReplacements) + ) + } + revisionCards.map { revisionCard -> + val imageReplacements = + images.computeReplacements(ImageContainerType.TOPIC, revisionCard.id.topicId) + val topicSummary = topicSummaries.getValue(revisionCard.id.topicId) + val subtopicSummary = + topicSummary.subtopicSummariesList.find { it.index == revisionCard.id.subtopicIndex } + ?: error("Could not find subtopic summary for revision card: ${revisionCard.id}.") + // TODO: The listOf() default here allows cards to have no translations. + val packs = revisionCardPacks[revisionCard.id] ?: listOf() + writeProtosAsync( + protoV1Dir, + revisionCard.id.collapse(), + revisionCard.convertToSubtopicRecord(imageReplacements, subtopicSummary, packs) + ) + } + writeProtosAsync( + protoV1Dir, + baseName = "skills", + convertToConceptCardList( + conceptCardImageReplacements, + // TODO: The listOf() default here allows cards to have no translations. + conceptCards.map { it to (conceptCardPacks[it.skillId] ?: listOf()) } + ) + ) + explorations.map { exp -> + val imageReplacements = images.computeReplacements(ImageContainerType.EXPLORATION, exp.id) + // TODO: The listOf() default here allows some explorations to have no translations. + val packs = explorationPacks[exp.id] ?: listOf() + writeProtosAsync(protoV1Dir, exp.id, exp.convertToExploration(imageReplacements, packs)) + } + writeProtosAsync( + protoV1Dir, + baseName = "classrooms", + listResponse.classroomsList.convertToClassroomList( + topicSummaries.values, classroomImageReplacements + ) + ) + + // Wait for all proto writes to finish. + (writeProtoV2AsyncResults + writeProtoV1AsyncResults).awaitAll() + println("Written proto locations:") + println("- Proto v2 text protos can be found in: ${textProtoV2Dir.path}") + println("- Proto v2 binary protos can be found in: ${binaryProtoV2Dir.path}") + println("- Proto v1 text protos can be found in: ${textProtoV1Dir.path}") + println("- Proto v1 binary protos can be found in: ${binaryProtoV1Dir.path}") + + // TODO: Reenable analysis for all structures when non-explorations are expected to be fully translated (or a version can be downloaded by the script). + val analyzer = CompatibilityAnalyzer(requestedLanguages + setOf(defaultLanguage)) + topicSummaries.values.forEach(analyzer::track) + upcomingTopics.forEach(analyzer::track) +// revisionCards.forEach(analyzer::track) + explorations.forEach(analyzer::track) +// conceptCards.forEach(analyzer::track) +// revisionCardPacks.values.flatten().forEach(analyzer::track) +// conceptCardPacks.values.flatten().forEach(analyzer::track) + explorationPacks.values.flatten().forEach(analyzer::track) + + val issues = analyzer.scanForIssues().sorted() + val imageInvalidExtIssues = + issues.filterIsInstance() + val imageInconsistencyIssues = + issues.filterIsInstance() + val htmlInvalidTagIssues = + issues.filterIsInstance() + val translationIssues = + issues.filterIsInstance() + println() + println("${issues.size} issues were found during import. High-level break-down:") + println("- ${imageInvalidExtIssues.size}/${issues.size} correspond to invalid image extensions") + println( + "- ${imageInconsistencyIssues.size}/${issues.size} correspond to images missing across" + + " translations" + ) + println( + "- ${htmlInvalidTagIssues.size}/${issues.size} correspond to invalid tags found in HTML" + ) + println("- ${translationIssues.size}/${issues.size} correspond to missing translations") + println() + println("Images with invalid extensions:") + imageInvalidExtIssues.groupBy { it.container }.forEach { (container, issues) -> + println("- Within ${container.referenceString}:") + issues.forEach { issue -> + println( + " - Image ${issue.filename} (language: ${issue.language.name}) has invalid extension:" + + " ${issue.invalidExtension}" + ) + } + } + println() + println("Images missing across translations: (Hidden)") +// imageInconsistencyIssues.groupBy { it.container }.forEach { (container, issues) -> +// println("- Within ${container.referenceString}:") +// issues.forEach { issue -> +// val missingLangs = issue.missingLanguages.joinToString { it.name } +// val presentLangs = issue.presentLanguages.joinToString { it.name } +// println(" - Image ${issue.filename} exists in languages: $presentLangs, but is missing in: $missingLangs") +// } +// } + println() + println("HTML strings with invalid tags:") + htmlInvalidTagIssues.groupBy { it.text.container }.forEach { (container, issues) -> + println("- Within ${container.referenceString}:") + issues.groupBy { it.language }.forEach { (language, perLangIssues) -> + println(" - For language ${language.name}:") + perLangIssues.forEach { issue -> + println( + " - Text with content ID ${issue.text.contentId} has references tag:" + + " ${issue.invalidTag}" + ) + } + } + } + println() + println("Strings missing translations:") + translationIssues.groupBy { it.text.container }.forEach { (container, issues) -> + println("- Within ${container.referenceString}:") + issues.forEach { issue -> + val missingLangs = issue.missingLanguages.joinToString { it.name } + val presentLangs = issue.presentLanguages.joinToString { it.name } + println( + " - Text with content ID ${issue.text.contentId} exists in languages: $presentLangs," + + " but is missing in: $missingLangs" + ) + } + } + + val imageDownloadFailures = images.values.filterIsInstance() + if (imageDownloadFailures.isNotEmpty()) { + println() + println("Images that failed to download:") + imageDownloadFailures.forEach { downloadedImage -> + val reference = downloadedImage.imageRef + val imageUrl = + imageDownloader.computeImageUrl( + reference.container.imageContainerType, + reference.imageType, + reference.container.entityId, + reference.filename + ) + val language = reference.container.language + println("- Image failed to download (could not find image, language: $language): $imageUrl") + } + } + + if (renamedImages.isNotEmpty() || convertedImages.isNotEmpty()) { + println("WARNING: Images needed to be auto-fixed. Please verify that they are correct") + println("(look at above output for specific images that require verification).") + } + +// val translationMetrics = analyzer.computeTranslationsUsageReport() +// val voiceoverMetrics = analyzer.computeVoiceoversUsageReport() +// println("#".repeat(CONSOLE_COLUMN_COUNT)) +// println() +// println("Translation statistics:") +// translationMetrics.forEach { usageReport -> +// println("- For ${usageReport.container.referenceString}:") +// usageReport.languageUsage.forEach { (language, usage) -> +// println(" - Language ${language.name} has ${usage.usageString} strings translated (${usage.percentageString})") +// } +// } +// println() +// println("Voiceover statistics:") +// voiceoverMetrics.forEach { usageReport -> +// println("- For ${usageReport.container.referenceString}:") +// usageReport.languageUsage.forEach { (language, usage) -> +// println(" - Language ${language.name} has ${usage.usageString} content strings subtitled by audio voiceovers (${usage.percentageString})") +// } +// } +// println() +// println("#".repeat(CONSOLE_COLUMN_COUNT)) +// println() + } + + private fun createDownloadContentRequest( + topicSummaries: List, + defaultLanguage: LanguageType, + requestedLanguages: List + ): TopicContentRequestDto { + return TopicContentRequestDto.newBuilder().apply { + val allIdentifiers = topicSummaries.flatMap { topicSummary -> + generateIdentifiersToDownloadTopic(topicSummary, defaultLanguage, requestedLanguages) + } + + protoVersion = ProtoVersionProvider.createLatestTopicContentProtoVersion() + clientContext = CLIENT_CONTEXT + addAllIdentifiers(allIdentifiers.distinct()) + requestedMaxPayloadSizeBytes = 0 // This isn't used for local emulation. + }.build() + } + + private fun generateIdentifiersToDownloadTopic( + topicSummary: DownloadableTopicSummaryDto, + defaultLanguage: LanguageType, + requestedLanguages: List + ): List { + return generateIdentifiersToDownloadRevisionCards( + topicSummary.id, + topicSummary.subtopicSummariesList, + defaultLanguage, + requestedLanguages + ) + generateIdentifiersToDownloadExplorations( + topicSummary, defaultLanguage, requestedLanguages + ) + generateIdentifiersToDownloadConceptCards( + topicSummary, defaultLanguage, requestedLanguages + ) + topicSummary.toStructureIdentifier(topicSummary.contentVersion) { setTopicSummaryId(it.id) } + } + + private fun generateIdentifiersToDownloadRevisionCards( + topicId: String, + subtopicSummaries: List, + defaultLanguage: LanguageType, + requestedLanguages: List + ): List { + return subtopicSummaries.associateBy { subtopicSummary -> + createSubtopicId(topicId, subtopicSummary.index) + }.flatMap { (subtopicId, subtopicSummary) -> + generateIdentifiersToDownloadStructure( + subtopicId, + subtopicSummary.contentVersion, + defaultLanguage, + requestedLanguages, + ::createLocalizedRevisionCardId, + setIdForStruct = DownloadReqStructIdDtoBuilder::setRevisionCard, + setIdForLangPack = DownloadReqStructIdDtoBuilder::setRevisionCardLanguagePack + ) + } + } + + private fun generateIdentifiersToDownloadExplorations( + downloadableTopicSummary: DownloadableTopicSummaryDto, + defaultLanguage: LanguageType, + requestedLanguages: List + ): List { + return downloadableTopicSummary.storySummariesList.flatMap { storySummary -> + storySummary.chaptersList + }.associateBy { chapterSummary -> + chapterSummary.explorationId + }.flatMap { (explorationId, chapterSummary) -> + generateIdentifiersToDownloadStructure( + explorationId, + chapterSummary.contentVersion, + defaultLanguage, + requestedLanguages, + ::createLocalizedExplorationId, + setIdForStruct = DownloadReqStructIdDtoBuilder::setExploration, + setIdForLangPack = DownloadReqStructIdDtoBuilder::setExplorationLanguagePack + ) + } + } + + private fun generateIdentifiersToDownloadConceptCards( + downloadableTopicSummary: DownloadableTopicSummaryDto, + defaultLanguage: LanguageType, + requestedLanguages: List + ): List { + return downloadableTopicSummary.referencedSkillsList.associateBy { skillSummary -> + skillSummary.id + }.flatMap { (skillId, skillSummary) -> + generateIdentifiersToDownloadStructure( + skillId, + skillSummary.contentVersion, + defaultLanguage, + requestedLanguages, + ::createLocalizedConceptCardId, + setIdForStruct = DownloadReqStructIdDtoBuilder::setConceptCard, + setIdForLangPack = DownloadReqStructIdDtoBuilder::setConceptCardLanguagePack + ) + } + } + + private fun generateIdentifiersToDownloadStructure( + id: I, + contentVersion: Int, + defaultLanguage: LanguageType, + requestedLanguages: List, + createLocalizedId: (I, LanguageType) -> L, + setIdForStruct: DownloadReqStructIdDtoBuilder.(L) -> DownloadReqStructIdDtoBuilder, + setIdForLangPack: DownloadReqStructIdDtoBuilder.(L) -> DownloadReqStructIdDtoBuilder + ): List { + return requestedLanguages.map { language -> + createLocalizedId(id, language).toStructureIdentifier(contentVersion, setIdForLangPack) + } + createLocalizedId(id, defaultLanguage).toStructureIdentifier(contentVersion, setIdForStruct) + } + + private fun writeProtosAsync( + protoV2Dir: File, + baseName: String, + message: Message + ): Deferred { + val textProtoV2Dir = File(protoV2Dir, "textproto") + val binaryProtoV2Dir = File(protoV2Dir, "binary") + return CoroutineScope(coroutineDispatcher).async { + writeTextProto(textProtoV2Dir, baseName, message) + writeBinaryProto(binaryProtoV2Dir, baseName, message) + } + } + + private suspend fun writeTextProto(destDir: File, baseName: String, message: Message) { + withContext(Dispatchers.IO) { + File(destDir, "$baseName.textproto") + .outputStream() + .bufferedWriter() + .use { textFormat.escapingNonAscii(false).print(message, it) } + } + } + + private suspend fun writeBinaryProto(destDir: File, baseName: String, message: Message) { + withContext(Dispatchers.IO) { + File(destDir, "$baseName.pb").outputStream().use(message::writeTo) + } + } + + private fun Collection.downloadAllAsync( + destDir: File, + reportProgress: (Int, Int) -> Unit + ): Deferred> { + check(destDir.deleteRecursively() && destDir.mkdir()) { + "Failed to clear & recreate image destination dir: ${destDir.path}." + } + val totalCount = size + val channel = Channel() + channel.consumeAsFlow().withIndex().onEach { (index, _) -> + reportProgress(index + 1, totalCount) + }.launchIn(CoroutineScope(coroutineDispatcher)) + return CoroutineScope(coroutineDispatcher).async { + mapIndexed { index, reference -> + reference.downloadAsync(destDir, index, channel) + }.awaitAll().groupBy { it.imageRef }.mapValues { (_, matches) -> + matches.single() + }.also { channel.close() } + } + } + + private fun ImageReference.downloadAsync( + destDir: File, + index: Int, + reportProgressChannel: SendChannel + ): Deferred { + val reference = this + return CoroutineScope(coroutineDispatcher).async { + imageDownloader.retrieveImageContentAsync( + container.imageContainerType, imageType, container.entityId, filename + ).await()?.let { imageData -> + withContext(Dispatchers.IO) { + when (val conv = imageRepairer.convertToPng(filename, imageData.decodeToString())) { + ImageRepairer.RepairedImage.NoRepairNeeded -> { + val imageFile = File(destDir, filename) + when { + !imageFile.exists() -> { + memoizedLoadedImageData[imageFile] = imageData + imageFile.writeBytes(imageData) + DownloadedImage.Succeeded(reference) + } + imageFile.isImageFileSameAs(imageData) -> DownloadedImage.Duplicated(reference) + else -> { + val newFile = computeNewUniqueFile(destDir, imageFile) + memoizedLoadedImageData[newFile] = imageData + newFile.writeBytes(imageData) + DownloadedImage.Renamed.ExistingFile( + reference, oldFile = imageFile, newFilename = newFile.name + ) + } + } + } + is ImageRepairer.RepairedImage.RenderedSvg -> { + val nameWithoutExt = filename.substringBeforeLast('.') + val expectedNewImageFile = File(destDir, "$nameWithoutExt.png") + val newImageFile = if (expectedNewImageFile.exists()) { + if (expectedNewImageFile.isImageFileSameAs(conv.pngContents.toByteArray())) { + // This is a rename since the original file is being converted from SVG. + return@withContext DownloadedImage.Renamed.ConvertedFile( + reference, + oldFilename = filename, + oldFileData = imageData.toList(), + newFilename = expectedNewImageFile.name + ) + } else computeNewUniqueFile(destDir, expectedNewImageFile) + } else expectedNewImageFile + val newImageData = conv.pngContents.toByteArray() + memoizedLoadedImageData[newImageFile] = newImageData + newImageFile.writeBytes(newImageData) + DownloadedImage.ConvertedSvgToPng( + reference, + newFilename = newImageFile.name, + imageData.toList(), + conv.pngContents, + conv.width, + conv.height + ) + } + } + } + }.also { reportProgressChannel.send(index) } ?: DownloadedImage.FailedCouldNotFind(reference) + } + } + + private fun File.isImageFileSameAs(imageData: ByteArray): Boolean { + val imageDataToCheck = memoizedLoadedImageData.getValue(this) + + // First, perform a byte-equality check. + if (imageDataToCheck.toList() == imageData.toList()) return true + + // Second, perform an image-equality check (exact match). + return imageRepairer.areEqualImages(extension, imageDataToCheck, imageData) + } + + private fun computeNewUniqueFile(destDir: File, imageFile: File): File { + val imageBaseFilename = imageFile.nameWithoutExtension + val copyCount = destDir.listFiles()?.count { + it.nameWithoutExtension.startsWith(imageBaseFilename) + }?.plus(1) ?: 0 // Zero shouldn't actually happen in practice. + val newFilename = "${imageBaseFilename}_$copyCount.${imageFile.extension}" + return File(destDir, newFilename) + } + + private sealed class DownloadedImage { + abstract val imageRef: ImageReference + + data class Succeeded(override val imageRef: ImageReference) : DownloadedImage() + + data class Duplicated(override val imageRef: ImageReference) : DownloadedImage() + + sealed class Renamed : DownloadedImage() { + abstract val oldFilename: String + abstract val newFilename: String + + abstract fun readOriginalFileData(): ByteArray + + data class ExistingFile( + override val imageRef: ImageReference, + val oldFile: File, + override val newFilename: String + ) : Renamed() { + override val oldFilename: String get() = oldFile.name + override fun readOriginalFileData(): ByteArray = oldFile.readBytes() + } + + data class ConvertedFile( + override val imageRef: ImageReference, + override val oldFilename: String, + val oldFileData: List, + override val newFilename: String + ) : Renamed() { + override fun readOriginalFileData(): ByteArray = oldFileData.toByteArray() + } + } + + data class ConvertedSvgToPng( + override val imageRef: ImageReference, + val newFilename: String, + val downloadedImageData: List, + val convertedImageData: List, + val width: Int, + val height: Int + ) : DownloadedImage() + + data class FailedCouldNotFind(override val imageRef: ImageReference) : DownloadedImage() + } + + private fun shutdownBlocking() { + coroutineDispatcher.close() + threadPool.tryShutdownFully(timeout = 5, unit = TimeUnit.SECONDS) + } + + private data class ImageContainer( + val imageContainerType: ImageContainerType, + val entityId: String, + val language: LanguageType + ) + + private data class ImageReference( + val container: ImageContainer, + val imageType: ImageType, + val filename: String + ) + + private fun Map.computeReplacements( + imageContainerType: ImageContainerType, + entityId: String + ): Map { + return filterKeys { ref -> + ref.container.imageContainerType == imageContainerType && ref.container.entityId == entityId + }.mapValues { (_, image) -> + when (image) { + is DownloadedImage.ConvertedSvgToPng -> image.imageRef.filename to image.newFilename + is DownloadedImage.Renamed -> image.oldFilename to image.newFilename + is DownloadedImage.Duplicated, is DownloadedImage.FailedCouldNotFind, + is DownloadedImage.Succeeded -> null + } + }.values.filterNotNull().groupBy { (oldFilename, _) -> + oldFilename + }.mapValues { (oldFilename, matches) -> + val uniqueReplacements = matches.mapTo(mutableSetOf()) { (_, replacement) -> replacement } + uniqueReplacements.singleOrNull() + ?: error("Multiple files correspond to image: $oldFilename: $uniqueReplacements.") + }.also { imageReplacements -> + val cyclicKeys = imageReplacements.keys.filter { it in imageReplacements.values } + check(cyclicKeys.isEmpty()) { + "Cycle(s) found in image replacements map: $cyclicKeys." + } + } + } + + private class CompatibilityAnalyzer(private val expectedLanguages: Set) { + private val texts by lazy { mutableListOf() } + private val localizations by lazy { mutableListOf() } + + fun track(dto: DownloadableTopicSummaryDto) { + val container = Container.Topic(dto.id) + if (dto.hasName()) texts += TextReference.Name(container, dto.name.contentId) + if (dto.hasDescription()) { + texts += TextReference.Description(container, dto.description.contentId) + } + if (dto.hasLocalizations()) track(container, dto.localizations) + dto.storySummariesList.forEach { track(container, it) } + dto.referencedSkillsList.forEach { track(container, it) } + } + + fun track(dto: UpcomingTopicSummaryDto) { + val container = Container.Topic(dto.id) + if (dto.hasName()) texts += TextReference.Name(container, dto.name.contentId) + if (dto.hasDescription()) { + texts += TextReference.Description(container, dto.description.contentId) + } + if (dto.hasLocalizations()) track(container, dto.localizations) + } + + fun track(dto: RevisionCardDto) { + val topic = Container.Topic(dto.id.topicId) + val container = Container.RevisionCard(topic, dto.id.subtopicIndex) + if (dto.hasTitle()) texts += TextReference.Title(container, dto.title.contentId) + if (dto.hasContent()) texts += TextReference.Content(container, dto.content.contentId) + if (dto.hasDefaultLocalization()) track(container, dto.defaultLocalization) + } + + fun track(dto: ConceptCardDto) { + val container = Container.ConceptCard(dto.skillId) + if (dto.hasDescription()) { + texts += TextReference.Description(container, dto.description.contentId) + } + if (dto.hasExplanation()) { + texts += TextReference.Explanation(container, dto.explanation.contentId) + } + if (dto.hasDefaultLocalization()) track(container, dto.defaultLocalization) + dto.workedExamplesList.forEachIndexed { index, example -> track(container, index, example) } + } + + fun track(dto: ExplorationDto) { + val container = Container.Exploration(dto.id) + if (dto.hasTitle()) texts += TextReference.Title(container, dto.title.contentId) + if (dto.hasDefaultLocalization()) track(container, dto.defaultLocalization) + dto.statesMap.forEach { (name, state) -> track(container, name, state) } + } + + fun track(dto: RevisionCardLanguagePackDto) { + val topic = Container.Topic(dto.id.id.topicId) + val container = Container.RevisionCard(topic, dto.id.id.subtopicIndex) + if (dto.hasLocalization()) track(container, dto.localization) + } + + fun track(dto: ConceptCardLanguagePackDto) { + val container = Container.ConceptCard(dto.id.skillId) + if (dto.hasLocalization()) track(container, dto.localization) + } + + fun track(dto: ExplorationLanguagePackDto) { + val container = Container.Exploration(dto.id.explorationId) + if (dto.hasLocalization()) track(container, dto.localization) + } + + fun scanForIssues(): List { + return scanForInvalidExtensions() + + scanForImageInconsistencies() + + scanForHtmlInvalidTags() + + scanForTextMissingTranslations() + } + + private fun scanForInvalidExtensions(): List { + return localizations.filterIsInstance().flatMap { images -> + images.filenames.filter { it.endsWith(".gif", ignoreCase = true) }.map { filename -> + Issue.ImageHasInvalidExtension( + images.container, images.language, filename, invalidExtension = "gif" + ) + } + } + localizations.filterIsInstance().mapNotNull { thumbnail -> + thumbnail.thumbnailFilename.takeIf { + it.endsWith(".gif", ignoreCase = true) + }?.let { filename -> + Issue.ImageHasInvalidExtension( + thumbnail.container, thumbnail.language, filename, invalidExtension = "gif" + ) + } + } + } + + private fun scanForImageInconsistencies(): List { + return localizations.filterIsInstance().flatMap { images -> + images.filenames.map { filename -> + (images.container to filename) to images.language + } + }.groupBy( + keySelector = { (key, _) -> key }, + valueTransform = { (_, value) -> value } + ).map { (key, languages) -> + val (container, filename) = key + val presentLanguages = languages.toSet() + Issue.ImageInconsistencies( + container, filename, presentLanguages, expectedLanguages - presentLanguages + ) + } + } + + private fun scanForHtmlInvalidTags(): List { + val invalidTags = listOf( + "oppia-noninteractive-link", + "oppia-noninteractive-tabs", + "oppia-noninteractive-video", + "oppia-noninteractive-collapsible" + ) + val textsByContentId = texts.groupBy { it.container.findRoot() to it.contentId } + return localizations.filterIsInstance().flatMap { translations -> + translations.translations.flatMap { translation -> + translation.htmls.flatMap { html -> + invalidTags.map { invalidTag -> + (html to invalidTag).takeIf { invalidTag in html } + } + }.filterNotNull().map { (html, invalidTag) -> + // Exactly one text reference should correspond to this content ID among all in the root + // container. + Issue.HtmlHasInvalidTag( + translations.language, + textsByContentId.getValue( + translations.container.findRoot() to translation.contentId + ).single(), + html, + invalidTag + ) + } + } + } + } + + private fun scanForTextMissingTranslations(): List { + val allTranslations = localizations.filterIsInstance() + val textLanguages = allTranslations.flatMap { translations -> + translations.translations.map { + (translations.container.findRoot() to it.contentId) to translations + } + }.groupBy( + keySelector = { (key, _) -> key }, + valueTransform = { (_, value) -> value.language } + ) + return texts.mapNotNull { text -> + val languages = + textLanguages[text.container.findRoot() to text.contentId]?.toSet() ?: emptySet() + val missingLanguages = expectedLanguages - languages + if (missingLanguages.isNotEmpty()) { + Issue.TextMissingTranslations(text, languages, missingLanguages) + } else null + } + } + + fun computeTranslationsUsageReport(): List { + val allTranslations = localizations.filterIsInstance() + val textLanguages = allTranslations.flatMap { translations -> + translations.translations.map { + (translations.container.findRoot() to it.contentId) to translations + } + }.groupBy( + keySelector = { (key, _) -> key }, + valueTransform = { (_, value) -> value.language } + ) + val textsByContainer = texts.groupBy { it.container } + return texts.flatMap { text -> + val translations = textLanguages[text.container.findRoot() to text.contentId]?.map { + it to text + } ?: emptyList() + translations + }.groupBy( + keySelector = { (language, text) -> text.container to language }, + valueTransform = { (_, text) -> text.contentId } + ).mapValues { (key, contentIds) -> + val (container, _) = key + MetricsReport.Usage( + usedCount = contentIds.size, totalCount = textsByContainer.getValue(container).size + ) + }.entries.groupBy( + keySelector = { (key, _) -> + val (container, language) = key + container.findRoot() to language + }, + valueTransform = { (_, metrics) -> metrics } + ).mapValues { (_, metrics) -> + metrics.reduce(MetricsReport.Usage::combineWith) + }.entries.groupBy( + keySelector = { (key, _) -> + val (container, _) = key + container + }, + valueTransform = { (key, combinedMetrics) -> + val (_, language) = key + language to combinedMetrics + } + ).entries.map { (container, values) -> + MetricsReport.TranslationUsage(container, values.toMap()) + } + } + + fun computeVoiceoversUsageReport(): List { + val allVoiceovers = localizations.filterIsInstance() + val voiceoverLanguages = allVoiceovers.flatMap { voiceovers -> + voiceovers.contentIds.map { (voiceovers.container.findRoot() to it) to voiceovers } + }.groupBy( + keySelector = { (key, _) -> key }, + valueTransform = { (_, value) -> value.language } + ) + val textsByContainer = texts.groupBy { it.container } + return texts.flatMap { text -> + val voiceovers = voiceoverLanguages[text.container.findRoot() to text.contentId]?.map { + it to text + } ?: emptyList() + voiceovers + }.groupBy( + keySelector = { (language, text) -> text.container to language }, + valueTransform = { (_, text) -> text.contentId } + ).mapValues { (key, contentIds) -> + val (container, _) = key + MetricsReport.Usage( + usedCount = contentIds.size, totalCount = textsByContainer.getValue(container).size + ) + }.entries.groupBy( + keySelector = { (key, _) -> + val (container, language) = key + container.findRoot() to language + }, + valueTransform = { (_, metrics) -> metrics } + ).mapValues { (_, metrics) -> + metrics.reduce(MetricsReport.Usage::combineWith) + }.entries.groupBy( + keySelector = { (key, _) -> + val (container, _) = key + container + }, + valueTransform = { (key, combinedMetrics) -> + val (_, language) = key + language to combinedMetrics + } + ).entries.map { (container, values) -> + MetricsReport.VoiceoverUsage(container, values.toMap()) + } + } + + private fun track(topic: Container.Topic, dto: StorySummaryDto) { + val container = Container.Story(topic, dto.id) + if (dto.hasTitle()) texts += TextReference.Title(container, dto.title.contentId) + if (dto.hasDescription()) { + texts += TextReference.Description(container, dto.description.contentId) + } + if (dto.hasLocalizations()) track(container, dto.localizations) + dto.chaptersList.forEach { track(container, it) } + } + + private fun track(story: Container.Story, dto: ChapterSummaryDto) { + val container = Container.Chapter(story, dto.explorationId) + if (dto.hasTitle()) texts += TextReference.Title(container, dto.title.contentId) + if (dto.hasDescription()) { + texts += TextReference.Description(container, dto.description.contentId) + } + if (dto.hasLocalizations()) track(container, dto.localizations) + } + + private fun track(topic: Container.Topic, dto: SkillSummaryDto) { + val container = Container.Skill(topic, dto.id) + if (dto.hasName()) texts += TextReference.Name(container, dto.name.contentId) + if (dto.hasLocalizations()) track(container, dto.localizations) + } + + private fun track(conceptCard: Container.ConceptCard, index: Int, dto: WorkedExampleDto) { + val container = Container.WorkedExample(conceptCard, index) + if (dto.hasQuestion()) texts += TextReference.Question(container, dto.question.contentId) + if (dto.hasExplanation()) { + texts += TextReference.Explanation(container, dto.explanation.contentId) + } + } + + private fun track(exploration: Container.Exploration, name: String, dto: StateDto) { + val container = Container.State(exploration, name) + if (dto.hasContent()) texts += TextReference.Content(container, dto.content.contentId) + when (dto.interaction.interactionTypeCase) { + InteractionTypeCase.CONTINUE_INSTANCE -> { + val interaction = dto.interaction.continueInstance + val args = interaction.customizationArgs + if (args.hasButtonText()) { + texts += TextReference.CustomizationArg.ButtonText(container, args.buttonText.contentId) + } +// if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + } + InteractionTypeCase.FRACTION_INPUT -> { + val interaction = dto.interaction.fractionInput + val args = interaction.customizationArgs + val solution = interaction.solution + if (args.hasPlaceholder()) { + texts += TextReference.CustomizationArg.Placeholder( + container, args.placeholder.contentId + ) + } + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + if (interaction.hasSolution() && solution.hasBaseSolution()) { + track(container, solution.baseSolution) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { index, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, index, answerGroup.baseAnswerGroup) + } + } + } + InteractionTypeCase.ITEM_SELECTION_INPUT -> { + val interaction = dto.interaction.itemSelectionInput + val args = interaction.customizationArgs + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + args.choicesList.forEachIndexed { index, choice -> + texts += TextReference.CustomizationArg.Choice(container, choice.contentId, index) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { agIndex, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, agIndex, answerGroup.baseAnswerGroup) + } + answerGroup.ruleSpecsList.forEachIndexed { rsIndex, ruleSpecDto -> + when (ruleSpecDto.ruleTypeCase) { + ItemSelRuleSpecDto.RuleTypeCase.EQUALS -> { + val ruleSpec = ruleSpecDto.equals + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + ItemSelRuleSpecDto.RuleTypeCase.CONTAINS_AT_LEAST_ONE_OF -> { + val ruleSpec = ruleSpecDto.containsAtLeastOneOf + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + ItemSelRuleSpecDto.RuleTypeCase.DOES_NOT_CONTAIN_AT_LEAST_ONE_OF -> { + val ruleSpec = ruleSpecDto.doesNotContainAtLeastOneOf + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + ItemSelRuleSpecDto.RuleTypeCase.IS_PROPER_SUBSET_OF -> { + val ruleSpec = ruleSpecDto.isProperSubsetOf + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + ItemSelRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $ruleSpecDto.") + } + } + } + } + InteractionTypeCase.MULTIPLE_CHOICE_INPUT -> { + val interaction = dto.interaction.multipleChoiceInput + val args = interaction.customizationArgs +// if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + args.choicesList.forEachIndexed { index, choice -> + texts += TextReference.CustomizationArg.Choice(container, choice.contentId, index) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { index, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, index, answerGroup.baseAnswerGroup) + } + } + } + InteractionTypeCase.NUMERIC_INPUT -> { + val interaction = dto.interaction.numericInput + val solution = interaction.solution + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + if (interaction.hasSolution() && solution.hasBaseSolution()) { + track(container, solution.baseSolution) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { index, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, index, answerGroup.baseAnswerGroup) + } + } + } + InteractionTypeCase.TEXT_INPUT -> { + val interaction = dto.interaction.textInput + val args = interaction.customizationArgs + val solution = interaction.solution + if (args.hasPlaceholder()) { + texts += TextReference.CustomizationArg.Placeholder( + container, args.placeholder.contentId + ) + } + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + if (interaction.hasSolution() && solution.hasBaseSolution()) { + track(container, solution.baseSolution) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { agIndex, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, agIndex, answerGroup.baseAnswerGroup) + } + answerGroup.ruleSpecsList.forEachIndexed { rsIndex, ruleSpecDto -> + when (ruleSpecDto.ruleTypeCase) { + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.EQUALS -> { + val ruleSpec = ruleSpecDto.equals + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.STARTS_WITH -> { + val ruleSpec = ruleSpecDto.startsWith + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.CONTAINS -> { + val ruleSpec = ruleSpecDto.contains + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.FUZZY_EQUALS -> { + val ruleSpec = ruleSpecDto.fuzzyEquals + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $ruleSpecDto.") + } + } + } + } + InteractionTypeCase.DRAG_AND_DROP_SORT_INPUT -> { + val interaction = dto.interaction.dragAndDropSortInput + val args = interaction.customizationArgs + val solution = interaction.solution + args.choicesList.forEachIndexed { index, choice -> + texts += TextReference.CustomizationArg.Choice(container, choice.contentId, index) + } + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + if (interaction.hasSolution() && solution.hasBaseSolution()) { + track(container, solution.baseSolution) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { agIndex, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, agIndex, answerGroup.baseAnswerGroup) + } + answerGroup.ruleSpecsList.forEachIndexed { rsIndex, ruleSpecDto -> + when (ruleSpecDto.ruleTypeCase) { + DragDropSortRuleSpecDto.RuleTypeCase.IS_EQUAL_TO_ORDERING -> { + val ruleSpec = ruleSpecDto.isEqualToOrdering + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + IS_EQUAL_TO_ORDERING_WITH_ONE_ITEM_AT_INCORRECT_POSITION -> { + val ruleSpec = ruleSpecDto.isEqualToOrderingWithOneItemAtIncorrectPosition + if (ruleSpec.hasInput()) track(container, agIndex, rsIndex, ruleSpec.input) + } + DragDropSortRuleSpecDto.RuleTypeCase.HAS_ELEMENT_X_AT_POSITION_Y -> { + val ruleSpec = ruleSpecDto.hasElementXAtPositionY + if (ruleSpec.hasElement()) { + track(container, agIndex, rsIndex, ruleSpec.element, context = "input") + } + } + DragDropSortRuleSpecDto.RuleTypeCase.HAS_ELEMENT_X_BEFORE_ELEMENT_Y -> { + val ruleSpec = ruleSpecDto.hasElementXBeforeElementY + if (ruleSpec.hasConsideredElement()) { + track( + container, + agIndex, + rsIndex, + ruleSpec.consideredElement, + context = "consideredElement" + ) + } + if (ruleSpec.hasLaterElement()) { + track( + container, agIndex, rsIndex, ruleSpec.laterElement, context = "laterElement" + ) + } + } + DragDropSortRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $ruleSpecDto.") + } + } + } + } + InteractionTypeCase.IMAGE_CLICK_INPUT -> { + val interaction = dto.interaction.imageClickInput + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { index, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, index, answerGroup.baseAnswerGroup) + } + } + } + InteractionTypeCase.RATIO_EXPRESSION_INPUT -> { + val interaction = dto.interaction.ratioExpressionInput + val args = interaction.customizationArgs + val solution = interaction.solution + if (args.hasPlaceholder()) { + texts += TextReference.CustomizationArg.Placeholder( + container, args.placeholder.contentId + ) + } + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + if (interaction.hasSolution() && solution.hasBaseSolution()) { + track(container, solution.baseSolution) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { index, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, index, answerGroup.baseAnswerGroup) + } + } + } + InteractionTypeCase.ALGEBRAIC_EXPRESSION_INPUT -> { + val interaction = dto.interaction.algebraicExpressionInput + val solution = interaction.solution + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + if (interaction.hasSolution() && solution.hasBaseSolution()) { + track(container, solution.baseSolution) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { index, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, index, answerGroup.baseAnswerGroup) + } + } + } + InteractionTypeCase.MATH_EQUATION_INPUT -> { + val interaction = dto.interaction.mathEquationInput + val solution = interaction.solution + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + if (interaction.hasSolution() && solution.hasBaseSolution()) { + track(container, solution.baseSolution) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { index, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, index, answerGroup.baseAnswerGroup) + } + } + } + InteractionTypeCase.NUMERIC_EXPRESSION_INPUT -> { + val interaction = dto.interaction.numericExpressionInput + val args = interaction.customizationArgs + val solution = interaction.solution + if (args.hasPlaceholder()) { + texts += TextReference.CustomizationArg.Placeholder( + container, args.placeholder.contentId + ) + } + if (interaction.hasDefaultOutcome()) track(container, interaction.defaultOutcome) + if (interaction.hasSolution() && solution.hasBaseSolution()) { + track(container, solution.baseSolution) + } + interaction.hintsList.forEachIndexed { index, hint -> track(container, index, hint) } + interaction.answerGroupsList.forEachIndexed { index, answerGroup -> + if (answerGroup.hasBaseAnswerGroup()) { + track(container, index, answerGroup.baseAnswerGroup) + } + } + } + InteractionTypeCase.END_EXPLORATION -> {} // Nothing to track. + InteractionTypeCase.INTERACTIONTYPE_NOT_SET, null -> + error("Invalid interaction: ${dto.interaction}.") + } + } + + private fun track(container: Container, dto: OutcomeDto) { + if (dto.hasFeedback()) texts += TextReference.Feedback(container, dto.feedback.contentId) + } + + private fun track(state: Container.State, index: Int, dto: BaseAnswerGroupDto) { + val container = Container.AnswerGroup(state, index) + if (dto.hasOutcome()) track(container, dto.outcome) + } + + private fun track( + state: Container.State, + answerGroupIndex: Int, + ruleSpecIndex: Int, + dto: ListOfSetsOfTranslatableHtmlContentIdsDto + ) { + val answerGroup = Container.AnswerGroup(state, answerGroupIndex) + val ruleSpec = Container.RuleSpec(answerGroup, ruleSpecIndex) + dto.contentIdSetsList.forEachIndexed { index, ids -> + track(ruleSpec, ids) { "ListOfSetsOfTranslatableHtmlContentIdsDto(index=$index, $it)" } + } + } + + private fun track( + state: Container.State, + answerGroupIndex: Int, + ruleSpecIndex: Int, + dto: SetOfTranslatableHtmlContentIdsDto + ) { + val answerGroup = Container.AnswerGroup(state, answerGroupIndex) + val ruleSpec = Container.RuleSpec(answerGroup, ruleSpecIndex) + track(ruleSpec, dto) + } + + private fun track( + container: Container, + dto: SetOfTranslatableHtmlContentIdsDto, + createExtraContext: (String) -> String = { it } + ) { + dto.contentIdsList.forEachIndexed { index, id -> + track( + container, id, context = createExtraContext("SetOfTranslatableHtmlContentIds(idx=$index)") + ) + } + } + + private fun track( + state: Container.State, + answerGroupIndex: Int, + ruleSpecIndex: Int, + dto: TranslatableSetOfNormalizedStringDto + ) { + val answerGroup = Container.AnswerGroup(state, answerGroupIndex) + val ruleSpec = Container.RuleSpec(answerGroup, ruleSpecIndex) + texts += TextReference.RuleInputTranslatableHtmlContentId( + ruleSpec, dto.contentId, context = "TranslatableSetOfNormalizedString" + ) + } + + private fun track(state: Container.State, index: Int, dto: HintDto) { + val container = Container.Hint(state, index) + if (dto.hasHintContent()) texts += TextReference.Content(container, dto.hintContent.contentId) + } + + private fun track(container: Container, dto: BaseSolutionDto) { + if (dto.hasExplanation()) { + texts += TextReference.SolutionExplanation(container, dto.explanation.contentId) + } + } + + private fun track(container: Container, dto: TranslatableHtmlContentIdDto, context: String) { + texts += TextReference.RuleInputTranslatableHtmlContentId(container, dto.contentId, context) + } + + private fun track( + state: Container.State, + answerGroupIndex: Int, + ruleSpecIndex: Int, + dto: TranslatableHtmlContentIdDto, + context: String + ) { + val answerGroup = Container.AnswerGroup(state, answerGroupIndex) + val ruleSpec = Container.RuleSpec(answerGroup, ruleSpecIndex) + texts += TextReference.RuleInputTranslatableHtmlContentId(ruleSpec, dto.contentId, context) + } + + private fun track(container: Container, dto: ContentLocalizationsDto) { + if (dto.hasDefaultMapping()) track(container, dto.defaultMapping) + dto.localizationsList.forEach { track(container, it) } + } + + private fun track(container: Container, dto: ContentLocalizationDto) { + if (dto.hasThumbnail()) { + localizations += Localizations.Thumbnail( + container, dto.language, dto.thumbnail.referencedImage.filename + ) + } + localizations += Localizations.Translations( + container, + dto.language, + dto.localizableTextContentMappingMap.map { (contentId, localizableText) -> + when (localizableText.dataFormatCase) { + LocalizableTextDto.DataFormatCase.SINGLE_LOCALIZABLE_TEXT -> { + Localizations.Translation.Single( + contentId, localizableText.singleLocalizableText.text + ) + } + LocalizableTextDto.DataFormatCase.SET_OF_LOCALIZABLE_TEXT -> { + Localizations.Translation.Multi( + contentId, localizableText.setOfLocalizableText.textList + ) + } + LocalizableTextDto.DataFormatCase.DATAFORMAT_NOT_SET, null -> + error("Invalid localizable text: $localizableText.") + } + } + ) + localizations += Localizations.Voiceovers( + container, dto.language, dto.voiceoverContentMappingMap.keys + ) + if (dto.hasLocalizedImageList()) { + localizations += Localizations.ImageReferences( + container, + dto.language, + dto.localizedImageList.referencedImagesList.map { it.filename }.toSet() + ) + } + } + + sealed class Container : Comparable { + abstract val parent: Container? + protected abstract val impliedTypeOrder: Int + protected abstract val selfReferenceString: String + + val referenceString: String + get() { + return parent?.let { "$selfReferenceString in ${it.referenceString}" } + ?: selfReferenceString + } + + override fun compareTo(other: Container): Int = COMPARATOR.compare(this, other) + + protected abstract fun compareToInternal(other: Container): Int + + data class Topic(val topicId: String) : Container() { + override val parent = null + override val selfReferenceString = "topic $topicId" + override val impliedTypeOrder = 0 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as Topic) + + private companion object { + private val COMPARATOR = compareBy(Topic::topicId) + } + } + + data class Story(override val parent: Topic, val storyId: String) : Container() { + override val selfReferenceString = "story $storyId" + override val impliedTypeOrder = 1 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as Story) + + private companion object { + private val COMPARATOR = compareBy(Story::storyId) + } + } + + data class Chapter(override val parent: Story, val explorationId: String) : Container() { + override val selfReferenceString = "chapter (exp: $explorationId)" + override val impliedTypeOrder = 2 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as Chapter) + + private companion object { + private val COMPARATOR = compareBy(Chapter::explorationId) + } + } + + data class Skill(override val parent: Topic, val skillId: String) : Container() { + override val selfReferenceString = "skill $skillId" + override val impliedTypeOrder = 3 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as Skill) + + private companion object { + private val COMPARATOR = compareBy(Skill::skillId) + } + } + + data class RevisionCard(override val parent: Topic, val index: Int) : Container() { + override val selfReferenceString = "revision card (subtopic: $index)" + override val impliedTypeOrder = 4 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as RevisionCard) + + private companion object { + private val COMPARATOR = compareBy(RevisionCard::index) + } + } + + data class ConceptCard(val skillId: String) : Container() { + override val parent = null + override val selfReferenceString = "concept card (skill: $skillId)" + override val impliedTypeOrder = 5 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as ConceptCard) + + private companion object { + private val COMPARATOR = compareBy(ConceptCard::skillId) + } + } + + data class WorkedExample(override val parent: ConceptCard, val index: Int) : Container() { + override val selfReferenceString = "worked example $index" + override val impliedTypeOrder = 6 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as WorkedExample) + + private companion object { + private val COMPARATOR = compareBy(WorkedExample::index) + } + } + + data class Exploration(val explorationId: String) : Container() { + override val parent = null + override val selfReferenceString = "exploration $explorationId" + override val impliedTypeOrder = 7 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as Exploration) + + private companion object { + private val COMPARATOR = compareBy(Exploration::explorationId) + } + } + + data class State(override val parent: Exploration, val name: String) : Container() { + override val selfReferenceString = "state '$name'" + override val impliedTypeOrder = 8 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as State) + + private companion object { + private val COMPARATOR = compareBy(State::name) + } + } + + data class AnswerGroup(override val parent: State, val index: Int) : Container() { + override val selfReferenceString = "answer group $index" + override val impliedTypeOrder = 9 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as AnswerGroup) + + private companion object { + private val COMPARATOR = compareBy(AnswerGroup::index) + } + } + + data class RuleSpec(override val parent: AnswerGroup, val index: Int) : Container() { + override val selfReferenceString = "rule spec $index" + override val impliedTypeOrder = 10 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as RuleSpec) + + private companion object { + private val COMPARATOR = compareBy(RuleSpec::index) + } + } + + data class Hint(override val parent: State, val index: Int) : Container() { + override val selfReferenceString = "hint $index" + override val impliedTypeOrder = 11 + + override fun compareToInternal(other: Container): Int = + COMPARATOR.compare(this, other as Hint) + + private companion object { + private val COMPARATOR = compareBy(Hint::index) + } + } + + companion object { + private val COMPARATOR = + compareBy(Container::impliedTypeOrder) + .thenBy(Container::parent) + .thenComparing(Container::compareToInternal) + } + } + + sealed class TextReference : Comparable { + abstract val container: Container + abstract val contentId: String + protected abstract val impliedTypeOrder: Int + + val referenceString: String get() = "$typeName in ${container.referenceString}" + + protected abstract val typeName: String + + override fun compareTo(other: TextReference): Int = COMPARATOR.compare(this, other) + + data class Name( + override val container: Container, + override val contentId: String + ) : TextReference() { + override val typeName = "name" + override val impliedTypeOrder = 0 + } + + data class Title( + override val container: Container, + override val contentId: String + ) : TextReference() { + override val typeName = "title" + override val impliedTypeOrder = 1 + } + + data class Description( + override val container: Container, + override val contentId: String + ) : TextReference() { + override val typeName = "description" + override val impliedTypeOrder = 2 + } + + data class Content( + override val container: Container, + override val contentId: String + ) : TextReference() { + override val typeName = "content" + override val impliedTypeOrder = 3 + } + + data class Explanation( + override val container: Container, + override val contentId: String + ) : TextReference() { + override val typeName = "explanation" + override val impliedTypeOrder = 4 + } + + data class Question( + override val container: Container, + override val contentId: String + ) : TextReference() { + override val typeName = "question" + override val impliedTypeOrder = 5 + } + + data class Feedback( + override val container: Container, + override val contentId: String + ) : TextReference() { + override val typeName = "feedback" + override val impliedTypeOrder = 6 + } + + data class SolutionExplanation( + override val container: Container, + override val contentId: String + ) : TextReference() { + override val typeName = "solution explanation" + override val impliedTypeOrder = 7 + } + + data class RuleInputTranslatableHtmlContentId( + override val container: Container, + override val contentId: String, + val context: String + ) : TextReference() { + override val typeName = "rule input ($context)" + override val impliedTypeOrder = 8 + } + + sealed class CustomizationArg : TextReference() { + override val typeName get() = "customization arg ($argName)" + + protected abstract val argName: String + + data class ButtonText( + override val container: Container, + override val contentId: String + ) : CustomizationArg() { + override val argName = "button text" + override val impliedTypeOrder = 9 + } + + data class Placeholder( + override val container: Container, + override val contentId: String + ) : CustomizationArg() { + override val argName = "placeholder" + override val impliedTypeOrder = 10 + } + + data class Choice( + override val container: Container, + override val contentId: String, + val index: Int + ) : CustomizationArg() { + override val argName = "choice $index" + override val impliedTypeOrder = 11 + } + } + + private companion object { + private val COMPARATOR = + compareBy(TextReference::container) + .thenBy(TextReference::contentId) + .thenBy(TextReference::impliedTypeOrder) + } + } + + sealed class Localizations { + abstract val container: Container + abstract val language: LanguageType + + data class Translations( + override val container: Container, + override val language: LanguageType, + val translations: List + ) : Localizations() + data class Voiceovers( + override val container: Container, + override val language: LanguageType, + val contentIds: Set + ) : Localizations() + data class ImageReferences( + override val container: Container, + override val language: LanguageType, + val filenames: Set + ) : Localizations() + data class Thumbnail( + override val container: Container, + override val language: LanguageType, + val thumbnailFilename: String + ) : Localizations() + + sealed class Translation { + abstract val contentId: String + abstract val htmls: List + + data class Single(override val contentId: String, val html: String) : Translation() { + override val htmls: List get() = listOf(html) + } + data class Multi( + override val contentId: String, + override val htmls: List + ) : Translation() + } + } + + sealed class Issue : Comparable { + protected abstract val referenceContainer: Container + protected abstract val impliedTypeOrder: Int + + override fun compareTo(other: Issue): Int = COMPARATOR.compare(this, other) + + protected abstract fun compareToInternal(other: Issue): Int + + data class ImageHasInvalidExtension( + val container: Container, + val language: LanguageType, + val filename: String, + val invalidExtension: String + ) : Issue() { + override val referenceContainer = container + override val impliedTypeOrder: Int = 0 + + override fun compareToInternal(other: Issue): Int = + COMPARATOR.compare(this, other as ImageHasInvalidExtension) + + private companion object { + private val COMPARATOR = + compareBy(ImageHasInvalidExtension::language) + .thenBy(ImageHasInvalidExtension::filename) + .thenBy(ImageHasInvalidExtension::invalidExtension) + } + } + + data class ImageInconsistencies( + val container: Container, + val filename: String, + val presentLanguages: Set, + val missingLanguages: Set + ) : Issue() { + override val referenceContainer = container + override val impliedTypeOrder: Int = 1 + + override fun compareToInternal(other: Issue): Int = + COMPARATOR.compare(this, other as ImageInconsistencies) + + private companion object { + private val COMPARATOR = compareBy(ImageInconsistencies::filename) + } + } + + data class HtmlHasInvalidTag( + val language: LanguageType, + val text: TextReference, + val html: String, + val invalidTag: String + ) : Issue() { + override val referenceContainer = text.container + override val impliedTypeOrder: Int = 2 + + override fun compareToInternal(other: Issue): Int = + COMPARATOR.compare(this, other as HtmlHasInvalidTag) + + private companion object { + private val COMPARATOR = + compareBy(HtmlHasInvalidTag::language) + .thenBy(HtmlHasInvalidTag::invalidTag) + .thenBy(HtmlHasInvalidTag::text) + } + } + + data class TextMissingTranslations( + val text: TextReference, + val presentLanguages: Set, + val missingLanguages: Set + ) : Issue() { + override val referenceContainer = text.container + override val impliedTypeOrder: Int = 3 + + override fun compareToInternal(other: Issue): Int = + COMPARATOR.compare(this, other as TextMissingTranslations) + + private companion object { + private val COMPARATOR = compareBy(TextMissingTranslations::text) + } + } + + private companion object { + private val COMPARATOR = + compareBy(Issue::referenceContainer) + .thenBy(Issue::impliedTypeOrder) + .thenComparing(Issue::compareToInternal) + } + } + + sealed class MetricsReport { + data class TranslationUsage( + val container: Container, + val languageUsage: Map + ) : MetricsReport() + data class VoiceoverUsage( + val container: Container, + val languageUsage: Map + ) : MetricsReport() + + data class Usage(val usedCount: Int, val totalCount: Int) { + val ratio: Float get() = usedCount.toFloat() / totalCount.toFloat() + val roundedPercentage: Float get() = (ratio * 1000).toInt() / 10.0f + val percentageString: String get() = if (totalCount != 0) "$roundedPercentage%" else "N/A" + val usageString: String get() = "$usedCount/$totalCount" + + fun combineWith(other: Usage): Usage = + Usage(usedCount = usedCount + other.usedCount, totalCount = totalCount + other.totalCount) + } + } + } + + private companion object { + private val CLIENT_CONTEXT = AndroidClientContextDto.newBuilder().apply { + appVersionName = checkNotNull(LessonDownloader::class.qualifiedName) + appVersionCode = 0 + }.build() + private const val CONSOLE_COLUMN_COUNT = 80 + + private fun ExecutorService.tryShutdownFully(timeout: Long, unit: TimeUnit) { + // Try to fully shutdown the executor service per https://stackoverflow.com/a/33690603 and + // https://docs.oracle.com/javase/7/docs/api/java/util/concurrent/ExecutorService.html. + shutdown() + try { + if (!awaitTermination(timeout, unit)) { + shutdownNow() + check(awaitTermination(timeout, unit)) { + "Executor service didn't fully shutdown: $this." + } + } + } catch (e: InterruptedException) { + shutdownNow() + Thread.currentThread().interrupt() + } + } + + private fun createSubtopicId(topicId: String, subtopicIndex: Int): SubtopicPageIdDto { + return SubtopicPageIdDto.newBuilder().apply { + this.topicId = topicId + this.subtopicIndex = subtopicIndex + }.build() + } + + private fun SubtopicPageIdDto.collapse(): String = "${topicId}_$subtopicIndex" + + private fun createLocalizedRevisionCardId( + id: SubtopicPageIdDto, + language: LanguageType + ): LocalizedRevisionCardIdDto { + return LocalizedRevisionCardIdDto.newBuilder().apply { + this.id = id + this.language = language + }.build() + } + + private fun LocalizedRevisionCardIdDto.collapse(): String = + "${id.collapse()}_${language.collapse()}" + + private fun createLocalizedExplorationId( + explorationId: String, + language: LanguageType + ): LocalizedExplorationIdDto { + return LocalizedExplorationIdDto.newBuilder().apply { + this.explorationId = explorationId + this.language = language + }.build() + } + + private fun LocalizedExplorationIdDto.collapse(): String = + "${explorationId}_${language.collapse()}" + + private fun createLocalizedConceptCardId( + skillId: String, + language: LanguageType + ): LocalizedConceptCardIdDto { + return LocalizedConceptCardIdDto.newBuilder().apply { + this.skillId = skillId + this.language = language + }.build() + } + + private fun LocalizedConceptCardIdDto.collapse(): String = "${skillId}_${language.collapse()}" + + private fun LanguageType.collapse(): String { + return when (this) { + LanguageType.ENGLISH -> "en" + LanguageType.ARABIC -> "ar" + LanguageType.HINDI -> "hi" + LanguageType.HINGLISH -> "hi-en" + LanguageType.BRAZILIAN_PORTUGUESE -> "pt-br" + LanguageType.SWAHILI -> "sw" + LanguageType.NIGERIAN_PIDGIN -> "pcm" + LanguageType.LANGUAGE_CODE_UNSPECIFIED, LanguageType.UNRECOGNIZED -> + error("Invalid language type: $this.") + } + } + + private fun T.toStructureIdentifier( + contentVersion: Int, + setValue: DownloadReqStructIdDtoBuilder.(T) -> DownloadReqStructIdDtoBuilder + ): DownloadRequestStructureIdentifierDto { + return DownloadRequestStructureIdentifierDto.newBuilder().apply { + this.contentVersion = contentVersion + this.setValue(this@toStructureIdentifier) + }.build() + } + + private fun TopicContentResponseDto.collectImageReferences(): List = + downloadResultsList.flatMap { it.collectImageReferences() } + + private fun DownloadResultDto.collectImageReferences(): List { + return when (resultTypeCase) { + SKIPPED_SHOULD_RETRY, SKIPPED_FROM_FAILURE, SKIPPED_DOES_NOT_EXIST -> emptyList() + TOPIC_SUMMARY -> topicSummary.collectImageReferences() + REVISION_CARD -> revisionCard.collectImageReferences() + CONCEPT_CARD -> conceptCard.collectImageReferences() + EXPLORATION -> exploration.collectImageReferences() + QUESTION_ID_LIST -> emptyList() // No translations for a question ID list. + QUESTION -> question.collectImageReferences() + REVISION_CARD_LANGUAGE_PACK -> revisionCardLanguagePack.collectImageReferences() + CONCEPT_CARD_LANGUAGE_PACK -> conceptCardLanguagePack.collectImageReferences() + EXPLORATION_LANGUAGE_PACK -> explorationLanguagePack.collectImageReferences() + QUESTION_LANGUAGE_PACK -> questionLanguagePack.collectImageReferences() + RESULTTYPE_NOT_SET, null -> error("Encountered invalid result: $this.") + } + } + + private fun DownloadableTopicSummaryDto.collectImageReferences(): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.TOPIC, + entityId = id, + language = localizations.defaultMapping.language + ) + return localizations.collectImageReferences(container) + + storySummariesList.flatMap { + it.collectImageReferences(localizations.defaultMapping.language) + } + referencedSkillsList.flatMap { it.collectImageReferences() } + } + + private fun RevisionCardDto.collectImageReferences(): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.TOPIC, + entityId = id.topicId, + language = defaultLocalization.language + ) + return defaultLocalization.collectImageReferences(container) + } + + private fun ConceptCardDto.collectImageReferences(): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.SKILL, + entityId = skillId, + language = defaultLocalization.language + ) + return defaultLocalization.collectImageReferences(container) + } + + private fun ExplorationDto.collectImageReferences(): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.EXPLORATION, + entityId = id, + language = defaultLocalization.language + ) + return defaultLocalization.collectImageReferences(container) + } + + private fun QuestionDto.collectImageReferences(): List { + // TODO: Should be using skill ID here? + val container = + ImageContainer( + imageContainerType = ImageContainerType.SKILL, + entityId = id, + language = defaultLocalization.language + ) + return defaultLocalization.collectImageReferences(container) + } + + private fun RevisionCardLanguagePackDto.collectImageReferences(): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.TOPIC, + entityId = id.id.topicId, + language = id.language + ) + return localization.collectImageReferences(container) + } + + private fun ConceptCardLanguagePackDto.collectImageReferences(): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.SKILL, + entityId = id.skillId, + language = id.language + ) + return localization.collectImageReferences(container) + } + + private fun ExplorationLanguagePackDto.collectImageReferences(): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.EXPLORATION, + entityId = id.explorationId, + language = id.language + ) + return localization.collectImageReferences(container) + } + + private fun QuestionLanguagePackDto.collectImageReferences(): List { + // TODO: Should be using skill ID here? + val container = + ImageContainer( + imageContainerType = ImageContainerType.SKILL, + entityId = id.questionId, + language = id.language + ) + return localization.collectImageReferences(container) + } + + private fun StorySummaryDto.collectImageReferences( + defaultLanguage: LanguageType + ): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.STORY, entityId = id, language = defaultLanguage + ) + return localizations.collectImageReferences(container) + + chaptersList.flatMap { + it.collectImageReferences(this@collectImageReferences.id, defaultLanguage) + } + } + + private fun ChapterSummaryDto.collectImageReferences( + storyId: String, + defaultLanguage: LanguageType + ): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.STORY, + entityId = storyId, + language = defaultLanguage + ) + return localizations.collectImageReferences(container) + } + + private fun SkillSummaryDto.collectImageReferences(): List { + val container = + ImageContainer( + imageContainerType = ImageContainerType.SKILL, + entityId = id, + language = localizations.defaultMapping.language + ) + return localizations.collectImageReferences(container) + } + + private fun ContentLocalizationsDto.collectImageReferences( + container: ImageContainer + ): List { + return defaultMapping.collectImageReferences(container) + + localizationsList.flatMap { it.collectImageReferences(container) } + } + + private fun ContentLocalizationDto.collectImageReferences( + container: ImageContainer + ): List { + return localizedImageList.collectImageReferences(container) + + (thumbnail.takeIf { hasThumbnail() }?.collectImageReferences(container) ?: emptyList()) + } + + private fun ReferencedImageListDto.collectImageReferences(container: ImageContainer) = + referencedImagesList.map { it.convertToImageReference(container) } + + private fun ThumbnailDto.collectImageReferences(container: ImageContainer) = + listOf(referencedImage.convertToImageReference(container, imageType = ImageType.THUMBNAIL)) + + private fun ReferencedImageDto.convertToImageReference( + container: ImageContainer, + imageType: ImageType = ImageType.HTML_IMAGE + ): ImageReference = ImageReference(container, imageType, filename) + + private fun CompatibilityAnalyzer.Container.findRoot(): CompatibilityAnalyzer.Container = + generateSequence(this) { it.parent }.last() + } +} diff --git a/scripts/src/java/org/oppia/android/scripts/assets/DtoProtoToLegacyProtoConverter.kt b/scripts/src/java/org/oppia/android/scripts/assets/DtoProtoToLegacyProtoConverter.kt new file mode 100644 index 00000000000..2ef581b5b29 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/assets/DtoProtoToLegacyProtoConverter.kt @@ -0,0 +1,1782 @@ +package org.oppia.android.scripts.assets + +import org.oppia.android.app.model.AnswerGroup +import org.oppia.android.app.model.ChapterRecord +import org.oppia.android.app.model.ClassroomList +import org.oppia.android.app.model.ClassroomRecord +import org.oppia.android.app.model.ConceptCard +import org.oppia.android.app.model.ConceptCardList +import org.oppia.android.app.model.CustomSchemaValue +import org.oppia.android.app.model.Exploration +import org.oppia.android.app.model.Fraction +import org.oppia.android.app.model.Hint +import org.oppia.android.app.model.HtmlTranslationList +import org.oppia.android.app.model.ImageWithRegions +import org.oppia.android.app.model.ImageWithRegions.LabeledRegion +import org.oppia.android.app.model.ImageWithRegions.LabeledRegion.Region.NormalizedRectangle2d +import org.oppia.android.app.model.Interaction +import org.oppia.android.app.model.InteractionObject +import org.oppia.android.app.model.LessonThumbnail +import org.oppia.android.app.model.ListOfSetsOfTranslatableHtmlContentIds +import org.oppia.android.app.model.Misconception +import org.oppia.android.app.model.Outcome +import org.oppia.android.app.model.Point2d +import org.oppia.android.app.model.RatioExpression +import org.oppia.android.app.model.RuleSpec +import org.oppia.android.app.model.SchemaObject +import org.oppia.android.app.model.SchemaObjectList +import org.oppia.android.app.model.SetOfTranslatableHtmlContentIds +import org.oppia.android.app.model.Solution +import org.oppia.android.app.model.State +import org.oppia.android.app.model.StoryRecord +import org.oppia.android.app.model.SubtitledHtml +import org.oppia.android.app.model.SubtitledUnicode +import org.oppia.android.app.model.SubtopicRecord +import org.oppia.android.app.model.TopicRecord +import org.oppia.android.app.model.TranslatableHtmlContentId +import org.oppia.android.app.model.TranslatableSetOfNormalizedString +import org.oppia.android.app.model.Translation +import org.oppia.android.app.model.TranslationMapping +import org.oppia.android.app.model.Voiceover +import org.oppia.android.app.model.VoiceoverMapping +import org.oppia.proto.v1.structure.AlgebraicExpressionInputInstanceDto +import org.oppia.proto.v1.structure.ChapterSummaryDto +import org.oppia.proto.v1.structure.ClassroomDto +import org.oppia.proto.v1.structure.ConceptCardDto +import org.oppia.proto.v1.structure.ConceptCardLanguagePackDto +import org.oppia.proto.v1.structure.ContentLocalizationDto +import org.oppia.proto.v1.structure.ContentLocalizationsDto +import org.oppia.proto.v1.structure.ContinueInstanceDto +import org.oppia.proto.v1.structure.DownloadableTopicSummaryDto +import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto +import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto.HasElementXBeforeElementYSpecDto +import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto.IsEqualToOrderingWithOneItemAtIncorrectPositionSpecDto +import org.oppia.proto.v1.structure.ExplorationDto +import org.oppia.proto.v1.structure.ExplorationLanguagePackDto +import org.oppia.proto.v1.structure.FractionDto +import org.oppia.proto.v1.structure.FractionInputInstanceDto +import org.oppia.proto.v1.structure.FractionInputInstanceDto.RuleSpecDto.HasFractionalPartExactlyEqualToSpecDto +import org.oppia.proto.v1.structure.FractionInputInstanceDto.RuleSpecDto.IsEquivalentToAndInSimplestFormSpecDto +import org.oppia.proto.v1.structure.HintDto +import org.oppia.proto.v1.structure.ImageClickInputInstanceDto +import org.oppia.proto.v1.structure.ImageClickInputInstanceDto.RuleSpecDto.IsInRegionSpecDto +import org.oppia.proto.v1.structure.ImageWithRegionsDto +import org.oppia.proto.v1.structure.ImageWithRegionsDto.LabeledRegionDto +import org.oppia.proto.v1.structure.ImageWithRegionsDto.LabeledRegionDto.NormalizedRectangle2dDto +import org.oppia.proto.v1.structure.ImageWithRegionsDto.LabeledRegionDto.RegionTypeCase.NORMALIZED_RECTANGLE_2D +import org.oppia.proto.v1.structure.InteractionInstanceDto +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.ALGEBRAIC_EXPRESSION_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.CONTINUE_INSTANCE +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.DRAG_AND_DROP_SORT_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.END_EXPLORATION +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.FRACTION_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.IMAGE_CLICK_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.INTERACTIONTYPE_NOT_SET +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.ITEM_SELECTION_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.MATH_EQUATION_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.MULTIPLE_CHOICE_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMERIC_EXPRESSION_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.NUMERIC_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.RATIO_EXPRESSION_INPUT +import org.oppia.proto.v1.structure.InteractionInstanceDto.InteractionTypeCase.TEXT_INPUT +import org.oppia.proto.v1.structure.ItemSelectionInputInstanceDto +import org.oppia.proto.v1.structure.LanguageType +import org.oppia.proto.v1.structure.ListOfSetsOfTranslatableHtmlContentIdsDto +import org.oppia.proto.v1.structure.LocalizableTextDto +import org.oppia.proto.v1.structure.MathEquationInputInstanceDto +import org.oppia.proto.v1.structure.MisconceptionDto +import org.oppia.proto.v1.structure.MultipleChoiceInputInstanceDto +import org.oppia.proto.v1.structure.NormalizedPoint2dDto +import org.oppia.proto.v1.structure.NumericExpressionInputInstanceDto +import org.oppia.proto.v1.structure.NumericInputInstanceDto +import org.oppia.proto.v1.structure.OutcomeDto +import org.oppia.proto.v1.structure.RatioExpressionDto +import org.oppia.proto.v1.structure.RatioExpressionInputInstanceDto +import org.oppia.proto.v1.structure.RevisionCardDto +import org.oppia.proto.v1.structure.RevisionCardLanguagePackDto +import org.oppia.proto.v1.structure.SetOfTranslatableHtmlContentIdsDto +import org.oppia.proto.v1.structure.StateDto +import org.oppia.proto.v1.structure.StorySummaryDto +import org.oppia.proto.v1.structure.SubtitledTextDto +import org.oppia.proto.v1.structure.SubtopicSummaryDto +import org.oppia.proto.v1.structure.TextInputInstanceDto +import org.oppia.proto.v1.structure.ThumbnailDto +import org.oppia.proto.v1.structure.TranslatableHtmlContentIdDto +import org.oppia.proto.v1.structure.TranslatableSetOfNormalizedStringDto +import org.oppia.proto.v1.structure.UpcomingTopicSummaryDto +import org.oppia.proto.v1.structure.VoiceoverFileDto +import org.oppia.proto.v1.structure.AlgebraicExpressionInputInstanceDto.RuleSpecDto as AlgebraRuleSpecDto +import org.oppia.proto.v1.structure.AlgebraicExpressionInputInstanceDto.RuleSpecDto.MatchesUpToTrivialManipulationsSpecDto as AlgebraMatchesUpToTrivialManipulationsSpecDto +import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto as DragDropSortRuleSpecDto +import org.oppia.proto.v1.structure.DragAndDropSortInputInstanceDto.RuleSpecDto.RuleTypeCase as DragDropSortRuleSpecTypeCase +import org.oppia.proto.v1.structure.FractionInputInstanceDto.RuleSpecDto as FractionRuleSpecDto +import org.oppia.proto.v1.structure.ItemSelectionInputInstanceDto.RuleSpecDto as ItemSelRuleSpecDto +import org.oppia.proto.v1.structure.MathEquationInputInstanceDto.RuleSpecDto as MathEqRuleSpecDto +import org.oppia.proto.v1.structure.MathEquationInputInstanceDto.RuleSpecDto.MatchesUpToTrivialManipulationsSpecDto as MathEqMatchesUpToTrivialManipulationsSpecDto +import org.oppia.proto.v1.structure.MultipleChoiceInputInstanceDto.RuleSpecDto as MultChoiceRuleSpecDto +import org.oppia.proto.v1.structure.NumericExpressionInputInstanceDto.RuleSpecDto as NumExpRuleSpecDto +import org.oppia.proto.v1.structure.NumericExpressionInputInstanceDto.RuleSpecDto.MatchesUpToTrivialManipulationsSpecDto as NumExpMatchesUpToTrivialManipulationsSpecDto +import org.oppia.proto.v1.structure.NumericInputInstanceDto.RuleSpecDto as NumInputRuleSpecDto +import org.oppia.proto.v1.structure.RatioExpressionInputInstanceDto.RuleSpecDto as RatioRuleSpecDto + +// TODO: For all "not used/unused" properties, remove them from the app's protos. + +private typealias XlatableContentIdsSetDto = SetOfTranslatableHtmlContentIdsDto +private typealias XlatableContentIdsSetListDto = ListOfSetsOfTranslatableHtmlContentIdsDto +private typealias XlatableContentIdsSet = SetOfTranslatableHtmlContentIds +private typealias XlatableContentIdsSetList = ListOfSetsOfTranslatableHtmlContentIds + +object DtoProtoToLegacyProtoConverter { + fun Iterable.convertToClassroomList( + topicSummaries: Iterable, + allImageReferenceReplacements: Map> + ): ClassroomList { + val dtos = this + val topicSummaryMap = topicSummaries.associateBy { it.id } + return ClassroomList.newBuilder().apply { + addAllClassrooms( + dtos.map { + it.convertToClassroomRecord( + topicSummaryMap, allImageReferenceReplacements.getValue(it.id) + ) + } + ) + }.build() + } + + fun DownloadableTopicSummaryDto.convertToTopicRecord( + imageReferenceReplacements: Map + ): TopicRecord { + val dto = this + return TopicRecord.newBuilder().apply { + this.id = dto.id + putAllWrittenTranslations(dto.localizations.toTranslationMappings(imageReferenceReplacements)) + this.translatableTitle = dto.localizations.extractDefaultSubtitledHtml(dto.name) + this.translatableDescription = dto.localizations.extractDefaultSubtitledHtml(dto.description) + addAllCanonicalStoryIds(dto.storySummariesList.map { it.id }) + addAllSubtopicIds(dto.subtopicSummariesList.map { it.index }) + this.isPublished = true + this.topicThumbnail = dto.localizations.extractDefaultThumbnail(imageReferenceReplacements) + }.build() + } + + fun UpcomingTopicSummaryDto.convertToTopicRecord( + imageReferenceReplacements: Map + ): TopicRecord { + val dto = this + return TopicRecord.newBuilder().apply { + this.id = dto.id + putAllWrittenTranslations(dto.localizations.toTranslationMappings(imageReferenceReplacements)) + this.translatableTitle = dto.localizations.extractDefaultSubtitledHtml(dto.name) + this.translatableDescription = dto.localizations.extractDefaultSubtitledHtml(dto.description) + this.isPublished = false + this.topicThumbnail = dto.localizations.extractDefaultThumbnail(imageReferenceReplacements) + }.build() + } + + fun StorySummaryDto.convertToStoryRecord( + imageReferenceReplacements: Map + ): StoryRecord { + val dto = this + return StoryRecord.newBuilder().apply { + this.storyId = dto.id + putAllWrittenTranslations(dto.localizations.toTranslationMappings(imageReferenceReplacements)) + this.translatableStoryName = dto.localizations.extractDefaultSubtitledHtml(dto.title) + this.storyThumbnail = dto.localizations.extractDefaultThumbnail(imageReferenceReplacements) + addAllChapters(dto.chaptersList.map { it.convertToChapterRecord(imageReferenceReplacements) }) + }.build() + } + + fun RevisionCardDto.convertToSubtopicRecord( + imageReferenceReplacements: Map, + subtopicSummaryDto: SubtopicSummaryDto, + languagePackDtos: List + ): SubtopicRecord { + val dto = this + val localizations = languagePackDtos.map { it.localization } + return SubtopicRecord.newBuilder().apply { + this.title = defaultLocalization.extractSubtitledHtml(dto.title) + this.pageContents = defaultLocalization.extractSubtitledHtml(dto.content) + putAllRecordedVoiceover(localizations.toVoiceoverMappings()) + putAllWrittenTranslation(localizations.toTranslationMappings(imageReferenceReplacements)) + addAllSkillIds(subtopicSummaryDto.referencedSkillIdsList) + this.subtopicThumbnail = dto.defaultLocalization.extractThumbnail(imageReferenceReplacements) + }.build() + } + + fun convertToConceptCardList( + allImageReferenceReplacements: Map>, + conceptCardDtos: List>> + ): ConceptCardList { + return ConceptCardList.newBuilder().apply { + addAllConceptCards( + conceptCardDtos.map { (conceptCard, packs) -> + val imageReferenceReplacements = + allImageReferenceReplacements.getValue(conceptCard.skillId) + conceptCard.convertToConceptCard(imageReferenceReplacements, packs) + } + ) + }.build() + } + + fun ExplorationDto.convertToExploration( + imageReferenceReplacements: Map, + languagePackDtos: List + ): Exploration { + val dto = this + val localizations = languagePackDtos.map { it.localization } + // Only top-level content IDs should be present at the exploration level. + val contentIdTracker = ContentIdTracker(dto.defaultLocalization) + return Exploration.newBuilder().apply { + this.id = dto.id + putAllStates( + dto.statesMap.mapValues { (name, stateDto) -> + stateDto.convertToState( + name, dto.defaultLocalization, localizations, imageReferenceReplacements + ) + } + ) + this.initStateName = dto.initStateName + this.languageCode = dto.defaultLocalization.language.toLegacyLanguageCode() + this.version = dto.contentVersion + this.translatableTitle = contentIdTracker.extractSubtitledHtml(dto.title) + putAllWrittenTranslations( + localizations.toTranslationMappings(imageReferenceReplacements, contentIdTracker.contentIds) + ) + // Correctness feedback, description, param changes, and param specs aren't used. + }.build() + } + + private fun ClassroomDto.convertToClassroomRecord( + topicSummaryMap: Map, + imageReferenceReplacements: Map + ): ClassroomRecord { + val dto = this + return ClassroomRecord.newBuilder().apply { + this.id = dto.id + this.classroomThumbnail = + dto.localizations.extractDefaultThumbnail(imageReferenceReplacements) + putAllWrittenTranslations(dto.localizations.toTranslationMappings(imageReferenceReplacements)) + this.translatableTitle = dto.localizations.extractDefaultSubtitledHtml(dto.name) + putAllTopicPrerequisites( + dto.topicIdsList.associateWith { topicId -> + ClassroomRecord.TopicIdList.newBuilder().apply { + addAllTopicIds(topicSummaryMap.getValue(topicId).prerequisiteTopicIdsList) + }.build() + } + ) + }.build() + } + + private fun ChapterSummaryDto.convertToChapterRecord( + imageReferenceReplacements: Map + ): ChapterRecord { + val dto = this + return ChapterRecord.newBuilder().apply { + this.explorationId = dto.explorationId + this.chapterThumbnail = dto.localizations.extractDefaultThumbnail(imageReferenceReplacements) + putAllWrittenTranslations(dto.localizations.toTranslationMappings(imageReferenceReplacements)) + this.translatableTitle = dto.localizations.extractDefaultSubtitledHtml(dto.title) + this.translatableDescription = dto.localizations.extractDefaultSubtitledHtml(dto.description) + }.build() + } + + private fun ConceptCardDto.convertToConceptCard( + imageReferenceReplacements: Map, + languagePackDtos: List + ): ConceptCard { + val dto = this + val localizations = languagePackDtos.map { it.localization } + return ConceptCard.newBuilder().apply { + this.skillId = dto.skillId + if (dto.hasDescription()) { + this.skillDescription = dto.defaultLocalization.extractSubtitledHtml(dto.description).html + } + if (dto.hasExplanation()) { + this.explanation = dto.defaultLocalization.extractSubtitledHtml(dto.explanation) + } + addAllWorkedExample( + dto.workedExamplesList.map { dto.defaultLocalization.extractSubtitledHtml(it.explanation) } + ) + putAllRecordedVoiceover(localizations.toVoiceoverMappings()) + putAllWrittenTranslation(localizations.toTranslationMappings(imageReferenceReplacements)) + }.build() + } + + private fun StateDto.convertToState( + name: String, + defaultLocalizationDto: ContentLocalizationDto, + localizations: List, + imageReferenceReplacements: Map + ): State { + val dto = this + // The content IDs associated with translations and voiceovers for a state should only + // correspond to those actually used within that state (since the new structure stores IDs at + // the structure level). + val contentIdTracker = ContentIdTracker(defaultLocalizationDto) + val allLocalizations = localizations + defaultLocalizationDto + return State.newBuilder().apply { + this.name = name + this.content = contentIdTracker.extractSubtitledHtml(dto.content) + this.interaction = dto.interaction.convertToInteraction(contentIdTracker) + putAllRecordedVoiceovers(allLocalizations.toVoiceoverMappings(contentIdTracker.contentIds)) + putAllWrittenTranslations( + localizations.toTranslationMappings(imageReferenceReplacements, contentIdTracker.contentIds) + ) + // Param changes, linked skill ID, classifier model ID, and answer soliciting aren't used. + }.build() + } + + private fun InteractionInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + return when (interactionTypeCase) { + CONTINUE_INSTANCE -> continueInstance.convertToInteraction(contentIdTracker) + FRACTION_INPUT -> fractionInput.convertToInteraction(contentIdTracker) + ITEM_SELECTION_INPUT -> itemSelectionInput.convertToInteraction(contentIdTracker) + MULTIPLE_CHOICE_INPUT -> multipleChoiceInput.convertToInteraction(contentIdTracker) + NUMERIC_INPUT -> numericInput.convertToInteraction(contentIdTracker) + TEXT_INPUT -> textInput.convertToInteraction(contentIdTracker) + DRAG_AND_DROP_SORT_INPUT -> dragAndDropSortInput.convertToInteraction(contentIdTracker) + IMAGE_CLICK_INPUT -> imageClickInput.convertToInteraction(contentIdTracker) + RATIO_EXPRESSION_INPUT -> ratioExpressionInput.convertToInteraction(contentIdTracker) + ALGEBRAIC_EXPRESSION_INPUT -> algebraicExpressionInput.convertToInteraction(contentIdTracker) + MATH_EQUATION_INPUT -> mathEquationInput.convertToInteraction(contentIdTracker) + NUMERIC_EXPRESSION_INPUT -> numericExpressionInput.convertToInteraction(contentIdTracker) + END_EXPLORATION -> Interaction.newBuilder().setId("EndExploration").build() + INTERACTIONTYPE_NOT_SET, null -> error("Invalid interaction instance: $this.") + } + } + + private fun ContinueInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "Continue" + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap(contentIdTracker)) + }.build() + } + + private fun ContinueInstanceDto.CustomizationArgsDto.convertToArgsMap( + contentIdTracker: ContentIdTracker + ) = mapOf("buttonText" to contentIdTracker.extractSubtitledUnicode(buttonText).wrap()) + + private fun FractionInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "FractionInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + dto.solution.takeIf { + dto.hasSolution() + }?.convertToSolution(contentIdTracker)?.let { this.solution = it } + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap(contentIdTracker)) + }.build() + } + + private fun FractionInputInstanceDto.SolutionDto.convertToSolution( + contentIdTracker: ContentIdTracker + ): Solution? { + val dto = this + return Solution.newBuilder().apply { + if (dto.baseSolution.hasExplanation()) { + this.explanation = contentIdTracker.extractSubtitledHtml(dto.baseSolution.explanation) + } + this.correctAnswer = dto.correctAnswer.convertToInteractionObject() + // Whether the answer is exclusive isn't used. + }.build().takeIf { it != Solution.getDefaultInstance() } + } + + private fun FractionInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun FractionRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + FractionRuleSpecDto.RuleTypeCase.IS_EXACTLY_EQUAL_TO -> isExactlyEqualTo.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.IS_EQUIVALENT_TO -> isEquivalentTo.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.IS_EQUIVALENT_TO_AND_IN_SIMPLEST_FORM -> + isEquivalentToAndInSimplestForm.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.IS_LESS_THAN -> isLessThan.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.IS_GREATER_THAN -> isGreaterThan.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.HAS_NUMERATOR_EQUAL_TO -> + hasNumeratorEqualTo.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.HAS_DENOMINATOR_EQUAL_TO -> + hasDenominatorEqualTo.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.HAS_INTEGER_PART_EQUAL_TO -> + hasIntegerPartEqualTo.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.HAS_NO_FRACTIONAL_PART -> + RuleSpec.newBuilder().setRuleType("HasNoFractionalPart").build() + FractionRuleSpecDto.RuleTypeCase.HAS_FRACTIONAL_PART_EXACTLY_EQUAL_TO -> + hasFractionalPartExactlyEqualTo.convertToRuleSpec() + FractionRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun FractionRuleSpecDto.IsExactlyEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsExactlyEqualTo" + putInput("f", dto.input.convertToInteractionObject()) + }.build() + } + + private fun FractionRuleSpecDto.IsEquivalentToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsEquivalentTo" + putInput("f", dto.input.convertToInteractionObject()) + }.build() + } + + private fun IsEquivalentToAndInSimplestFormSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsEquivalentToAndInSimplestForm" + putInput("f", dto.input.convertToInteractionObject()) + }.build() + } + + private fun FractionRuleSpecDto.IsLessThanSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsLessThan" + putInput("f", dto.input.convertToInteractionObject()) + }.build() + } + + private fun FractionRuleSpecDto.IsGreaterThanSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsGreaterThan" + putInput("f", dto.input.convertToInteractionObject()) + }.build() + } + + private fun FractionRuleSpecDto.HasNumeratorEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "HasNumeratorEqualTo" + putInput("x", dto.input.convertToSignedInteractionObject()) + }.build() + } + + private fun FractionRuleSpecDto.HasDenominatorEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "HasDenominatorEqualTo" + putInput("x", dto.input.convertToNonNegativeInteractionObject()) + }.build() + } + + private fun FractionRuleSpecDto.HasIntegerPartEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "HasIntegerPartEqualTo" + putInput("x", dto.input.convertToSignedInteractionObject()) + }.build() + } + + private fun HasFractionalPartExactlyEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "HasFractionalPartExactlyEqualTo" + putInput("f", dto.input.convertToInteractionObject()) + }.build() + } + + private fun FractionInputInstanceDto.CustomizationArgsDto.convertToArgsMap( + contentIdTracker: ContentIdTracker + ) = mapOf( + "requireSimplestForm" to requiresSimplestForm.wrap(), + "allowImproperFraction" to allowImproperFractions.wrap(), + "allowNonzeroIntegerPart" to allowNonzeroIntegerPart.wrap(), + "customPlaceholder" to contentIdTracker.extractSubtitledUnicode(placeholder).wrap() + ) + + private fun ItemSelectionInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "ItemSelectionInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap(contentIdTracker)) + }.build() + } + + private fun ItemSelectionInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun ItemSelRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + ItemSelRuleSpecDto.RuleTypeCase.EQUALS -> equals.convertToRuleSpec() + ItemSelRuleSpecDto.RuleTypeCase.CONTAINS_AT_LEAST_ONE_OF -> + containsAtLeastOneOf.convertToRuleSpec() + ItemSelRuleSpecDto.RuleTypeCase.DOES_NOT_CONTAIN_AT_LEAST_ONE_OF -> + doesNotContainAtLeastOneOf.convertToRuleSpec() + ItemSelRuleSpecDto.RuleTypeCase.IS_PROPER_SUBSET_OF -> isProperSubsetOf.convertToRuleSpec() + ItemSelRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun ItemSelRuleSpecDto.EqualsSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "Equals" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun ItemSelRuleSpecDto.ContainsAtLeastOneOfSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "ContainsAtLeastOneOf" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun ItemSelRuleSpecDto.DoesNotContainAtLeastOneOfSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "DoesNotContainAtLeastOneOf" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun ItemSelRuleSpecDto.IsProperSubsetOfSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsProperSubsetOf" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun ItemSelectionInputInstanceDto.CustomizationArgsDto.convertToArgsMap( + contentIdTracker: ContentIdTracker + ) = mapOf( + "minAllowableSelectionCount" to minAllowableSelectionCount.wrap(), + "maxAllowableSelectionCount" to maxAllowableSelectionCount.wrap(), + "choices" to choicesList.map { contentIdTracker.extractSubtitledHtml(it).wrap() }.wrap() + ) + + private fun MultipleChoiceInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "MultipleChoiceInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap(contentIdTracker)) + }.build() + } + + private fun MultipleChoiceInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun MultChoiceRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + MultChoiceRuleSpecDto.RuleTypeCase.EQUALS -> equals.convertToRuleSpec() + MultChoiceRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun MultChoiceRuleSpecDto.EqualsSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "Equals" + putInput("x", dto.input.convertToNonNegativeInteractionObject()) + }.build() + } + + private fun MultipleChoiceInputInstanceDto.CustomizationArgsDto.convertToArgsMap( + contentIdTracker: ContentIdTracker + ): Map { + return mapOf( + "choices" to choicesList.map { contentIdTracker.extractSubtitledHtml(it).wrap() }.wrap() + ) + } + + private fun NumericInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "NumericInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + dto.solution.takeIf { + dto.hasSolution() + }?.convertToSolution(contentIdTracker)?.let { this.solution = it } + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + }.build() + } + + private fun NumericInputInstanceDto.SolutionDto.convertToSolution( + contentIdTracker: ContentIdTracker + ): Solution? { + val dto = this + return Solution.newBuilder().apply { + if (dto.baseSolution.hasExplanation()) { + this.explanation = contentIdTracker.extractSubtitledHtml(dto.baseSolution.explanation) + } + this.correctAnswer = dto.correctAnswer.convertToInteractionObject() + // Whether the answer is exclusive isn't used. + }.build().takeIf { it != Solution.getDefaultInstance() } + } + + private fun NumericInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun NumInputRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + NumInputRuleSpecDto.RuleTypeCase.EQUALS -> equals.convertToRuleSpec() + NumInputRuleSpecDto.RuleTypeCase.IS_LESS_THAN -> isLessThan.convertToRuleSpec() + NumInputRuleSpecDto.RuleTypeCase.IS_GREATER_THAN -> isGreaterThan.convertToRuleSpec() + NumInputRuleSpecDto.RuleTypeCase.IS_LESS_THAN_OR_EQUAL_TO -> + isLessThanOrEqualTo.convertToRuleSpec() + NumInputRuleSpecDto.RuleTypeCase.IS_GREATER_THAN_OR_EQUAL_TO -> + isGreaterThanOrEqualTo.convertToRuleSpec() + NumInputRuleSpecDto.RuleTypeCase.IS_INCLUSIVELY_BETWEEN -> + isInclusivelyBetween.convertToRuleSpec() + NumInputRuleSpecDto.RuleTypeCase.IS_WITHIN_TOLERANCE -> isWithinTolerance.convertToRuleSpec() + NumInputRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun NumInputRuleSpecDto.EqualsSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "Equals" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun NumInputRuleSpecDto.IsLessThanSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsLessThan" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun NumInputRuleSpecDto.IsGreaterThanSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsGreaterThan" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun NumInputRuleSpecDto.IsLessThanOrEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsLessThanOrEqualTo" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun NumInputRuleSpecDto.IsGreaterThanOrEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsGreaterThanOrEqualTo" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun NumInputRuleSpecDto.IsInclusivelyBetweenSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsInclusivelyBetween" + putInput("a", dto.inputLowerInclusive.convertToInteractionObject()) + putInput("b", dto.inputUpperInclusive.convertToInteractionObject()) + }.build() + } + + private fun NumInputRuleSpecDto.IsWithinToleranceSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsWithinTolerance" + putInput("tol", dto.inputTolerance.convertToInteractionObject()) + putInput("x", dto.inputComparedValue.convertToInteractionObject()) + }.build() + } + + private fun TextInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "TextInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + dto.solution.takeIf { + dto.hasSolution() + }?.convertToSolution(contentIdTracker)?.let { this.solution = it } + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap(contentIdTracker)) + }.build() + } + + private fun TextInputInstanceDto.SolutionDto.convertToSolution( + contentIdTracker: ContentIdTracker + ): Solution? { + val dto = this + return Solution.newBuilder().apply { + if (dto.baseSolution.hasExplanation()) { + this.explanation = contentIdTracker.extractSubtitledHtml(dto.baseSolution.explanation) + } + this.correctAnswer = dto.correctAnswer.convertToNormalizedStringObject() + // Whether the answer is exclusive isn't used. + }.build().takeIf { it != Solution.getDefaultInstance() } + } + + private fun TextInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec(contentIdTracker) }) + // Training data isn't used. + }.build() + } + + private fun TextInputInstanceDto.RuleSpecDto.convertToRuleSpec( + contentIdTracker: ContentIdTracker + ): RuleSpec { + return when (ruleTypeCase) { + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.EQUALS -> + equals.convertToRuleSpec(contentIdTracker) + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.STARTS_WITH -> + startsWith.convertToRuleSpec(contentIdTracker) + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.CONTAINS -> + contains.convertToRuleSpec(contentIdTracker) + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.FUZZY_EQUALS -> + fuzzyEquals.convertToRuleSpec(contentIdTracker) + TextInputInstanceDto.RuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun TextInputInstanceDto.RuleSpecDto.EqualsSpecDto.convertToRuleSpec( + contentIdTracker: ContentIdTracker + ): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "Equals" + putInput("x", dto.input.convertToInteractionObject(contentIdTracker)) + }.build() + } + + private fun TextInputInstanceDto.RuleSpecDto.StartsWithSpecDto.convertToRuleSpec( + contentIdTracker: ContentIdTracker + ): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "StartsWith" + putInput("x", dto.input.convertToInteractionObject(contentIdTracker)) + }.build() + } + + private fun TextInputInstanceDto.RuleSpecDto.ContainsSpecDto.convertToRuleSpec( + contentIdTracker: ContentIdTracker + ): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "Contains" + putInput("x", dto.input.convertToInteractionObject(contentIdTracker)) + }.build() + } + + private fun TextInputInstanceDto.RuleSpecDto.FuzzyEqualsSpecDto.convertToRuleSpec( + contentIdTracker: ContentIdTracker + ): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "FuzzyEquals" + putInput("x", dto.input.convertToInteractionObject(contentIdTracker)) + }.build() + } + + private fun TextInputInstanceDto.CustomizationArgsDto.convertToArgsMap( + contentIdTracker: ContentIdTracker + ) = mapOf( + "placeholder" to contentIdTracker.extractSubtitledUnicode(placeholder).wrap(), + "rows" to rows.wrap() + ) + + private fun DragAndDropSortInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "DragAndDropSortInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + dto.solution.takeIf { + dto.hasSolution() + }?.convertToSolution(contentIdTracker)?.let { this.solution = it } + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap(contentIdTracker)) + }.build() + } + + private fun DragAndDropSortInputInstanceDto.SolutionDto.convertToSolution( + contentIdTracker: ContentIdTracker + ): Solution? { + val dto = this + return Solution.newBuilder().apply { + if (dto.baseSolution.hasExplanation()) { + this.explanation = contentIdTracker.extractSubtitledHtml(dto.baseSolution.explanation) + } + this.correctAnswer = dto.correctAnswer.convertToInteractionObject() + // Whether the answer is exclusive isn't used. + }.build().takeIf { it != Solution.getDefaultInstance() } + } + + private fun DragAndDropSortInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun DragDropSortRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + DragDropSortRuleSpecTypeCase.IS_EQUAL_TO_ORDERING -> isEqualToOrdering.convertToRuleSpec() + DragDropSortRuleSpecTypeCase.IS_EQUAL_TO_ORDERING_WITH_ONE_ITEM_AT_INCORRECT_POSITION -> + isEqualToOrderingWithOneItemAtIncorrectPosition.convertToRuleSpec() + DragDropSortRuleSpecTypeCase.HAS_ELEMENT_X_AT_POSITION_Y -> + hasElementXAtPositionY.convertToRuleSpec() + DragDropSortRuleSpecTypeCase.HAS_ELEMENT_X_BEFORE_ELEMENT_Y -> + hasElementXBeforeElementY.convertToRuleSpec() + DragDropSortRuleSpecTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun DragDropSortRuleSpecDto.IsEqualToOrderingSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsEqualToOrdering" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun IsEqualToOrderingWithOneItemAtIncorrectPositionSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsEqualToOrderingWithOneItemAtIncorrectPosition" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun DragDropSortRuleSpecDto.HasElementXAtPositionYSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "HasElementXAtPositionY" + putInput("x", dto.element.convertToInteractionObject()) + putInput("y", dto.position.convertToNonNegativeInteractionObject()) + }.build() + } + + private fun HasElementXBeforeElementYSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "HasElementXBeforeElementY" + putInput("x", dto.consideredElement.convertToInteractionObject()) + putInput("y", dto.laterElement.convertToInteractionObject()) + }.build() + } + + private fun DragAndDropSortInputInstanceDto.CustomizationArgsDto.convertToArgsMap( + contentIdTracker: ContentIdTracker + ) = mapOf( + "choices" to choicesList.map { contentIdTracker.extractSubtitledHtml(it).wrap() }.wrap(), + "allowMultipleItemsInSamePosition" to allowMultipleItemsInSamePosition.wrap() + ) + + private fun ImageClickInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "ImageClickInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap()) + }.build() + } + + private fun ImageClickInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun ImageClickInputInstanceDto.RuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + ImageClickInputInstanceDto.RuleSpecDto.RuleTypeCase.IS_IN_REGION -> + isInRegion.convertToRuleSpec() + ImageClickInputInstanceDto.RuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun IsInRegionSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsInRegion" + putInput("x", dto.inputRegion.convertToNormalizedStringObject()) + }.build() + } + + private fun ImageClickInputInstanceDto.CustomizationArgsDto.convertToArgsMap() = + mapOf("imageAndRegions" to imageAndRegions.convertToImageWithRegions().wrap()) + + private fun RatioExpressionInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "RatioExpressionInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + dto.solution.takeIf { + dto.hasSolution() + }?.convertToSolution(contentIdTracker)?.let { this.solution = it } + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap(contentIdTracker)) + }.build() + } + + private fun RatioExpressionInputInstanceDto.SolutionDto.convertToSolution( + contentIdTracker: ContentIdTracker + ): Solution? { + val dto = this + return Solution.newBuilder().apply { + if (dto.baseSolution.hasExplanation()) { + this.explanation = contentIdTracker.extractSubtitledHtml(dto.baseSolution.explanation) + } + this.correctAnswer = dto.correctAnswer.convertToInteractionObject() + // Whether the answer is exclusive isn't used. + }.build().takeIf { it != Solution.getDefaultInstance() } + } + + private fun RatioExpressionInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun RatioRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + RatioRuleSpecDto.RuleTypeCase.EQUALS -> equals.convertToRuleSpec() + RatioRuleSpecDto.RuleTypeCase.IS_EQUIVALENT -> isEquivalent.convertToRuleSpec() + RatioRuleSpecDto.RuleTypeCase.HAS_NUMBER_OF_TERMS_EQUAL_TO -> + hasNumberOfTermsEqualTo.convertToRuleSpec() + RatioRuleSpecDto.RuleTypeCase.HAS_SPECIFIC_TERM_EQUAL_TO -> + hasSpecificTermEqualTo.convertToRuleSpec() + RatioRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun RatioRuleSpecDto.EqualsSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "Equals" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun RatioRuleSpecDto.IsEquivalentSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsEquivalent" + putInput("x", dto.input.convertToInteractionObject()) + }.build() + } + + private fun RatioRuleSpecDto.HasNumberOfTermsEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "HasNumberOfTermsEqualTo" + putInput("y", dto.inputTermCount.convertToNonNegativeInteractionObject()) + }.build() + } + + private fun RatioRuleSpecDto.HasSpecificTermEqualToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "HasSpecificTermEqualTo" + putInput("x", dto.inputTermIndex.convertToNonNegativeInteractionObject()) + putInput("y", dto.inputExpectedTermValue.convertToNonNegativeInteractionObject()) + }.build() + } + + private fun RatioExpressionInputInstanceDto.CustomizationArgsDto.convertToArgsMap( + contentIdTracker: ContentIdTracker + ) = mapOf( + "placeholder" to contentIdTracker.extractSubtitledUnicode(placeholder).wrap(), + "numberOfTerms" to numberOfTerms.wrap() + ) + + private fun AlgebraicExpressionInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "AlgebraicExpressionInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + dto.solution.takeIf { + dto.hasSolution() + }?.convertToSolution(contentIdTracker)?.let { this.solution = it } + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap()) + }.build() + } + + private fun AlgebraicExpressionInputInstanceDto.SolutionDto.convertToSolution( + contentIdTracker: ContentIdTracker + ): Solution? { + val dto = this + return Solution.newBuilder().apply { + if (dto.baseSolution.hasExplanation()) { + this.explanation = contentIdTracker.extractSubtitledHtml(dto.baseSolution.explanation) + } + this.correctAnswer = dto.correctAnswer.convertToMathExpressionObject() + // Whether the answer is exclusive isn't used. + }.build().takeIf { it != Solution.getDefaultInstance() } + } + + private fun AlgebraicExpressionInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun AlgebraRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + AlgebraRuleSpecDto.RuleTypeCase.MATCHES_EXACTLY_WITH -> matchesExactlyWith.convertToRuleSpec() + AlgebraRuleSpecDto.RuleTypeCase.MATCHES_UP_TO_TRIVIAL_MANIPULATIONS -> + matchesUpToTrivialManipulations.convertToRuleSpec() + AlgebraRuleSpecDto.RuleTypeCase.IS_EQUIVALENT_TO -> isEquivalentTo.convertToRuleSpec() + AlgebraRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun AlgebraRuleSpecDto.MatchesExactlyWithSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "MatchesExactlyWith" + putInput("x", dto.algebraicExpression.convertToMathExpressionObject()) + }.build() + } + + private fun AlgebraMatchesUpToTrivialManipulationsSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "MatchesUpToTrivialManipulations" + putInput("x", dto.algebraicExpression.convertToMathExpressionObject()) + }.build() + } + + private fun AlgebraRuleSpecDto.IsEquivalentToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsEquivalentTo" + putInput("x", dto.algebraicExpression.convertToMathExpressionObject()) + }.build() + } + + private fun AlgebraicExpressionInputInstanceDto.CustomizationArgsDto.convertToArgsMap() = mapOf( + "customOskLetters" to customOskLettersList.map { it.wrap() }.wrap(), + "useFractionForDivision" to useFractionForDivision.wrap() + ) + + private fun MathEquationInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "MathEquationInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + dto.solution.takeIf { + dto.hasSolution() + }?.convertToSolution(contentIdTracker)?.let { this.solution = it } + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap()) + }.build() + } + + private fun MathEquationInputInstanceDto.SolutionDto.convertToSolution( + contentIdTracker: ContentIdTracker + ): Solution? { + val dto = this + return Solution.newBuilder().apply { + if (dto.baseSolution.hasExplanation()) { + this.explanation = contentIdTracker.extractSubtitledHtml(dto.baseSolution.explanation) + } + this.correctAnswer = dto.correctAnswer.convertToMathExpressionObject() + // Whether the answer is exclusive isn't used. + }.build().takeIf { it != Solution.getDefaultInstance() } + } + + private fun MathEquationInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun MathEqRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + MathEqRuleSpecDto.RuleTypeCase.MATCHES_EXACTLY_WITH -> matchesExactlyWith.convertToRuleSpec() + MathEqRuleSpecDto.RuleTypeCase.MATCHES_UP_TO_TRIVIAL_MANIPULATIONS -> + matchesUpToTrivialManipulations.convertToRuleSpec() + MathEqRuleSpecDto.RuleTypeCase.IS_EQUIVALENT_TO -> isEquivalentTo.convertToRuleSpec() + MathEqRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun MathEqRuleSpecDto.MatchesExactlyWithSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "MatchesExactlyWith" + putInput("x", dto.mathEquation.convertToMathExpressionObject()) + }.build() + } + + private fun MathEqMatchesUpToTrivialManipulationsSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "MatchesUpToTrivialManipulations" + putInput("x", dto.mathEquation.convertToMathExpressionObject()) + }.build() + } + + private fun MathEqRuleSpecDto.IsEquivalentToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsEquivalentTo" + putInput("x", dto.mathEquation.convertToMathExpressionObject()) + }.build() + } + + private fun MathEquationInputInstanceDto.CustomizationArgsDto.convertToArgsMap() = mapOf( + "customOskLetters" to customOskLettersList.map { it.wrap() }.wrap(), + "useFractionForDivision" to useFractionForDivision.wrap() + ) + + private fun NumericExpressionInputInstanceDto.convertToInteraction( + contentIdTracker: ContentIdTracker + ): Interaction { + val dto = this + return Interaction.newBuilder().apply { + this.id = "NumericExpressionInput" + addAllAnswerGroups(dto.answerGroupsList.map { it.convertToAnswerGroup(contentIdTracker) }) + dto.solution.takeIf { + dto.hasSolution() + }?.convertToSolution(contentIdTracker)?.let { this.solution = it } + addAllHint(dto.hintsList.map { it.convertToOutcome(contentIdTracker) }) + this.defaultOutcome = dto.defaultOutcome.convertToOutcome(contentIdTracker) + putAllCustomizationArgs(dto.customizationArgs.convertToArgsMap(contentIdTracker)) + }.build() + } + + private fun NumericExpressionInputInstanceDto.SolutionDto.convertToSolution( + contentIdTracker: ContentIdTracker + ): Solution? { + val dto = this + return Solution.newBuilder().apply { + if (dto.baseSolution.hasExplanation()) { + this.explanation = contentIdTracker.extractSubtitledHtml(dto.baseSolution.explanation) + } + this.correctAnswer = dto.correctAnswer.convertToMathExpressionObject() + // Whether the answer is exclusive isn't used. + }.build().takeIf { it != Solution.getDefaultInstance() } + } + + private fun NumericExpressionInputInstanceDto.AnswerGroupDto.convertToAnswerGroup( + contentIdTracker: ContentIdTracker + ): AnswerGroup { + val dto = this + return AnswerGroup.newBuilder().apply { + this.taggedSkillMisconception = + dto.baseAnswerGroup.taggedSkillMisconception.convertToMisconception() + this.outcome = dto.baseAnswerGroup.outcome.convertToOutcome(contentIdTracker) + addAllRuleSpecs(dto.ruleSpecsList.map { it.convertToRuleSpec() }) + // Training data isn't used. + }.build() + } + + private fun NumExpRuleSpecDto.convertToRuleSpec(): RuleSpec { + return when (ruleTypeCase) { + NumExpRuleSpecDto.RuleTypeCase.MATCHES_EXACTLY_WITH -> matchesExactlyWith.convertToRuleSpec() + NumExpRuleSpecDto.RuleTypeCase.MATCHES_UP_TO_TRIVIAL_MANIPULATIONS -> + matchesUpToTrivialManipulations.convertToRuleSpec() + NumExpRuleSpecDto.RuleTypeCase.IS_EQUIVALENT_TO -> isEquivalentTo.convertToRuleSpec() + NumExpRuleSpecDto.RuleTypeCase.RULETYPE_NOT_SET, null -> + error("Invalid rule spec: $this.") + } + } + + private fun NumExpRuleSpecDto.MatchesExactlyWithSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "MatchesExactlyWith" + putInput("x", dto.numericExpression.convertToMathExpressionObject()) + }.build() + } + + private fun NumExpMatchesUpToTrivialManipulationsSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "MatchesUpToTrivialManipulations" + putInput("x", dto.numericExpression.convertToMathExpressionObject()) + }.build() + } + + private fun NumExpRuleSpecDto.IsEquivalentToSpecDto.convertToRuleSpec(): RuleSpec { + val dto = this + return RuleSpec.newBuilder().apply { + this.ruleType = "IsEquivalentTo" + putInput("x", dto.numericExpression.convertToMathExpressionObject()) + }.build() + } + + private fun NumericExpressionInputInstanceDto.CustomizationArgsDto.convertToArgsMap( + contentIdTracker: ContentIdTracker + ) = mapOf( + "placeholder" to contentIdTracker.extractSubtitledUnicode(placeholder).wrap(), + "useFractionForDivision" to useFractionForDivision.wrap() + ) + + private fun OutcomeDto.convertToOutcome(contentIdTracker: ContentIdTracker): Outcome { + val dto = this + return Outcome.newBuilder().apply { + this.destStateName = dto.destinationState + this.feedback = contentIdTracker.extractSubtitledHtml(dto.feedback) + this.labelledAsCorrect = dto.labelledAsCorrect + // Refresher exploration ID, param changes, and prerequisite skill ID are not used. + }.build() + } + + private fun HintDto.convertToOutcome(contentIdTracker: ContentIdTracker): Hint { + val dto = this + return Hint.newBuilder().apply { + this.hintContent = contentIdTracker.extractSubtitledHtml(dto.hintContent) + }.build() + } + + private fun MisconceptionDto.convertToMisconception(): Misconception { + val dto = this + return Misconception.newBuilder().apply { + this.skillId = dto.skillId + this.misconceptionId = dto.misconceptionId + }.build() + } + + private fun ContentLocalizationsDto.extractDefaultSubtitledHtml(text: SubtitledTextDto) = + defaultMapping.extractSubtitledHtml(text) + + private fun ContentLocalizationsDto.extractDefaultThumbnail( + imageReferenceReplacements: Map + ) = defaultMapping.extractThumbnail(imageReferenceReplacements) + + private fun ContentLocalizationsDto.toTranslationMappings( + imageReferenceReplacements: Map + ) = localizationsList.toTranslationMappings(imageReferenceReplacements) + + private fun ContentLocalizationDto.extractSubtitledHtml(text: SubtitledTextDto): SubtitledHtml = + localizableTextContentMappingMap.getValue(text.contentId).toSubtitledHtml(text.contentId) + + private fun ContentLocalizationDto.extractSubtitledUnicode( + text: SubtitledTextDto + ) = localizableTextContentMappingMap.getValue(text.contentId).toSubtitledUnicode(text.contentId) + + private fun ContentLocalizationDto.extractStringList(contentId: String) = + localizableTextContentMappingMap.getValue(contentId).toStringList() + + private fun ContentLocalizationDto.extractThumbnail( + imageReferenceReplacements: Map + ): LessonThumbnail = thumbnail.toThumbnail(imageReferenceReplacements) + + private fun ContentIdTracker.extractSubtitledHtml(text: SubtitledTextDto): SubtitledHtml = + localizationDto.extractSubtitledHtml(text).also { trackContentId(text.contentId) } + + private fun ContentIdTracker.extractSubtitledUnicode(text: SubtitledTextDto) = + localizationDto.extractSubtitledUnicode(text).also { trackContentId(text.contentId) } + + private fun ContentIdTracker.extractStringList(contentId: String) = + localizationDto.extractStringList(contentId).also { trackContentId(contentId) } + + private fun List.toTranslationMappings( + imageReferenceReplacements: Map + ) = toTranslationMappings(imageReferenceReplacements, filterContentIds = null) + + private fun List.toTranslationMappings( + imageReferenceReplacements: Map, + filterContentIds: Set? + ): Map { + return associateUniquely( + keySelector = { it.language.toLegacyLanguageCode() }, + valueSelector = { + it.localizableTextContentMappingMap.filterKeys { contentId -> + filterContentIds == null || contentId in filterContentIds + }.mapValues { (_, dto) -> dto.toTranslation(imageReferenceReplacements) } + } + ).flipMapping().mapValues { (_, languageMap) -> languageMap.toTranslationMapping() } + } + + private fun List.toVoiceoverMappings(): Map = + toVoiceoverMappings(filterContentIds = null) + + private fun List.toVoiceoverMappings( + filterContentIds: Set? + ): Map { + return associateUniquely( + // Oppia web currently uses 'pt' for Brazilian Portuguese voiceovers. + keySelector = { it.language.toLegacyLanguageCode(portugueseOverride = "pt") }, + valueSelector = { + it.voiceoverContentMappingMap.filterKeys { contentId -> + filterContentIds == null || contentId in filterContentIds + }.mapValues { (_, dto) -> dto.toVoiceover() } + } + ).flipMapping().mapValues { (_, languageMap) -> languageMap.toVoiceoverMapping() } + } + + private fun LocalizableTextDto.toSubtitledHtml(contentId: String): SubtitledHtml { + val dto = this + require(dto.dataFormatCase == LocalizableTextDto.DataFormatCase.SINGLE_LOCALIZABLE_TEXT) { + "Error: localizable text is not a single value and can't be converted to SubtitledHtml." + } + return SubtitledHtml.newBuilder().apply { + this.contentId = contentId + this.html = dto.singleLocalizableText.text + }.build() + } + + private fun LocalizableTextDto.toSubtitledUnicode(contentId: String): SubtitledUnicode { + val dto = this + require(dto.dataFormatCase == LocalizableTextDto.DataFormatCase.SINGLE_LOCALIZABLE_TEXT) { + "Error: localizable text is not a single value and can't be converted to SubtitledHtml." + } + return SubtitledUnicode.newBuilder().apply { + this.contentId = contentId + this.unicodeStr = dto.singleLocalizableText.text + }.build() + } + + private fun LocalizableTextDto.toStringList(): List { + val dto = this + require(dto.dataFormatCase == LocalizableTextDto.DataFormatCase.SET_OF_LOCALIZABLE_TEXT) { + "Error: localizable text is not a multi-value and can't be converted to list of strings." + } + return dto.setOfLocalizableText.textList + } + + private fun LocalizableTextDto.toTranslation( + imageReferenceReplacements: Map + ): Translation { + val dto = this + return Translation.newBuilder().apply { + when (dto.dataFormatCase) { + LocalizableTextDto.DataFormatCase.SINGLE_LOCALIZABLE_TEXT -> + this.html = dto.singleLocalizableText.text.fixImageReferences(imageReferenceReplacements) + LocalizableTextDto.DataFormatCase.SET_OF_LOCALIZABLE_TEXT -> { + this.htmlList = HtmlTranslationList.newBuilder().apply { + addAllHtml( + dto.setOfLocalizableText.textList.fixImageReferences(imageReferenceReplacements) + ) + }.build() + } + LocalizableTextDto.DataFormatCase.DATAFORMAT_NOT_SET, null -> + error("Invalid localizable text: $dto.") + } + }.build() + } + + private fun VoiceoverFileDto.toVoiceover(): Voiceover { + val dto = this + return Voiceover.newBuilder().apply { + this.fileSizeBytes = dto.fileSizeBytes.toLong() + this.fileName = dto.filename + }.build() + } + + private fun ThumbnailDto.toThumbnail( + imageReferenceReplacements: Map + ): LessonThumbnail { + val dto = this + val oldFilename = dto.referencedImage.filename + return LessonThumbnail.newBuilder().apply { + this.thumbnailFilename = imageReferenceReplacements[oldFilename] ?: oldFilename + this.backgroundColorRgb = dto.backgroundColorRgb + }.build() + } + + private fun LanguageType.toLegacyLanguageCode(portugueseOverride: String? = null): String { + return when (this) { + LanguageType.ENGLISH -> "en" + LanguageType.ARABIC -> "ar" + LanguageType.HINDI -> "hi" + LanguageType.HINGLISH -> "hi-en" + LanguageType.BRAZILIAN_PORTUGUESE -> portugueseOverride ?: "pt-BR" + LanguageType.SWAHILI -> "sw" + LanguageType.NIGERIAN_PIDGIN -> "pcm" + LanguageType.LANGUAGE_CODE_UNSPECIFIED, LanguageType.UNRECOGNIZED -> + error("Invalid language code: $this.") + } + } + + private fun Map.toTranslationMapping(): TranslationMapping { + return TranslationMapping.newBuilder().apply { + putAllTranslationMapping(this@toTranslationMapping) + }.build() + } + + private fun Map.toVoiceoverMapping(): VoiceoverMapping = + VoiceoverMapping.newBuilder().apply { putAllVoiceoverMapping(this@toVoiceoverMapping) }.build() + + private fun ImageWithRegionsDto.convertToImageWithRegions(): ImageWithRegions { + val dto = this + return ImageWithRegions.newBuilder().apply { + this.imagePath = dto.imageFilePath + addAllLabelRegions(dto.labeledRegionsList.map { it.convertToLabeledRegion() }) + }.build() + } + + private fun LabeledRegionDto.convertToLabeledRegion(): LabeledRegion { + val dto = this + return LabeledRegion.newBuilder().apply { + this.label = dto.label + check(dto.regionTypeCase == NORMALIZED_RECTANGLE_2D) { "Invalid region: $dto." } + this.region = LabeledRegion.Region.newBuilder().apply { + this.regionType = LabeledRegion.Region.RegionType.RECTANGLE + this.area = dto.normalizedRectangle2D.convertToLabeledRegion() + }.build() + }.build() + } + + private fun NormalizedRectangle2dDto.convertToLabeledRegion(): NormalizedRectangle2d { + val dto = this + return NormalizedRectangle2d.newBuilder().apply { + this.upperLeft = dto.topLeft.convertToPoint2d() + this.lowerRight = dto.bottomRight.convertToPoint2d() + }.build() + } + + private fun NormalizedPoint2dDto.convertToPoint2d(): Point2d { + val dto = this + return Point2d.newBuilder().apply { + this.x = dto.x.toFloat() + this.y = dto.y.toFloat() + }.build() + } + + private fun String.convertToNormalizedStringObject(): InteractionObject = + InteractionObject.newBuilder().setNormalizedString(this).build() + + private fun Int.convertToSignedInteractionObject(): InteractionObject = + InteractionObject.newBuilder().setSignedInt(this).build() + + private fun Int.convertToNonNegativeInteractionObject(): InteractionObject = + InteractionObject.newBuilder().setNonNegativeInt(this).build() + + private fun Double.convertToInteractionObject(): InteractionObject = + InteractionObject.newBuilder().setReal(this).build() + + private fun FractionDto.convertToInteractionObject(): InteractionObject = + InteractionObject.newBuilder().setFraction(convertToFraction()).build() + + private fun RatioExpressionDto.convertToInteractionObject(): InteractionObject = + InteractionObject.newBuilder().setRatioExpression(convertToRatioExpression()).build() + + private fun TranslatableSetOfNormalizedStringDto.convertToInteractionObject( + contentIdTracker: ContentIdTracker + ): InteractionObject { + val dto = this + return InteractionObject.newBuilder().apply { + this.translatableSetOfNormalizedString = dto.convertToSetOfNormalizedString(contentIdTracker) + }.build() + } + + private fun TranslatableHtmlContentIdDto.convertToInteractionObject(): InteractionObject { + return InteractionObject.newBuilder().setTranslatableHtmlContentId( + convertToTranslatableContentId() + ).build() + } + + private fun SetOfTranslatableHtmlContentIdsDto.convertToInteractionObject(): InteractionObject { + return InteractionObject.newBuilder().setSetOfTranslatableHtmlContentIds( + convertToProtoV1Version() + ).build() + } + + private fun XlatableContentIdsSetListDto.convertToInteractionObject(): InteractionObject { + return InteractionObject.newBuilder().setListOfSetsOfTranslatableHtmlContentIds( + convertToProtoV1Version() + ).build() + } + + private fun String.convertToMathExpressionObject(): InteractionObject = + InteractionObject.newBuilder().setMathExpression(this).build() + + private fun FractionDto.convertToFraction(): Fraction { + val dto = this + return Fraction.newBuilder().apply { + this.isNegative = dto.isNegative + this.wholeNumber = dto.wholeNumber + this.numerator = dto.numerator + this.denominator = dto.denominator + }.build() + } + + private fun RatioExpressionDto.convertToRatioExpression(): RatioExpression = + RatioExpression.newBuilder().addAllRatioComponent(componentsList).build() + + private fun TranslatableSetOfNormalizedStringDto.convertToSetOfNormalizedString( + contentIdTracker: ContentIdTracker + ): TranslatableSetOfNormalizedString { + val dto = this + return TranslatableSetOfNormalizedString.newBuilder().apply { + this.contentId = dto.contentId + addAllNormalizedStrings(contentIdTracker.extractStringList(dto.contentId)) + }.build() + } + + private fun TranslatableHtmlContentIdDto.convertToTranslatableContentId() = + TranslatableHtmlContentId.newBuilder().setContentId(contentId).build() + + // Uses a different name since the conventional version would be too long to fit on one line. + private fun XlatableContentIdsSetDto.convertToProtoV1Version(): XlatableContentIdsSet { + val dto = this + return SetOfTranslatableHtmlContentIds.newBuilder().apply { + addAllContentIds(dto.contentIdsList.map { it.convertToTranslatableContentId() }) + }.build() + } + + // Uses a different name since the conventional version would be too long to fit on one line. + private fun XlatableContentIdsSetListDto.convertToProtoV1Version(): XlatableContentIdsSetList { + val dto = this + return ListOfSetsOfTranslatableHtmlContentIds.newBuilder().apply { + addAllContentIdLists(dto.contentIdSetsList.map { it.convertToProtoV1Version() }) + }.build() + } + + private fun SubtitledUnicode.wrap(): SchemaObject = + SchemaObject.newBuilder().apply { this.subtitledUnicode = this@wrap }.build() + + private fun SubtitledHtml.wrap(): SchemaObject { + return SchemaObject.newBuilder().apply { + this.customSchemaValue = CustomSchemaValue.newBuilder().apply { + this.subtitledHtml = this@wrap + }.build() + }.build() + } + + private fun Boolean.wrap(): SchemaObject = + SchemaObject.newBuilder().apply { this.boolValue = this@wrap }.build() + + private fun Int.wrap(): SchemaObject = + SchemaObject.newBuilder().apply { this.signedInt = this@wrap }.build() + + private fun String.wrap(): SchemaObject = + SchemaObject.newBuilder().apply { this.normalizedString = this@wrap }.build() + + private fun ImageWithRegions.wrap(): SchemaObject { + return SchemaObject.newBuilder().apply { + this.customSchemaValue = CustomSchemaValue.newBuilder().apply { + this.imageWithRegions = this@wrap + }.build() + }.build() + } + + private fun List.wrap(): SchemaObject { + return SchemaObject.newBuilder().apply { + this.schemaObjectList = SchemaObjectList.newBuilder().apply { + addAllSchemaObject(this@wrap) + }.build() + }.build() + } + + private const val CUSTOM_IMG_TAG = "oppia-noninteractive-image" + private const val CUSTOM_MATH_TAG = "oppia-noninteractive-math" + + private fun String.fixImageReferences(imageReferenceReplacements: Map): String = + fixImageTags(imageReferenceReplacements).fixMathTags(imageReferenceReplacements) + + private fun Iterable.fixImageReferences( + imageReferenceReplacements: Map + ): List = map { it.fixImageReferences(imageReferenceReplacements) } + + private fun String.fixImageTags(imageReferenceReplacements: Map): String = + fixTags(CUSTOM_IMG_TAG, imageReferenceReplacements) + + private fun String.fixMathTags(imageReferenceReplacements: Map): String = + fixTags(CUSTOM_MATH_TAG, imageReferenceReplacements) + + private fun String.fixTags(tag: String, imageReferenceReplacements: Map): String { + // Replace tags in reverse order to avoid invalidating ranges. + return findTags(tag).reversed().fold(this) { updatedStr, nextElementRange -> + updatedStr.replaceReferences(nextElementRange, imageReferenceReplacements) + } + } + + private fun String.replaceReferences( + placement: IntRange, + imageReferenceReplacements: Map + ): String { + return substring(placement).let { element -> + // This could be done much more efficiently by extracting the image reference. + imageReferenceReplacements.entries.find { (needle, _) -> + needle in element + }?.let { (needle, replacement) -> element.replace(needle, replacement) } + }?.let { replaceRange(placement, it) } ?: this + } + + private fun String.findTags(tag: String): List { + return generateSequence(findNextTag(tag, startFrom = 0)) { previousTagRange -> + findNextTag(tag, previousTagRange.last + 1) + }.toList() + } + + private fun String.findNextTag(tag: String, startFrom: Int): IntRange? { + val startTagIndex = indexOf("<$tag", startFrom).takeIf { it != -1 } ?: return null + val endTagIndex1 = indexOf("/>", startTagIndex).takeIf { it != -1 } + val endTagIndex2 = indexOf(" endTagIndex1.coerceAtMost(endTagIndex2) + endTagIndex1 != null -> endTagIndex1 + endTagIndex2 != null -> endTagIndex2 + else -> return null + } + return startTagIndex until endTagIndex + } + + private fun Iterable.associateUniquely( + keySelector: (T) -> K, + valueSelector: (T) -> V + ): Map { + return groupBy(keySelector, valueSelector).mapValues { (key, values) -> + values.singleOrNull() ?: error("Error: $key was present more than once in collection.") + } + } + + private fun Map>.flipMapping(): Map> { + // First, create the outer map with all possible keys. + val allNewOuterKeys = values.flatMapTo(mutableSetOf()) { it.keys } + return allNewOuterKeys.associateWith { mutableMapOf() }.also { newOuterMap -> + // Next, iterate across all previous inner maps to create the new entries. + entries.forEach { (prevOuterKey, prevInnerMap) -> + prevInnerMap.entries.forEach { (prevInnerKey, value) -> + newOuterMap.getValue(prevInnerKey)[prevOuterKey] = value + } + } + } + } + + private class ContentIdTracker(val localizationDto: ContentLocalizationDto) { + private val internalContentIds by lazy { mutableSetOf() } + val contentIds: Set get() = internalContentIds + + fun trackContentId(contentId: String) { + internalContentIds += contentId + } + } +} diff --git a/scripts/src/java/org/oppia/android/scripts/assets/ImageRepairer.kt b/scripts/src/java/org/oppia/android/scripts/assets/ImageRepairer.kt new file mode 100644 index 00000000000..a3eed2782b1 --- /dev/null +++ b/scripts/src/java/org/oppia/android/scripts/assets/ImageRepairer.kt @@ -0,0 +1,122 @@ +package org.oppia.android.scripts.assets + +import com.github.weisj.jsvg.parser.SVGLoader +import org.oppia.android.scripts.assets.ImageRepairer.Companion.resizeTo +import java.awt.Color +import java.awt.Image +import java.awt.image.BufferedImage +import java.io.ByteArrayOutputStream +import javax.imageio.ImageIO +import kotlin.math.roundToInt + +class ImageRepairer { + fun convertToPng(filename: String, svgImageContents: String): RepairedImage { + if ("data:image/png;base64" !in svgImageContents) return RepairedImage.NoRepairNeeded + val loader = SVGLoader() + val svgDocument = + svgImageContents.byteInputStream().use { loader.load(it) } + ?: error("Failed to load: $filename.") + val size = svgDocument.size() + val extractedWidth = filename.extractWidthFromFilename().toFloat() + val extractedHeight = filename.extractHeightFromFilename().toFloat() + val chosenWidth = extractedWidth.convertOppiaPxToStandardAndroidPx().roundToInt() + val chosenHeight = extractedHeight.convertOppiaPxToStandardAndroidPx().roundToInt() + // Render at a larger size to reduce aliasing from the underlying rendering library (but render + // at no larger than 5x the . + val renderWidth = chosenWidth * 5 + val renderHeight = chosenHeight * 5 + val scaleFactorX = renderWidth.toDouble() / size.getWidth() + val scaleFactorY = renderHeight.toDouble() / size.getHeight() + val renderImage = BufferedImage(renderWidth, renderHeight, BufferedImage.TYPE_INT_ARGB) + val graphics = renderImage.createGraphics() + graphics.scale(scaleFactorX, scaleFactorY) + svgDocument.render(/* component = */ null, graphics) + graphics.dispose() + val image = renderImage.resizeTo(chosenWidth, chosenHeight) + return ByteArrayOutputStream().use { + ImageIO.write(image, /* formatName = */ "png", it) + return@use RepairedImage.RenderedSvg(it.toByteArray().toList(), chosenWidth, chosenHeight) + } + } + + fun areEqualImages(extension: String, imageData1: ByteArray, imageData2: ByteArray): Boolean { + if (extension == "svg") return imageData1.decodeToString() == imageData2.decodeToString() + val image1 = imageData1.inputStream().use { + ImageIO.read(it) + } ?: error("Cannot read file of type $extension (data size: ${imageData1.size} bytes).") + val image2 = imageData2.inputStream().use { + ImageIO.read(it) + } ?: error("Cannot read file of type $extension (data size: ${imageData2.size} bytes).") + return areImagesEqual(image1, image2) + } + + sealed class RepairedImage { + data class RenderedSvg( + val pngContents: List, + val width: Int, + val height: Int + ) : RepairedImage() + + object NoRepairNeeded : RepairedImage() + } + + private fun areImagesEqual(image1: BufferedImage, image2: BufferedImage): Boolean { + if (image1.width != image2.width) return false + if (image1.height != image2.height) return false + for (y in 0 until image1.height) { + for (x in 0 until image1.width) { + if (image1.getRGB(x, y) != image2.getRGB(x, y)) return false + } + } + return true + } + + private companion object { + private val WIDTH_REGEX by lazy { "width_(\\d+)".toRegex() } + private val HEIGHT_REGEX by lazy { "height_(\\d+)".toRegex() } + private val TRANSPARENT = Color(/* r = */ 0, /* g = */ 0, /* b = */ 0, /* a = */ 0) + + private const val REFERENCE_MONITOR_PPI = 81.589f + private const val RELATIVE_SIZE_ADJUSTMENT_FACTOR = 0.15f + private const val MOST_COMMON_DEVICE_PPI = 270f // Based on observed usage. + private const val PRESUMED_MOST_COMMON_DEVICE_DENSITY = 320f + // This is computed per Android density documentation. + private const val COMMON_DEVICE_DENSITY_SCALAR = PRESUMED_MOST_COMMON_DEVICE_DENSITY / 160f + private const val OPPIA_LOCAL_IMAGE_SPACE_CONVERSION_FACTOR = + // The conversion here is from Oppia pixel to MDPI pixel (which can be treated as dp since + // 1px=1dp in MDPI) for later scaling according to the user's set display density. + (MOST_COMMON_DEVICE_PPI / REFERENCE_MONITOR_PPI) * RELATIVE_SIZE_ADJUSTMENT_FACTOR + + private fun String.extractWidthFromFilename(): Int { + return WIDTH_REGEX.find(this)?.destructured?.component1()?.toIntOrNull() + ?: error("Invalid filename: $this.") + } + + private fun String.extractHeightFromFilename(): Int { + return HEIGHT_REGEX.find(this)?.destructured?.component1()?.toIntOrNull() + ?: error("Invalid filename: $this.") + } + + private fun BufferedImage.resizeTo(newWidth: Int, newHeight: Int): BufferedImage = + getScaledInstance(newWidth, newHeight, Image.SCALE_SMOOTH).buffered(type) + + private fun Image.buffered(imageType: Int): BufferedImage { + return BufferedImage( + getWidth(/* observer = */ null), getHeight(/* observer = */ null), imageType + ).also { + val graphics = it.createGraphics() + graphics.drawImage( + /* img = */ this, + /* x = */ 0, + /* y = */ 0, + /* bgColor = */ TRANSPARENT, + /* observer = */ null + ) + graphics.dispose() + } + } + + private fun Float.convertOppiaPxToStandardAndroidPx() = + this * COMMON_DEVICE_DENSITY_SCALAR * OPPIA_LOCAL_IMAGE_SPACE_CONVERSION_FACTOR + } +} diff --git a/scripts/src/java/org/oppia/android/scripts/gae/GaeAndroidEndpointJsonImpl.kt b/scripts/src/java/org/oppia/android/scripts/gae/GaeAndroidEndpointJsonImpl.kt index fa2f80771a1..3fb4dd39376 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/GaeAndroidEndpointJsonImpl.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/GaeAndroidEndpointJsonImpl.kt @@ -17,6 +17,7 @@ import org.oppia.android.scripts.gae.compat.StructureCompatibilityChecker.Compat import org.oppia.android.scripts.gae.compat.TopicPackRepository import org.oppia.android.scripts.gae.compat.TopicPackRepository.MetricCallbacks.DataGroupType import org.oppia.android.scripts.gae.json.AndroidActivityHandlerService +import org.oppia.android.scripts.gae.json.GaeClassroom import org.oppia.android.scripts.gae.json.GaeSkill import org.oppia.android.scripts.gae.json.GaeStory import org.oppia.android.scripts.gae.json.GaeSubtopic @@ -67,7 +68,6 @@ class GaeAndroidEndpointJsonImpl( cacheDir: File?, forceCacheLoad: Boolean, private val coroutineDispatcher: CoroutineDispatcher, - private val topicDependencies: Map>, private val imageDownloader: ImageDownloader, private val forcedVersions: DownloadListVersions? ) : GaeAndroidEndpoint { @@ -76,11 +76,7 @@ class GaeAndroidEndpointJsonImpl( apiSecret, gaeBaseUrl, cacheDir, forceCacheLoad, coroutineDispatcher ) } - private val converterInitializer by lazy { - ConverterInitializer( - activityService, coroutineDispatcher, topicDependencies, imageDownloader - ) - } + private lateinit var converterInitializer: ConverterInitializer private val contentCache by lazy { ContentCache() } // TODO: Document that reportProgress's total can change over time (since it starts as an @@ -109,6 +105,14 @@ class GaeAndroidEndpointJsonImpl( coroutineDispatcher, reportProgress ) + + val classrooms = fetchAllClassroomsAsync(tracker).await() + val topicDependencies = classrooms.flatMap { + it.topicIdToPrereqTopicIds.entries + }.groupBy { (topicId, _) -> topicId }.mapValues { (id, prereqsList) -> + check(prereqsList.size == 1) { "Expected one prerequisite topic list for topic: $id." } + prereqsList.single().value.toSet() + } val constraints = CompatibilityConstraints( supportedInteractionIds = SUPPORTED_INTERACTION_IDS, @@ -121,11 +125,15 @@ class GaeAndroidEndpointJsonImpl( topicDependencies = topicDependencies, forcedVersions = forcedVersions ) + converterInitializer = ConverterInitializer( + activityService, coroutineDispatcher, topicDependencies, imageDownloader + ) val jsonConverter = converterInitializer.getJsonToProtoConverter() val topicRepository = converterInitializer.getTopicPackRepository(constraints) - val topicIds = fetchAllClassroomTopicIdsAsync(tracker).await() + val topicIds = classrooms.flatMap { it.topicIdToPrereqTopicIds.keys }.distinct() + tracker.countEstimator.setTopicCount(topicIds.size) val topicCountsTracker = TopicCountsTracker.createFrom(tracker, topicIds) val availableTopicPacks = topicIds.mapIndexed { index, topicId -> @@ -140,6 +148,7 @@ class GaeAndroidEndpointJsonImpl( }.awaitAll().associate { it.id to it.payload } contentCache.addPacks(availableTopicPacks) + jsonConverter.trackClassroomTranslations(classrooms) jsonConverter.trackTopicTranslations(contentCache.topics) jsonConverter.trackStoryTranslations(contentCache.stories) jsonConverter.trackExplorationTranslations(contentCache.explorations.toPacks()) @@ -179,6 +188,11 @@ class GaeAndroidEndpointJsonImpl( }.build() } ) + addAllClassrooms( + classrooms.map { classroom -> + jsonConverter.convertToClassroom(classroom, defaultLanguage) + } + ) }.build() } } @@ -215,31 +229,33 @@ class GaeAndroidEndpointJsonImpl( } } - private fun fetchAllClassroomTopicIdsAsync( + private fun fetchAllClassroomsAsync( tracker: DownloadProgressTracker - ): Deferred> { - // TODO: Revert the temp change once all classrooms are supported in the new format. + ): Deferred> { // TODO: Double check the language verification (since sWBXKH4PZcK6 Swahili isn't 100%). return CoroutineScope(coroutineDispatcher).async { - listOf( - "iX9kYCjnouWN", "sWBXKH4PZcK6", "C4fqwrvqWpRm", "qW12maD4hiA8", "0abdeaJhmfPm", - "5g0nxGUmx5J5" - ).also { - tracker.countEstimator.setTopicCount(it.size) - tracker.reportDownloaded("math") - } -// SUPPORTED_CLASSROOMS.map { classroomName -> -// CoroutineScope(coroutineDispatcher).async { -// activityService.fetchLatestClassroomAsync(classroomName).await().also { -// tracker.reportDownloaded(classroomName) -// } -// } -// }.awaitAll().flatMap(GaeClassroom::topicIds).distinct().also { -// tracker.countEstimator.setTopicCount(it.size) -// } + SUPPORTED_CLASSROOMS.map { classroomName -> + CoroutineScope(coroutineDispatcher).async { + activityService.fetchLatestClassroomAsync(classroomName).await().also { + tracker.reportDownloaded(classroomName) + }.payload + } + }.awaitAll().map { it.filterTopics() } } } + // TODO: Remover this filter once downloading & checking all the other topics works correctly. + private fun GaeClassroom.filterTopics(): GaeClassroom { + return copy( + topicIdToPrereqTopicIds = topicIdToPrereqTopicIds.filterKeys { + it in listOf( + "iX9kYCjnouWN", "sWBXKH4PZcK6", "C4fqwrvqWpRm", "qW12maD4hiA8", "0abdeaJhmfPm", + "5g0nxGUmx5J5" + ) + } + ) + } + private suspend fun fetchStructure( identifier: DownloadRequestStructureIdentifierDto ): DownloadResultDto { @@ -710,7 +726,7 @@ class GaeAndroidEndpointJsonImpl( ) { private var localizationTracker: LocalizationTracker? = null private var jsonToProtoConverter: JsonToProtoConverter? = null - private var topicPackRepositories = + private val topicPackRepositories = mutableMapOf() suspend fun getLocalizationTracker(): LocalizationTracker = diff --git a/scripts/src/java/org/oppia/android/scripts/gae/compat/TopicPackRepository.kt b/scripts/src/java/org/oppia/android/scripts/gae/compat/TopicPackRepository.kt index cf31ef13317..d3c76233025 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/compat/TopicPackRepository.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/compat/TopicPackRepository.kt @@ -130,7 +130,7 @@ class TopicPackRepository( gaeTopic: GaeTopic, metricCallbacks: MetricCallbacks ): List> { - // TODO: Batch different results? + // TODO: Batch results from different types of requests? Should be *much* more efficient. val subtopicsResult = tryLoadSubtopics(gaeTopic.id, gaeTopic.computeContainedSubtopicMap(), metricCallbacks) val storiesResult = tryLoadStories(gaeTopic.computeReferencedStoryIds(), metricCallbacks) @@ -350,12 +350,18 @@ class TopicPackRepository( ): GenericLoadResult { return when (val result = versionStructureMapManager.lookUp(structureId, reference)) { is LoadResult.Pending -> { + val nextVersions = versionStructureMapManager.computeNextBatchOfVersionsToCheck(structureId) + check(nextVersions.isNotEmpty()) { + "At least one reference should be pending for: $reference." + } versionStructureMapManager.update( - structureId, reference.loadVersioned(androidService, compatibilityChecker) + structureId, reference.loadVersioned(androidService, compatibilityChecker, nextVersions) ) // This should be present now. versionStructureMapManager.lookUp(structureId, reference).also { - check(it !is LoadResult.Pending) { "Expected reference to be loaded: $it." } + check(it !is LoadResult.Pending) { + "Expected reference to be loaded: $reference (found: $it)." + } } } is LoadResult.Success, is LoadResult.Failure -> result @@ -551,8 +557,6 @@ private interface VersionedStructureFetcher { } private sealed class VersionedStructureReference { - // TODO: Try 50 or a higher number once multi-version fetching works on Oppia web (see https://github.com/oppia/oppia/issues/18241). - val defaultVersionFetchCount: Int = 1 abstract val structureId: I abstract val version: Int abstract val fetcher: VersionedStructureFetcher @@ -577,16 +581,15 @@ private sealed class VersionedStructureReference { suspend fun loadVersioned( service: AndroidActivityHandlerService, - checker: StructureCompatibilityChecker + checker: StructureCompatibilityChecker, + versionsToRequest: List ): Map, LoadResult> { - val oldestVersionToRequest = (version - defaultVersionFetchCount).coerceAtLeast(1) - val versionsToRequest = (oldestVersionToRequest until version).toList() val structures = fetcher.fetchMultiFromRemoteAsync( structureId, versionsToRequest, service ).toLoadResult(checker) - return versionsToRequest.zip(structures).toMap().mapKeys { (version, _) -> - toNewVersion(version) - } + return versionsToRequest.zip(structures).mapNotNull { (version, result) -> + if (result != null) toNewVersion(version) to result else null + }.toMap() } private suspend fun Deferred>.toLoadResult( @@ -594,19 +597,19 @@ private sealed class VersionedStructureReference { ): LoadResult = await().toLoadResult(checker) @JvmName("listToLoadResult") - private suspend fun Deferred>>.toLoadResult( + private suspend fun Deferred?>>.toLoadResult( checker: StructureCompatibilityChecker - ): List> = await().map { it.toLoadResult(checker) } + ): List?> = await().map { it?.toLoadResult(checker) } private fun VersionedStructure.toLoadResult( checker: StructureCompatibilityChecker ): LoadResult { return when (val compatibilityResult = checkCompatibility(checker, payload)) { Compatible -> LoadResult.Success(payload) - is Incompatible -> LoadResult.Failure(compatibilityResult.failures).also { - // TODO: Remove. - error("Failed to load: $it.") - } + is Incompatible -> LoadResult.Failure(compatibilityResult.failures) // .also { + // // TODO: Remove. + // error("Failed to load: $it.") + // } } } @@ -679,6 +682,9 @@ private sealed class VersionedStructureReference { companion object { const val INVALID_VERSION = 0 + + // TODO: Try 50 or a higher number once multi-version fetching works on Oppia web (see https://github.com/oppia/oppia/issues/18241). + val defaultVersionFetchCount: Int = 1 } } @@ -822,6 +828,8 @@ private interface VersionStructureMapManager { fun findMostRecent(structureId: StructureId): GenericStructureReference + fun computeNextBatchOfVersionsToCheck(structureId: StructureId): List + fun invalidateVersion(structureId: StructureId, reference: GenericStructureReference) fun update( @@ -878,6 +886,14 @@ private class VersionStructureMapManagerTakeLatestImpl( } } + override fun computeNextBatchOfVersionsToCheck(structureId: StructureId): List { + return lock.withLock { + cachedStructures.getValue(structureId).filter { (_, result) -> + result is LoadResult.Pending + }.map { (reference, _) -> reference.version } + }.sortedDescending().take(VersionedStructureReference.defaultVersionFetchCount) + } + override fun invalidateVersion(structureId: StructureId, reference: GenericStructureReference) { lock.withLock { val structureMap = cachedStructures.getValue(structureId) @@ -947,6 +963,14 @@ private class VersionStructureMapManagerFixVersionsImpl( // There's only at most one version per ID, so that's always the 'latest'. override fun findMostRecent(structureId: StructureId) = lookUp(structureId).first + override fun computeNextBatchOfVersionsToCheck(structureId: StructureId): List { + return lock.withLock { + // There's only at most one version per ID, and it's either loaded or isn't. + val (reference, result) = lookUp(structureId) + return@withLock if (result is LoadResult.Pending) listOf(reference.version) else listOf() + } + } + override fun invalidateVersion(structureId: StructureId, reference: GenericStructureReference) { error("Cannot invalidate versions when versions are fixed, for reference:\n$reference") } diff --git a/scripts/src/java/org/oppia/android/scripts/gae/gcs/GcsService.kt b/scripts/src/java/org/oppia/android/scripts/gae/gcs/GcsService.kt index b8df150dd0c..009c1ef33d1 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/gcs/GcsService.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/gcs/GcsService.kt @@ -79,7 +79,8 @@ class GcsService(private val baseUrl: String, private val gcsBucket: String) { EXPLORATION(httpRepresentation = "exploration"), SKILL(httpRepresentation = "skill"), TOPIC(httpRepresentation = "topic"), - STORY(httpRepresentation = "story") + STORY(httpRepresentation = "story"), + CLASSROOM(httpRepresentation = "unknown") // TODO: Figure out what this should be. } enum class ImageType(val httpRepresentation: String) { diff --git a/scripts/src/java/org/oppia/android/scripts/gae/json/AndroidActivityHandlerService.kt b/scripts/src/java/org/oppia/android/scripts/gae/json/AndroidActivityHandlerService.kt index 2cb5cd5b895..3820cd9c2fa 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/json/AndroidActivityHandlerService.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/json/AndroidActivityHandlerService.kt @@ -280,7 +280,11 @@ class AndroidActivityHandlerService( val expectedPrefix = computeFileNamePrefix(type, id, version = "") val mostRecentVersion = cacheDir.listFiles()?.filter { it.extension == "json" && it.nameWithoutExtension.startsWith(expectedPrefix) - }?.maxOfOrNull { it.nameWithoutExtension.substringAfter(expectedPrefix).toInt() } + }?.mapNotNull { file -> + // Files with "latest" 'versions' should always be refetched unless there's an explicitly + // cached version on disk. + file.nameWithoutExtension.substringAfter(expectedPrefix).takeIf { it != "latest" } + }?.maxOfOrNull { it.toInt() } if (mostRecentVersion != null) { return@async checkNotNull(tryLoadFromCache(type, NonLocalized(id, mostRecentVersion))) { "Something went wrong when trying to fetch latest $type from disk: $id." @@ -290,12 +294,12 @@ class AndroidActivityHandlerService( val request = AndroidActivityRequests.Latest(LatestVersion(id)) val remoteStructure = fetch(request).resolveAsync(id).await() - // Ensure that the returned structure has the correct version. - val updatedStructure = if (retrieveStructureVersion != null) { - remoteStructure.copy(version = retrieveStructureVersion(remoteStructure.payload)) - } else remoteStructure - maybeSaveToCache(type, NonLocalized(id, updatedStructure.expectedVersion), updatedStructure) - return@async updatedStructure + // Ensure that the returned structure has the correct version (if it's known). + return@async if (retrieveStructureVersion != null) { + remoteStructure.copy(version = retrieveStructureVersion(remoteStructure.payload)).also { + maybeSaveToCache(type, NonLocalized(id, it.expectedVersion), it) + } + } else remoteStructure.also { maybeSaveToCache(type, LatestVersion(id), it) } } } @@ -359,6 +363,7 @@ class AndroidActivityHandlerService( } } + // TODO: Update caching to ensure all versions are cached along with their analysis results (as part of the repository creation?). This can provide substantially more debugging insight when something goes wrong. private suspend inline fun maybeSaveToCache( type: String, request: ActivityRequest, @@ -444,7 +449,7 @@ class AndroidActivityHandlerService( private companion object { private fun ActivityRequest.convertToFileName(type: String): String { return when (this) { - is LatestVersion -> error("Cannot load/save latest versions of structures.") + is LatestVersion -> "${computeFileNamePrefix(type, id, "latest")}.json" is NonLocalized -> "${computeFileNamePrefix(type, id, version.toString())}.json" is Localized -> "${computeFileNamePrefix(type, id, version.toString())}_lang-$languageCode.json" diff --git a/scripts/src/java/org/oppia/android/scripts/gae/json/GaeClassroom.kt b/scripts/src/java/org/oppia/android/scripts/gae/json/GaeClassroom.kt index c932942c289..38b61191570 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/json/GaeClassroom.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/json/GaeClassroom.kt @@ -5,9 +5,11 @@ import com.squareup.moshi.JsonClass @JsonClass(generateAdapter = true) data class GaeClassroom( + @Json(name = "classroom_id") val id: String, @Json(name = "name") val name: String, @Json(name = "url_fragment") val urlFragment: String, - @Json(name = "topic_ids") val topicIds: List, + @Json(name = "topic_id_to_prerequisite_topic_ids") + val topicIdToPrereqTopicIds: Map>, @Json(name = "course_details") val courseDetails: String, @Json(name = "topic_list_intro") val topicListIntro: String ) diff --git a/scripts/src/java/org/oppia/android/scripts/gae/proto/JsonToProtoConverter.kt b/scripts/src/java/org/oppia/android/scripts/gae/proto/JsonToProtoConverter.kt index fe66cbb4627..23f8078ddb5 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/proto/JsonToProtoConverter.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/proto/JsonToProtoConverter.kt @@ -1,6 +1,7 @@ package org.oppia.android.scripts.gae.proto import org.oppia.android.scripts.gae.json.GaeAnswerGroup +import org.oppia.android.scripts.gae.json.GaeClassroom import org.oppia.android.scripts.gae.json.GaeCustomizationArgValue import org.oppia.android.scripts.gae.json.GaeCustomizationArgValue.GaeImageWithRegions import org.oppia.android.scripts.gae.json.GaeCustomizationArgValue.GaeImageWithRegions.GaeLabeledRegion @@ -48,6 +49,7 @@ import org.oppia.proto.v1.structure.AlgebraicExpressionInputInstanceDto import org.oppia.proto.v1.structure.BaseAnswerGroupDto import org.oppia.proto.v1.structure.BaseSolutionDto import org.oppia.proto.v1.structure.ChapterSummaryDto +import org.oppia.proto.v1.structure.ClassroomDto import org.oppia.proto.v1.structure.ConceptCardDto import org.oppia.proto.v1.structure.ConceptCardDto.WorkedExampleDto import org.oppia.proto.v1.structure.ConceptCardLanguagePackDto @@ -168,6 +170,17 @@ class JsonToProtoConverter( private val localizationTracker: LocalizationTracker, private val topicDependencies: Map> ) { + fun trackClassroomTranslations(classrooms: List) { + for (classroom in classrooms) { + val containerId = LocalizationTracker.ContainerId.createFrom(classroom) + // TODO: Classrooms don't have a language code exposed. + val defaultLanguage = LanguageType.ENGLISH + // TODO: Add missing thumbnail once it's available. + localizationTracker.initializeContainer(containerId, defaultLanguage) + localizationTracker.trackContainerText(containerId, TITLE, classroom.name) + } + } + fun trackTopicTranslations(topics: Map) { for (topic in topics.values) { val containerId = LocalizationTracker.ContainerId.createFrom(topic) @@ -318,6 +331,21 @@ class JsonToProtoConverter( } } + suspend fun convertToClassroom( + gaeClassroom: GaeClassroom, + defaultLanguage: LanguageType + ): ClassroomDto { + val containerId = LocalizationTracker.ContainerId.createFrom(gaeClassroom) + return ClassroomDto.newBuilder().apply { + this.protoVersion = ProtoVersionProvider.createLatestClassroomProtoVersion() + this.id = gaeClassroom.id + this.name = localizationTracker.convertContainerText(containerId, TITLE) + addAllTopicIds(gaeClassroom.topicIdToPrereqTopicIds.keys) + this.localizations = + localizationTracker.computeCompleteLocalizationPack(containerId, defaultLanguage) + }.build() + } + suspend fun convertToDownloadableTopicSummary( gaeTopic: GaeTopic, defaultLanguage: LanguageType, diff --git a/scripts/src/java/org/oppia/android/scripts/gae/proto/LocalizationTracker.kt b/scripts/src/java/org/oppia/android/scripts/gae/proto/LocalizationTracker.kt index 01135bd8db7..a6cdf4baccd 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/proto/LocalizationTracker.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/proto/LocalizationTracker.kt @@ -6,6 +6,7 @@ import com.squareup.moshi.Moshi import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory import kotlinx.coroutines.awaitAll import org.oppia.android.scripts.gae.gcs.GcsService +import org.oppia.android.scripts.gae.json.GaeClassroom import org.oppia.android.scripts.gae.json.GaeEntityTranslations import org.oppia.android.scripts.gae.json.GaeExploration import org.oppia.android.scripts.gae.json.GaeRecordedVoiceovers @@ -183,7 +184,11 @@ class LocalizationTracker private constructor( suspend fun computeSpecificContentLocalization( id: ContainerId, language: LanguageType - ): ContentLocalizationDto = getExpectedContainer(id).computeSpecificContentLocalization(language) + ): ContentLocalizationDto { + return getExpectedContainer(id).also { + checkForNoErrors() + }.computeSpecificContentLocalization(language) + } // TODO: Document that 'defaultLanguage' can redefine the default language of the container based // on available languages. @@ -191,6 +196,7 @@ class LocalizationTracker private constructor( id: ContainerId, defaultLanguage: LanguageType ): ContentLocalizationsDto { + checkForNoErrors() return getExpectedContainer(id).computeCompleteLocalizationPack(defaultLanguage) } @@ -211,6 +217,17 @@ class LocalizationTracker private constructor( return containers.getValue(id) } + private fun checkForNoErrors() { + val allErrors = containers.values.flatMapTo(mutableSetOf()) { it.allErrors } + if (allErrors.isNotEmpty()) { + println("${allErrors.size} errors found:") + allErrors.forEach { + println("- $it") + } + error("Errors found.") + } + } + sealed class ContainerId { abstract val webTranslatableActivityId: TranslatableActivityId? abstract val gcsImageContainerType: GcsService.ImageContainerType @@ -245,6 +262,12 @@ class LocalizationTracker private constructor( override val gcsEntityId: String = subtopicPageIdDto.topicId } + data class Classroom(val id: String) : ContainerId() { + override val webTranslatableActivityId by lazy { TranslatableActivityId.Classroom(id) } + override val gcsImageContainerType = GcsService.ImageContainerType.CLASSROOM + override val gcsEntityId = id + } + data class Topic(val id: String) : ContainerId() { override val webTranslatableActivityId by lazy { TranslatableActivityId.Topic(id) } override val gcsImageContainerType = GcsService.ImageContainerType.TOPIC @@ -276,6 +299,8 @@ class LocalizationTracker private constructor( } companion object { + fun createFrom(gaeClassroom: GaeClassroom): ContainerId = Classroom(gaeClassroom.id) + fun createFrom(gaeTopic: GaeTopic): ContainerId = Topic(gaeTopic.id) fun createFrom(topicId: String, gaeSubtopic: GaeSubtopic): ContainerId { @@ -341,6 +366,8 @@ class LocalizationTracker private constructor( private val defaultAssets: TrackedAssets get() = languages.getValue(defaultLanguage) private val defaultContentIds: Set get() = defaultAssets.allContentIds private val contextsToDownloadFromOppiaWeb = mutableSetOf() + private val errors = mutableSetOf() + val allErrors: Set get() = errors + languages.values.flatMap { it.errors } fun recordDefaultThumbnail(thumbnail: ThumbnailDto) = defaultAssets.recordThumbnail(id, thumbnail) @@ -430,10 +457,40 @@ class LocalizationTracker private constructor( languages.getOrPut(language) { TrackedAssets(language) } private fun ensureDefaultLanguageHasContent(contentId: String) { - check(contentId in defaultContentIds) { - "Attempting to add an asset for a content ID that hasn't been defaulted in container:" + + // TODO: Remove these specific exemptions once they're fixed on upstream web. + val expectedExemptionCase = when { + id !is ContainerId.Exploration -> 0 + id.id == "bWHHbghtVQKU" && contentId == "hint_46" -> 10 + id.id == "W50hotX4h_Up" -> when (contentId) { + "hint_22" -> 20 + "hint_23" -> 21 + else -> 0 + } + id.id == "C8QUgzIETvRv" -> when (contentId) { + "content_220" -> 30 + "feedback_161" -> 31 + "feedback_222" -> 32 + "default_outcome_160" -> 33 + "default_outcome_221" -> 34 + "ca_choices_223" -> 35 + "ca_choices_224" -> 36 + "ca_choices_225" -> 37 + "ca_choices_226" -> 38 + else -> 0 + } + else -> 0 + } + if (contentId !in defaultContentIds && expectedExemptionCase > 0) return + check(expectedExemptionCase == 0) { "Exemption $expectedExemptionCase should be removed." } + if (contentId !in defaultContentIds) { + errors += + "Attempting to add an asset for a content ID that hasn't been defaulted in container:" + " $id, content ID: $contentId." } + // check(contentId in defaultContentIds) { + // "Attempting to add an asset for a content ID that hasn't been defaulted in container:" + + // " $id, content ID: $contentId." + // } } } @@ -442,6 +499,8 @@ class LocalizationTracker private constructor( val textTranslations: MutableMap = mutableMapOf(), val voiceovers: MutableMap = mutableMapOf() ) { + val errors = mutableSetOf() + var thumbnail: ThumbnailDto? = null val allContentIds: Set get() = textTranslations.keys + voiceovers.keys @@ -529,6 +588,27 @@ class LocalizationTracker private constructor( contentId: String, localization: LocalizableTextDto ) { + // TODO: Remove these exemptions once they're fixed in web. + val expectedExemption = when { + id !is ContainerId.Exploration -> false + id.id == "xtbP46LKl1uj" && contentId == "solution_137" -> true + id.id == "ua7FTOXRaRjb" && contentId == "solution_139" -> true + id.id == "sRqParMOyWWB" && contentId == "solution_121" -> true + id.id == "Sl4TGJQhSjmk" && contentId == "solution_85" -> true + id.id == "2EOuIfQHljkN" -> when (contentId) { + "solution_127" -> true + "solution_130" -> true + else -> false + } + else -> false + } + if (contentId in textTranslations && expectedExemption) return + if (contentId in textTranslations) { + errors += + "Translation already recorded for content ID: $contentId, for language: $language, in" + + " container: $id." + return + } require(contentId !in textTranslations) { "Translation already recorded for content ID: $contentId, for language: $language, in" + " container: $id." diff --git a/scripts/src/java/org/oppia/android/scripts/gae/proto/OppiaWebTranslationExtractor.kt b/scripts/src/java/org/oppia/android/scripts/gae/proto/OppiaWebTranslationExtractor.kt index a63b4e61ce7..cd22d99bf87 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/proto/OppiaWebTranslationExtractor.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/proto/OppiaWebTranslationExtractor.kt @@ -30,6 +30,13 @@ class OppiaWebTranslationExtractor private constructor( internal fun computeWebKeyForContent(contentId: String): String = "I18N_${upperCasedActivityType}_${activityId}_${contentId.uppercase()}" + // TODO: Figure out if this should be ID or URL fragment (or if there's even web translations for these). + data class Classroom( + val classroomId: String + ) : TranslatableActivityId(activityType = "classroom") { + override val activityId: String = classroomId + } + data class Topic(val topicId: String) : TranslatableActivityId(activityType = "topic") { override val activityId: String = topicId } diff --git a/scripts/src/java/org/oppia/android/scripts/gae/proto/ProtoVersionProvider.kt b/scripts/src/java/org/oppia/android/scripts/gae/proto/ProtoVersionProvider.kt index 930d0a7e872..892b1d0240a 100644 --- a/scripts/src/java/org/oppia/android/scripts/gae/proto/ProtoVersionProvider.kt +++ b/scripts/src/java/org/oppia/android/scripts/gae/proto/ProtoVersionProvider.kt @@ -4,6 +4,7 @@ import com.google.protobuf.Descriptors.Descriptor import com.google.protobuf.Message import org.oppia.proto.v1.api.ClientCompatibilityContextDto import org.oppia.proto.v1.versions.ApiVersions +import org.oppia.proto.v1.versions.ClassroomProtoVersion import org.oppia.proto.v1.versions.ConceptCardProtoVersion import org.oppia.proto.v1.versions.ExplorationProtoVersion import org.oppia.proto.v1.versions.ImageProtoVersion @@ -25,6 +26,7 @@ object ProtoVersionProvider { private val DEF_STATE_VER = StateProtoVersion.getDefaultInstance() private val DEF_LANGUAGE_VER = LanguageProtosVersion.getDefaultInstance() private val DEF_IMAGE_VER = ImageProtoVersion.getDefaultInstance() + private val DEF_CLASSROOM_VER = ClassroomProtoVersion.getDefaultInstance() private val DEF_TOPIC_LIST_REQ_RESP_VER = TopicListRequestResponseProtoVersion.getDefaultInstance() private val DEF_TOPIC_CONTENT_REQ_RESP_VER = @@ -54,6 +56,9 @@ object ProtoVersionProvider { fun createLatestImageProtoVersion(): ImageProtoVersion = createStructureVersionProto(DEF_IMAGE_VER, ImageProtoVersion.Builder::setVersion) + fun createLatestClassroomProtoVersion(): ClassroomProtoVersion = + createStructureVersionProto(DEF_CLASSROOM_VER, ClassroomProtoVersion.Builder::setVersion) + fun createLatestTopicListProtoVersion(): TopicListRequestResponseProtoVersion { return createApiVersionProto( DEF_TOPIC_LIST_REQ_RESP_VER, TopicListRequestResponseProtoVersion.Builder::setVersion @@ -78,6 +83,7 @@ object ProtoVersionProvider { stateProtoVersion = createLatestStateProtoVersion() languageProtosVersion = createLatestLanguageProtosVersion() imageProtoVersion = createLatestImageProtoVersion() + classroomProtoVersion = createLatestClassroomProtoVersion() }.build() } diff --git a/third_party/maven_install.json b/third_party/maven_install.json index c9a27f18450..a48c60e62d0 100644 --- a/third_party/maven_install.json +++ b/third_party/maven_install.json @@ -1,7 +1,7 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": 1041130706, - "__RESOLVED_ARTIFACTS_HASH": 1257218600, + "__INPUT_ARTIFACTS_HASH": 396608269, + "__RESOLVED_ARTIFACTS_HASH": -1572470801, "conflict_resolution": { "androidx.constraintlayout:constraintlayout:1.1.3": "androidx.constraintlayout:constraintlayout:2.0.1", "androidx.core:core:1.0.1": "androidx.core:core:1.3.1", @@ -718,6 +718,12 @@ }, "version": "4.11.0" }, + "com.github.weisj:jsvg": { + "shasums": { + "jar": "23ac24ccd459447b2074d25f53886a4f42bf896cd6f5e286630e3622e8205f72" + }, + "version": "1.0.0" + }, "com.google.android.apps.common.testing.accessibility.framework:accessibility-test-framework": { "shasums": { "jar": "cdf16ef8f5b8023d003ce3cc1b0d51bda737762e2dab2fedf43d1c4292353f7f" @@ -4216,6 +4222,35 @@ "com.github.bumptech.glide:disklrucache": [ "com.bumptech.glide.disklrucache" ], + "com.github.weisj:jsvg": [ + "com.github.weisj.jsvg", + "com.github.weisj.jsvg.attributes", + "com.github.weisj.jsvg.attributes.filter", + "com.github.weisj.jsvg.attributes.font", + "com.github.weisj.jsvg.attributes.paint", + "com.github.weisj.jsvg.attributes.stroke", + "com.github.weisj.jsvg.attributes.text", + "com.github.weisj.jsvg.geometry", + "com.github.weisj.jsvg.geometry.mesh", + "com.github.weisj.jsvg.geometry.noise", + "com.github.weisj.jsvg.geometry.path", + "com.github.weisj.jsvg.geometry.size", + "com.github.weisj.jsvg.geometry.util", + "com.github.weisj.jsvg.nodes", + "com.github.weisj.jsvg.nodes.animation", + "com.github.weisj.jsvg.nodes.container", + "com.github.weisj.jsvg.nodes.filter", + "com.github.weisj.jsvg.nodes.mesh", + "com.github.weisj.jsvg.nodes.prototype", + "com.github.weisj.jsvg.nodes.prototype.impl", + "com.github.weisj.jsvg.nodes.prototype.spec", + "com.github.weisj.jsvg.nodes.text", + "com.github.weisj.jsvg.parser", + "com.github.weisj.jsvg.parser.css", + "com.github.weisj.jsvg.parser.css.impl", + "com.github.weisj.jsvg.renderer", + "com.github.weisj.jsvg.util" + ], "com.google.android.apps.common.testing.accessibility.framework:accessibility-test-framework": [ "com.google.android.apps.common.testing.accessibility.framework", "com.google.android.apps.common.testing.accessibility.framework.integrations", @@ -6489,6 +6524,7 @@ "com.github.bumptech.glide:gifdecoder:aar", "com.github.bumptech.glide:glide:aar", "com.github.bumptech.glide:mocks:aar", + "com.github.weisj:jsvg", "com.google.android.apps.common.testing.accessibility.framework:accessibility-test-framework", "com.google.android.datatransport:transport-api:aar", "com.google.android.datatransport:transport-backend-cct:aar", @@ -6781,6 +6817,7 @@ "com.github.bumptech.glide:gifdecoder:aar", "com.github.bumptech.glide:glide:aar", "com.github.bumptech.glide:mocks:aar", + "com.github.weisj:jsvg", "com.google.android.apps.common.testing.accessibility.framework:accessibility-test-framework", "com.google.android.datatransport:transport-api:aar", "com.google.android.datatransport:transport-backend-cct:aar", @@ -7073,6 +7110,7 @@ "com.github.bumptech.glide:gifdecoder:aar", "com.github.bumptech.glide:glide:aar", "com.github.bumptech.glide:mocks:aar", + "com.github.weisj:jsvg", "com.google.android.apps.common.testing.accessibility.framework:accessibility-test-framework", "com.google.android.datatransport:transport-api:aar", "com.google.android.datatransport:transport-backend-cct:aar", @@ -7365,6 +7403,7 @@ "com.github.bumptech.glide:gifdecoder:aar", "com.github.bumptech.glide:glide:aar", "com.github.bumptech.glide:mocks:aar", + "com.github.weisj:jsvg", "com.google.android.apps.common.testing.accessibility.framework:accessibility-test-framework", "com.google.android.datatransport:transport-api:aar", "com.google.android.datatransport:transport-backend-cct:aar", diff --git a/third_party/versions.bzl b/third_party/versions.bzl index f9e0864672d..9ad4cef7555 100644 --- a/third_party/versions.bzl +++ b/third_party/versions.bzl @@ -102,6 +102,7 @@ MAVEN_TEST_DEPENDENCY_VERSIONS = { "androidx.work:work-testing": "2.4.0", "com.android.tools.apkparser:apkanalyzer": "30.0.4", "com.github.bumptech.glide:mocks": "4.11.0", + "com.github.weisj:jsvg": "1.0.0", "com.google.protobuf:protobuf-java": "3.17.3", "com.google.protobuf:protobuf-java-util": "3.17.3", "com.google.truth.extensions:truth-liteproto-extension": "1.1.3",