From eae235493a5bea6454be6d1e7d953f50a9b4fe10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Dr=C3=A4ger?= Date: Wed, 29 Nov 2023 21:30:25 +0100 Subject: [PATCH] refactor: bump projects plugin version, migrate to kotlin 1.9 and switch to detekt default coding styles --- .../matchers/ContentTypeHeaderMatchers.kt | 13 +- .../it/skrape/matchers/StatusMatchers.kt | 5 +- build.gradle.kts | 2 +- buildSrc/build.gradle.kts | 10 +- buildSrc/src/main/kotlin/Deps.kt | 2 +- .../buildsrc/convention/detekt.gradle.kts | 9 +- .../buildsrc/convention/kotlin-jvm.gradle.kts | 6 +- detekt.yml | 509 ------------------ .../it/skrape/{Dsl.kt => SkrapeItDsl.kt} | 0 .../test/kotlin/BleedingEdgeReleaseTest.kt | 5 - .../fetcher/{extensions.kt => Extensions.kt} | 5 +- .../main/kotlin/it/skrape/fetcher/Cookie.kt | 10 +- .../main/kotlin/it/skrape/fetcher/Result.kt | 9 +- .../main/kotlin/it/skrape/fetcher/Scraper.kt | 12 +- .../it/skrape/fetcher/request/UrlBuilder.kt | 5 +- .../it/skrape/fetcher/AuthenticationKtTest.kt | 2 +- .../it/skrape/fetcher/KtorAdapterTest.kt | 7 +- .../it/skrape/fetcher/request/FormKtTest.kt | 3 +- .../it/skrape/fetcher/request/JsonKtTest.kt | 5 +- .../it/skrape/fetcher/BrowserFetcherTest.kt | 31 +- .../kotlin/it/skrape/fetcher/extensions.kt | 10 +- gradle.properties | 6 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../src/main/kotlin/it/skrape/core/Parser.kt | 21 +- .../kotlin/it/skrape/selects/CssSelectable.kt | 48 +- .../kotlin/it/skrape/selects/DocElement.kt | 20 +- .../selects/html5/CustomTagSelectors.kt | 2 +- .../html5/DemarcatingEditsSelectors.kt | 4 +- .../selects/html5/EmbeddedContentSelectors.kt | 33 +- .../it/skrape/selects/html5/FormSelectors.kt | 58 +- .../selects/html5/InteractiveSelectors.kt | 10 +- .../skrape/selects/html5/MainRootSelectors.kt | 2 +- .../it/skrape/selects/html5/MediaSelectors.kt | 12 +- .../skrape/selects/html5/MetadataSelectors.kt | 12 +- .../selects/html5/ScriptingSelectors.kt | 6 +- .../selects/html5/SectioningSelectors.kt | 32 +- .../it/skrape/selects/html5/TableSelectors.kt | 20 +- .../selects/html5/TextContentSelectors.kt | 54 +- .../selects/html5/TextSemanticsSelectors.kt | 121 ++--- .../selects/html5/WebComponentsSelectors.kt | 8 +- .../it/skrape/selects/CssSelectorTest.kt | 109 ++-- .../it/skrape/selects/DocElementKtTest.kt | 108 ++-- .../test/kotlin/it/skrape/selects/DocTest.kt | 25 +- .../skrape/selects/ElementExtractorsKtTest.kt | 6 +- .../html5/DemarcatingEditsSelectorsKtTest.kt | 2 +- .../html5/EmbeddedContentSelectorsKtTest.kt | 2 +- .../html5/InteractiveSelectorsKtTest.kt | 2 +- .../selects/html5/MediaSelectorsKtTest.kt | 2 +- .../selects/html5/MetadataSelectorsKtTest.kt | 3 +- .../selects/html5/ScriptingSelectorsKtTest.kt | 2 +- .../html5/SectioningSelectorsKtTest.kt | 3 +- .../selects/html5/TableSelectorsKtTest.kt | 2 +- .../html5/TextContentSelectorsKtTest.kt | 2 +- .../html5/TextSemanticsSelectorsKtTest.kt | 2 +- .../html5/WebComponentsSelectorsKtTest.kt | 2 +- integrationtests/src/test/kotlin/DslTest.kt | 66 +-- .../src/test/kotlin/ExperimentalDslTest.kt | 5 +- .../it.skrape.mockmvc/MockMvcExtension.kt | 2 + test-utils/src/main/kotlin/WireMockStubs.kt | 58 +- .../src/test/kotlin/TestcontainerTest.kt | 10 +- 60 files changed, 524 insertions(+), 1020 deletions(-) rename dsl/src/main/kotlin/it/skrape/{Dsl.kt => SkrapeItDsl.kt} (100%) rename fetcher/async-fetcher/src/main/kotlin/it/skrape/fetcher/{extensions.kt => Extensions.kt} (96%) diff --git a/assertions/src/main/kotlin/it/skrape/matchers/ContentTypeHeaderMatchers.kt b/assertions/src/main/kotlin/it/skrape/matchers/ContentTypeHeaderMatchers.kt index 713aafd1..21d00cf9 100644 --- a/assertions/src/main/kotlin/it/skrape/matchers/ContentTypeHeaderMatchers.kt +++ b/assertions/src/main/kotlin/it/skrape/matchers/ContentTypeHeaderMatchers.kt @@ -1,7 +1,10 @@ +@file:Suppress("MatchingDeclarationName") + package it.skrape.matchers import it.skrape.SkrapeItDsl import it.skrape.fetcher.ContentType +import java.util.* @SkrapeItDsl public enum class ContentTypes(public val value: String) { @@ -28,14 +31,14 @@ public enum class ContentTypes(public val value: String) { } public infix fun ContentType.toBe(expected: ContentTypes): ContentType /* = kotlin.String? */ = - this.apply { generalAssertion(raw() == expected.value, expected) } + this.apply { generalAssertion(raw() == expected.value, expected) } public infix fun ContentType.toBeNot(expected: ContentTypes): ContentType /* = kotlin.String? */ = - this.apply { generalAssertion(raw() != expected.value, expected) } + this.apply { generalAssertion(raw() != expected.value, expected) } public infix fun ContentType.toContain(expected: ContentTypes): ContentType /* = kotlin.String? */ = - this.apply { generalAssertion(raw().contains(expected.value), expected) } + this.apply { generalAssertion(raw().contains(expected.value), expected) } private fun ContentType.raw() = (this as String) - .toLowerCase() - .replace("\\s".toRegex(), "") + .lowercase(Locale.getDefault()) + .replace("\\s".toRegex(), "") diff --git a/assertions/src/main/kotlin/it/skrape/matchers/StatusMatchers.kt b/assertions/src/main/kotlin/it/skrape/matchers/StatusMatchers.kt index 572a40f9..e3f87c3c 100644 --- a/assertions/src/main/kotlin/it/skrape/matchers/StatusMatchers.kt +++ b/assertions/src/main/kotlin/it/skrape/matchers/StatusMatchers.kt @@ -1,4 +1,4 @@ -@file:Suppress("FunctionName", "EnumEntryName") +@file:Suppress("FunctionName", "EnumEntryName", "MatchingDeclarationName") package it.skrape.matchers @@ -78,7 +78,8 @@ public enum class HttpStatus(public val code: Int, public val message: String) { `508_Loop_Detected`(508, "Loop Detected"), `509_Not_Extended`(509, "Not Extended"), `510_Gone`(510, "Gone"), - `511_Network_Authentication_Required`(511, "Network Authentication Required"); + `511_Network_Authentication_Required`(511, "Network Authentication Required"), + ; public fun toStatus(): Result.Status = Result.Status(code, message) } diff --git a/build.gradle.kts b/build.gradle.kts index 0142afe6..8ca48334 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -39,6 +39,6 @@ nexusPublishing { } tasks.wrapper { - gradleVersion = "7.5" + gradleVersion = "8.4" distributionType = Wrapper.DistributionType.ALL } diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 6f8e2d49..5ca71b4a 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -1,15 +1,15 @@ plugins { `kotlin-dsl` - kotlin("jvm") version "1.6.21" + kotlin("jvm") version "1.9.10" } // set the versions of Gradle plugins that the subprojects will use here -val kotlinVersion = "1.6.21" +val kotlinVersion = "1.9.10" -val detektPlugin = "1.19.0" +val detektPlugin = "1.23.4" val gradleNexusPublishPlugin = "1.3.0" -val gradleTestLoggerPlugin = "3.1.0" -val gradleVersionsPlugin = "0.39.0" +val gradleTestLoggerPlugin = "4.0.0" +val gradleVersionsPlugin = "0.50.0" val kotlinDokkaPlugin = "1.9.10" val kotlinxKoverPlugin = "0.7.5" val useLatestVersionsPlugin = "0.2.18" diff --git a/buildSrc/src/main/kotlin/Deps.kt b/buildSrc/src/main/kotlin/Deps.kt index 252bde6d..4fe00617 100644 --- a/buildSrc/src/main/kotlin/Deps.kt +++ b/buildSrc/src/main/kotlin/Deps.kt @@ -1,5 +1,5 @@ object Versions { - const val kotlin = "1.6.21" + const val kotlin = "1.9.10" const val coroutines = "1.6.1" const val ktor = "2.0.3" const val serialization = "1.0.1" diff --git a/buildSrc/src/main/kotlin/buildsrc/convention/detekt.gradle.kts b/buildSrc/src/main/kotlin/buildsrc/convention/detekt.gradle.kts index 6277a3ef..b20428e9 100644 --- a/buildSrc/src/main/kotlin/buildsrc/convention/detekt.gradle.kts +++ b/buildSrc/src/main/kotlin/buildsrc/convention/detekt.gradle.kts @@ -1,18 +1,17 @@ package buildsrc.convention -import io.gitlab.arturbosch.detekt.extensions.DetektExtension.Companion.DEFAULT_SRC_DIR_KOTLIN - plugins { base id("io.gitlab.arturbosch.detekt") } detekt { - toolVersion = "1.19.0" + toolVersion = "1.23.4" autoCorrect = true buildUponDefaultConfig = true - source = files(DEFAULT_SRC_DIR_KOTLIN) - config = files("$rootDir/detekt.yml") + parallel = true + config.setFrom("$rootDir/detekt.yml") + buildUponDefaultConfig = true } tasks.check { diff --git a/buildSrc/src/main/kotlin/buildsrc/convention/kotlin-jvm.gradle.kts b/buildSrc/src/main/kotlin/buildsrc/convention/kotlin-jvm.gradle.kts index 964826f2..84d8f1b5 100644 --- a/buildSrc/src/main/kotlin/buildsrc/convention/kotlin-jvm.gradle.kts +++ b/buildSrc/src/main/kotlin/buildsrc/convention/kotlin-jvm.gradle.kts @@ -22,7 +22,7 @@ dependencies { kotlin { explicitApi() jvmToolchain { - (this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of("8")) + languageVersion.set(JavaLanguageVersion.of("8")) } } @@ -39,8 +39,8 @@ tasks.withType().configureEach { kotlinOptions.apply { jvmTarget = "1.8" freeCompilerArgs += listOf("-Xjsr305=strict") - apiVersion = "1.4" - languageVersion = "1.4" + apiVersion = "1.6" + languageVersion = "1.6" } } diff --git a/detekt.yml b/detekt.yml index 25819d2c..f18d44fd 100644 --- a/detekt.yml +++ b/detekt.yml @@ -9,515 +9,6 @@ build: # style: 1 # comments: 1 -processors: - active: true - exclude: - # - 'FunctionCountProcessor' - # - 'PropertyCountProcessor' - # - 'ClassCountProcessor' - # - 'PackageCountProcessor' - # - 'KtFileCountProcessor' - -console-reports: - active: true - exclude: - # - 'ProjectStatisticsReport' - # - 'ComplexityReport' - # - 'NotificationReport' - # - 'FindingsReport' - # - 'BuildFailureReport' - -comments: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - CommentOverPrivateFunction: - active: false - CommentOverPrivateProperty: - active: false - EndOfSentenceFormat: - active: false - endOfSentenceFormat: ([.?!][ \t\n\r\f<])|([.?!]$) - UndocumentedPublicClass: - active: false - searchInNestedClass: true - searchInInnerClass: true - searchInInnerObject: true - searchInInnerInterface: true - UndocumentedPublicFunction: - active: false - -complexity: - active: true - ComplexCondition: - active: true - threshold: 4 - ComplexInterface: - active: false - threshold: 10 - includeStaticDeclarations: false - ComplexMethod: - active: true - threshold: 10 - ignoreSingleWhenExpression: false - ignoreSimpleWhenEntries: false - LabeledExpression: - active: false - ignoredLabels: "" - LargeClass: - active: true - threshold: 600 - LongMethod: - active: true - threshold: 60 - LongParameterList: - active: true - functionThreshold: 6 - constructorThreshold: 6 - ignoreDefaultParameters: false - MethodOverloading: - active: false - threshold: 6 - NestedBlockDepth: - active: true - threshold: 4 - StringLiteralDuplication: - active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - threshold: 3 - ignoreAnnotation: true - excludeStringsWithLessThan5Characters: true - ignoreStringsRegex: '$^' - TooManyFunctions: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - thresholdInFiles: 33 - thresholdInClasses: 11 - thresholdInInterfaces: 11 - thresholdInObjects: 11 - thresholdInEnums: 11 - ignoreDeprecated: false - ignorePrivate: false - ignoreOverridden: false - -empty-blocks: - active: true - EmptyCatchBlock: - active: true - allowedExceptionNameRegex: "^(_|(ignore|expected).*)" - EmptyClassBlock: - active: true - EmptyDefaultConstructor: - active: true - EmptyDoWhileBlock: - active: true - EmptyElseBlock: - active: true - EmptyFinallyBlock: - active: true - EmptyForBlock: - active: true - EmptyFunctionBlock: - active: true - ignoreOverridden: false - EmptyIfBlock: - active: true - EmptyInitBlock: - active: true - EmptyKtFile: - active: true - EmptySecondaryConstructor: - active: true - EmptyWhenBlock: - active: true - EmptyWhileBlock: - active: true - -exceptions: - active: true - ExceptionRaisedInUnexpectedLocation: - active: false - methodNames: 'toString,hashCode,equals,finalize' - InstanceOfCheckForException: - active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - NotImplementedDeclaration: - active: false - PrintStackTrace: - active: false - RethrowCaughtException: - active: false - ReturnFromFinally: - active: false - SwallowedException: - active: false - ignoredExceptionTypes: 'InterruptedException,NumberFormatException,ParseException,MalformedURLException' - ThrowingExceptionFromFinally: - active: false - ThrowingExceptionInMain: - active: false - ThrowingExceptionsWithoutMessageOrCause: - active: false - exceptions: 'IllegalArgumentException,IllegalStateException,IOException' - ThrowingNewInstanceOfSameException: - active: false - TooGenericExceptionCaught: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - exceptionNames: - - ArrayIndexOutOfBoundsException - - Error - - Exception - - IllegalMonitorStateException - - NullPointerException - - IndexOutOfBoundsException - - RuntimeException - - Throwable - allowedExceptionNameRegex: "^(_|(ignore|expected).*)" - TooGenericExceptionThrown: - active: true - exceptionNames: - - Error - - Exception - - Throwable - - RuntimeException - -formatting: - active: true - android: false - autoCorrect: true - ChainWrapping: - active: true - autoCorrect: true - CommentSpacing: - active: true - autoCorrect: true - Filename: - active: true - FinalNewline: - active: true - autoCorrect: true - ImportOrdering: - active: false - Indentation: - active: true - autoCorrect: true - indentSize: 4 - continuationIndentSize: 4 - MaximumLineLength: - active: true - maxLineLength: 120 - ModifierOrdering: - active: true - autoCorrect: true - NoBlankLineBeforeRbrace: - active: true - autoCorrect: true - NoConsecutiveBlankLines: - active: true - autoCorrect: true - NoEmptyClassBody: - active: true - autoCorrect: true - NoLineBreakAfterElse: - active: true - autoCorrect: true - NoLineBreakBeforeAssignment: - active: true - autoCorrect: true - NoMultipleSpaces: - active: true - autoCorrect: true - NoSemicolons: - active: true - autoCorrect: true - NoTrailingSpaces: - active: true - autoCorrect: true - NoUnitReturn: - active: true - autoCorrect: true - NoUnusedImports: - active: true - autoCorrect: true - NoWildcardImports: - active: true - autoCorrect: true - PackageName: - active: true - autoCorrect: true - ParameterListWrapping: - active: true - autoCorrect: true - indentSize: 4 - SpacingAroundColon: - active: true - autoCorrect: true - SpacingAroundComma: - active: true - autoCorrect: true - SpacingAroundCurly: - active: true - autoCorrect: true - SpacingAroundKeyword: - active: true - autoCorrect: true - SpacingAroundOperators: - active: true - autoCorrect: true - SpacingAroundParens: - active: true - autoCorrect: true - SpacingAroundRangeOperator: - active: true - autoCorrect: true - StringTemplate: - active: true - autoCorrect: true - -naming: - active: true - ClassNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - classPattern: '[A-Z$][a-zA-Z0-9$]*' - ConstructorParameterNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - parameterPattern: '[a-z][A-Za-z0-9]*' - privateParameterPattern: '[a-z][A-Za-z0-9]*' - excludeClassPattern: '$^' - EnumNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - enumEntryPattern: '^[A-Z][_a-zA-Z0-9]*' - ForbiddenClassName: - active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - forbiddenName: '' - FunctionMaxLength: - active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - maximumFunctionNameLength: 30 - FunctionMinLength: - active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - minimumFunctionNameLength: 3 - FunctionNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - functionPattern: '^([a-z$][a-zA-Z$0-9]*)|(`.*`)$' - excludeClassPattern: '$^' - ignoreOverridden: true - FunctionParameterNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - parameterPattern: '[a-z][A-Za-z0-9]*' - excludeClassPattern: '$^' - ignoreOverridden: true - MatchingDeclarationName: - active: false - MemberNameEqualsClassName: - active: true - ignoreOverridden: true - ObjectPropertyNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - constantPattern: '[A-Za-z][_A-Za-z0-9]*' - propertyPattern: '[A-Za-z][_A-Za-z0-9]*' - privatePropertyPattern: '(_)?[A-Za-z][_A-Za-z0-9]*' - PackageNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - packagePattern: '^[a-z]+(\.[a-z][A-Za-z0-9]*)*$' - TopLevelPropertyNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - constantPattern: '[A-Z][_A-Z0-9]*' - propertyPattern: '[A-Za-z][_A-Za-z0-9]*' - privatePropertyPattern: '(_)?[A-Za-z][A-Za-z0-9]*' - VariableMaxLength: - active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - maximumVariableNameLength: 64 - VariableMinLength: - active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - minimumVariableNameLength: 1 - VariableNaming: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - variablePattern: '[a-z][A-Za-z0-9]*' - privateVariablePattern: '(_)?[a-z][A-Za-z0-9]*' - excludeClassPattern: '$^' - ignoreOverridden: true - -performance: - active: true - ArrayPrimitive: - active: false - ForEachOnRange: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - SpreadOperator: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - UnnecessaryTemporaryInstantiation: - active: true - -potential-bugs: - active: true - DuplicateCaseInWhenExpression: - active: true - EqualsAlwaysReturnsTrueOrFalse: - active: true - EqualsWithHashCodeExist: - active: true - ExplicitGarbageCollectionCall: - active: true - InvalidRange: - active: false - IteratorHasNextCallsNextMethod: - active: false - IteratorNotThrowingNoSuchElementException: - active: false - LateinitUsage: - active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - excludeAnnotatedProperties: "" - ignoreOnClassesPattern: "" - UnconditionalJumpStatementInLoop: - active: false - UnreachableCode: - active: true - UnsafeCallOnNullableType: - active: false - UnsafeCast: - active: false - UselessPostfixExpression: - active: false - WrongEqualsTypeParameter: - active: false - style: - active: true - CollapsibleIfStatements: - active: false - DataClassContainsFunctions: - active: false - conversionFunctionPrefix: 'to' - EqualsNullCall: - active: false - EqualsOnSignatureLine: - active: false - ExplicitItLambdaParameter: - active: false - ExpressionBodySyntax: - active: false - includeLineWrapping: false - ForbiddenComment: - active: false - values: 'TODO:,FIXME:,STOPSHIP:' - ForbiddenImport: - active: false - imports: '' - ForbiddenVoid: - active: false - FunctionOnlyReturningConstant: - active: false - ignoreOverridableFunction: true - excludedFunctions: 'describeContents' - LoopWithTooManyJumpStatements: - active: false - maxJumpCount: 1 - MagicNumber: - active: true - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - ignoreNumbers: '-1,0,1,2' - ignoreHashCodeFunction: true - ignorePropertyDeclaration: false - ignoreConstantDeclaration: true - ignoreCompanionObjectPropertyDeclaration: true - ignoreAnnotation: false - ignoreNamedArgument: true - ignoreEnums: false - MandatoryBracesIfStatements: - active: false - MaxLineLength: - active: true - maxLineLength: 120 - excludePackageStatements: true - excludeImportStatements: true - excludeCommentStatements: true - MayBeConst: - active: false - ModifierOrder: - active: true - NestedClassesVisibility: - active: false - NewLineAtEndOfFile: - active: true - NoTabs: - active: false - OptionalAbstractKeyword: - active: true - OptionalUnit: - active: false - OptionalWhenBraces: - active: false - PreferToOverPairSyntax: - active: false - ProtectedMemberInFinalClass: - active: false - RedundantVisibilityModifierRule: - active: false - ReturnCount: - active: true - max: 2 - excludedFunctions: "equals" - excludeLabeled: false - excludeReturnFromLambda: true - SafeCast: - active: true - SerialVersionUIDInSerializableClass: - active: false - SpacingBetweenPackageAndImports: - active: false - ThrowsCount: - active: true - max: 2 - TrailingWhitespace: - active: true - UnderscoresInNumericLiterals: - active: true - acceptableDecimalLength: 5 - UnnecessaryAbstractClass: - active: false - excludeAnnotatedClasses: "dagger.Module" - UnnecessaryApply: - active: false - UnnecessaryInheritance: - active: false - UnnecessaryLet: - active: false - UnnecessaryParentheses: - active: false - UntilInsteadOfRangeTo: - active: false - UnusedImports: - active: false - UnusedPrivateClass: - active: false - UnusedPrivateMember: - active: false - allowedNames: "(_|ignored|expected|serialVersionUID)" - UseDataClass: - active: true - excludeAnnotatedClasses: "" - UtilityClassWithPublicConstructor: - active: false - VarCouldBeVal: - active: true WildcardImport: active: false - excludes: "**/test/**,**/androidTest/**,**/*.Test.kt,**/*.Spec.kt,**/*.Spek.kt" - excludeImports: 'it.skrape.core.*' \ No newline at end of file diff --git a/dsl/src/main/kotlin/it/skrape/Dsl.kt b/dsl/src/main/kotlin/it/skrape/SkrapeItDsl.kt similarity index 100% rename from dsl/src/main/kotlin/it/skrape/Dsl.kt rename to dsl/src/main/kotlin/it/skrape/SkrapeItDsl.kt diff --git a/examples/use-pre-release-version/src/test/kotlin/BleedingEdgeReleaseTest.kt b/examples/use-pre-release-version/src/test/kotlin/BleedingEdgeReleaseTest.kt index 1e79ec70..44c06bb9 100644 --- a/examples/use-pre-release-version/src/test/kotlin/BleedingEdgeReleaseTest.kt +++ b/examples/use-pre-release-version/src/test/kotlin/BleedingEdgeReleaseTest.kt @@ -1,4 +1,3 @@ - import it.skrape.core.htmlDocument import it.skrape.fetcher.* import it.skrape.selects.html5.span @@ -8,9 +7,7 @@ import strikt.api.expectThat import strikt.assertions.isEqualTo import strikt.assertions.isGreaterThanOrEqualTo - class BleedingEdgeReleaseTest { - @Test internal fun `can use latest aggregator version from maven central and scrape page without JS`() { val projectsGithubStars = skrape(HttpFetcher) { @@ -35,7 +32,6 @@ class BleedingEdgeReleaseTest { } } } - expectThat(projectsGithubStars).isGreaterThanOrEqualTo("449") } @@ -81,7 +77,6 @@ class BleedingEdgeReleaseTest { } } } - expectThat(projectsGithubStars).isGreaterThanOrEqualTo("239") } } diff --git a/fetcher/async-fetcher/src/main/kotlin/it/skrape/fetcher/extensions.kt b/fetcher/async-fetcher/src/main/kotlin/it/skrape/fetcher/Extensions.kt similarity index 96% rename from fetcher/async-fetcher/src/main/kotlin/it/skrape/fetcher/extensions.kt rename to fetcher/async-fetcher/src/main/kotlin/it/skrape/fetcher/Extensions.kt index 8e96e205..0f0d0713 100644 --- a/fetcher/async-fetcher/src/main/kotlin/it/skrape/fetcher/extensions.kt +++ b/fetcher/async-fetcher/src/main/kotlin/it/skrape/fetcher/Extensions.kt @@ -1,3 +1,5 @@ +@file:Suppress("TooManyFunctions") + package it.skrape.fetcher import io.ktor.client.* @@ -13,6 +15,7 @@ import org.apache.http.conn.ssl.NoopHostnameVerifier import org.apache.http.conn.ssl.TrustSelfSignedStrategy import org.apache.http.ssl.SSLContextBuilder import java.net.Proxy +import java.util.* internal fun Request.toHttpRequest(): HttpRequestBuilder { val request = this @@ -80,7 +83,7 @@ internal fun String?.toExpires(): Expires { } } -internal fun String?.toSameSite(): SameSite = when (this?.toLowerCase()) { +internal fun String?.toSameSite(): SameSite = when (this?.lowercase(Locale.getDefault())) { "strict" -> SameSite.STRICT "lax" -> SameSite.LAX "none" -> SameSite.NONE diff --git a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Cookie.kt b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Cookie.kt index eb808f99..120b4707 100644 --- a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Cookie.kt +++ b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Cookie.kt @@ -1,5 +1,9 @@ +@file:Suppress("MaxLineLength") + package it.skrape.fetcher +import java.util.* + /** * Object representing a web cookie * @param name The name of the cookie @@ -54,8 +58,8 @@ public fun String.toCookie(origin: String): Cookie { else -> Domain(domainUrl, true) } val sameSite = attributes.getAttribute("samesite").toSameSite() - val secure = attributes.any { it.toLowerCase() == "secure" } - val httpOnly = attributes.any { it.toLowerCase() == "httponly" } + val secure = attributes.any { it.lowercase(Locale.getDefault()) == "secure" } + val httpOnly = attributes.any { it.lowercase(Locale.getDefault()) == "httponly" } return Cookie(name, value, expires, maxAge, domain, path, sameSite, secure, httpOnly) } @@ -63,7 +67,7 @@ private fun List.getAttribute(attributeName: String) = this.find { it.startsWith("${attributeName}=", ignoreCase = true) }?.takeLastWhile { it != '=' } private fun String?.toSameSite(): SameSite { - return when (this?.toLowerCase()) { + return when (this?.lowercase(Locale.getDefault())) { "strict" -> SameSite.STRICT "lax" -> SameSite.LAX "none" -> SameSite.NONE diff --git a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Result.kt b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Result.kt index 0d5e17c8..b79c4b2d 100644 --- a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Result.kt +++ b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Result.kt @@ -18,8 +18,9 @@ public class Result( public val contentType: ContentType, public val headers: Map, public val baseUri: String = "", - public val cookies: List + public val cookies: List, ) { + @Suppress("MaxLineLength") /** * Will return a certain response headers value * @see Explanation about response headers. @@ -44,12 +45,12 @@ public class Result( } public fun cookies(init: List.() -> Unit): List = - cookies.apply(init) + cookies.apply(init) @SkrapeItDsl public data class Status( - val code: Int, - val message: String + val code: Int, + val message: String, ) } diff --git a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt index d181d30e..6d46b795 100644 --- a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt +++ b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/Scraper.kt @@ -20,10 +20,11 @@ public class Scraper(public val fetcher: NonBlockingFetcher, public val pr /** * Decorator that Implements a non-blocking fetcher with a given blocking Fetcher. * Hint: The thereby created NonBlockingFetcher will still behave like a BlockingFetcher. - * It will only be converted to reduce library internal code duplication by handling everything as a coroutine internally. + * It will only be converted to reduce library internal code duplication + * by handling everything as a coroutine internally. */ private class FetcherConverter( - val blockingFetcher: BlockingFetcher + val blockingFetcher: BlockingFetcher, ) : NonBlockingFetcher { override suspend fun fetch(request: T): Result { @@ -87,7 +88,7 @@ public suspend fun Scraper<*>.response(result: suspend Result.() -> T): T = scrape().result() /** - * Blocking implementation of 'extract' as convenience function to call it outside of an coroutine. + * Blocking implementation of 'extract' as convenience function to call it outside coroutine. * @return T */ @SkrapeItDsl @@ -95,7 +96,8 @@ public fun Scraper<*>.extractBlocking(result: suspend Result.() -> T): T = runBlocking { response(result) } /** - * Execute http call with a given Fetcher implementation and invoke the fetching result as this and any given generic as it. + * Execute http call with a given Fetcher implementation + * and invoke the fetching result as this and any given generic as it. * Attention: extract to class is only supported for classes where all parameters have default values * @return T */ @@ -106,7 +108,7 @@ public suspend inline fun Scraper<*>.extractIt(crossinline res } /** - * Blocking implementation of 'extractIt' as convenience function to call it outside of an coroutine. + * Blocking implementation of 'extractIt' as convenience function to call it outside coroutine. * @return T */ @SkrapeItDsl diff --git a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/request/UrlBuilder.kt b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/request/UrlBuilder.kt index 43deff14..fc480b33 100644 --- a/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/request/UrlBuilder.kt +++ b/fetcher/base-fetcher/src/main/kotlin/it/skrape/fetcher/request/UrlBuilder.kt @@ -42,7 +42,8 @@ public data class UrlBuilder( HTTP("http"), HTTPS("https"), FTP("ftp"), - FILE("file"); + FILE("file"), + ; internal fun findOrDefault(value: String): Protocol = values().find { it.value == value } ?: HTTP } @@ -60,6 +61,8 @@ public data class UrlBuilder( @SkrapeItDsl public class QueryParam { private val params: MutableMap = mutableMapOf() + + @Suppress("ForbiddenComment") public infix fun String.to(value: Any?) { // TODO: do not ignore entries with value null params.computeIfAbsent(this) { diff --git a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/AuthenticationKtTest.kt b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/AuthenticationKtTest.kt index 54d21341..7df09233 100644 --- a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/AuthenticationKtTest.kt +++ b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/AuthenticationKtTest.kt @@ -15,4 +15,4 @@ class AuthenticationKtTest { expectThat(authenticationHeader).isEqualTo("Basic dXNlcjpzZWNyZXQ=") } -} \ No newline at end of file +} diff --git a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt index 47d8d4c5..78b56c0e 100644 --- a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt +++ b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/KtorAdapterTest.kt @@ -21,6 +21,7 @@ private val wiremock = Testcontainer.wiremock @DisabledOnOs(OS.WINDOWS) class KtorAdapterTest { + @Suppress("UNCHECKED_CAST") class KtorBlockingFetcher(val ktorClient: HttpClient) : BlockingFetcher { override fun fetch(request: HttpRequestBuilder): Result = runBlocking { with(ktorClient.request(request)) { @@ -30,7 +31,7 @@ class KtorAdapterTest { contentType = contentType()?.toString(), headers = headers.toMap().mapValues { it.value.firstOrNull().orEmpty() }, baseUri = request.url.toString(), - cookies = listOf(setCookie()) as List + cookies = listOf(setCookie()) as List, ) } } @@ -39,13 +40,13 @@ class KtorAdapterTest { get() = HttpRequestBuilder() } - val KTOR_CLIENT = HttpClient(Apache) + private val ktorClient = HttpClient(Apache) @Test fun `dsl can skrape by url`() { wiremock.setupStub(path = "/example") - val result = skrape(KtorBlockingFetcher(KTOR_CLIENT)) { + val result = skrape(KtorBlockingFetcher(ktorClient)) { request { url("${wiremock.httpUrl}/example") } diff --git a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/request/FormKtTest.kt b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/request/FormKtTest.kt index f01c760c..0e5483fc 100644 --- a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/request/FormKtTest.kt +++ b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/request/FormKtTest.kt @@ -59,5 +59,4 @@ internal class FormKtTest { } expectThat("$form").isEqualTo("a=null") } - -} \ No newline at end of file +} diff --git a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/request/JsonKtTest.kt b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/request/JsonKtTest.kt index 8a89138d..a9c33d45 100644 --- a/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/request/JsonKtTest.kt +++ b/fetcher/base-fetcher/src/test/kotlin/it/skrape/fetcher/request/JsonKtTest.kt @@ -60,7 +60,6 @@ internal class JsonKtTest { expectThat("$json").isEqualTo("""{"a":[1,2.0,null,3],"b":2}""") } - @Test fun `can create json with list of String`() { val json = Json().json { @@ -101,7 +100,6 @@ internal class JsonKtTest { expectThat("$json").isEqualTo("""{"a":null}""") } - @Test fun `can create json with inner list of any`() { val json = Json().json { @@ -138,5 +136,4 @@ internal class JsonKtTest { } expectThat("$json").isEqualTo("""{"a":null}""") } - -} \ No newline at end of file +} diff --git a/fetcher/browser-fetcher/src/test/kotlin/it/skrape/fetcher/BrowserFetcherTest.kt b/fetcher/browser-fetcher/src/test/kotlin/it/skrape/fetcher/BrowserFetcherTest.kt index bb907b4c..19447c28 100644 --- a/fetcher/browser-fetcher/src/test/kotlin/it/skrape/fetcher/BrowserFetcherTest.kt +++ b/fetcher/browser-fetcher/src/test/kotlin/it/skrape/fetcher/BrowserFetcherTest.kt @@ -31,13 +31,15 @@ class BrowserFetcherTest { fun `can render from string`() { val renderedHtml = BrowserFetcher.render(getMarkupFromFile("es6.html")) - expectThat(renderedHtml).contains(""" + expectThat(renderedHtml).contains( + """ |

| | dynamically added | |

- """.trimMargin().replace("\n", "\r\n")) + """.trimMargin().replace("\n", "\r\n"), + ) } @Test @@ -58,7 +60,7 @@ class BrowserFetcherTest { val request = baseRequest.copy( url = "${wiremock.httpsUrl}/example", - sslRelaxed = true + sslRelaxed = true, ) val fetched = BrowserFetcher.fetch(request) @@ -103,7 +105,7 @@ class BrowserFetcherTest { val request = baseRequest.copy( url = "${wiremock.httpsUrl}/cookies", - sslRelaxed = true + sslRelaxed = true, ) val fetched = BrowserFetcher.fetch(request) @@ -119,16 +121,16 @@ class BrowserFetcherTest { "/cookies", SameSite.STRICT, true, - httpOnly = true + httpOnly = true, ), Cookie( "expireTest", "value", Expires.Date("Wed, 21 Oct 2015 07:28:00 GMT"), 2592000, - Domain("localhost", false) - ) - ) + Domain("localhost", false), + ), + ), ) } @@ -145,10 +147,9 @@ class BrowserFetcherTest { @Test fun `can POST body`() { - val request = Request( url = "$httpBin/post", - method = Method.POST + method = Method.POST, ).apply { body { json { @@ -175,7 +176,7 @@ class BrowserFetcherTest { wiremock.setupStub(fileName = "js.html") val request = baseRequest.copy( url = wiremock.httpsUrl, - sslRelaxed = true + sslRelaxed = true, ) val fetched = BrowserFetcher.fetch(request) @@ -202,10 +203,9 @@ class BrowserFetcherTest { @Test fun `will extract headers to map`() { - val htmlUnitHeaders = listOf( NameValuePair("first-name", "first-value"), - NameValuePair("second-name", "second-value") + NameValuePair("second-name", "second-value"), ) val result = htmlUnitHeaders.toMap() expectThat(result).isEqualTo(mapOf("first-name" to "first-value", "second-name" to "second-value")) @@ -213,14 +213,13 @@ class BrowserFetcherTest { @Test fun `will create raw cookie syntax representation of map`() { - val cookiesAsMap = mapOf( "first-name" to "first-value", - "second-name" to "second-value" + "second-name" to "second-value", ) val result = cookiesAsMap.asRawCookieSyntax() expectThat(result).isEqualTo("first-name=first-value;second-name=second-value;") } private fun getMarkupFromFile(file: String) = javaClass.getResource("/__files/$file").readText() -} \ No newline at end of file +} diff --git a/fetcher/http-fetcher/src/main/kotlin/it/skrape/fetcher/extensions.kt b/fetcher/http-fetcher/src/main/kotlin/it/skrape/fetcher/extensions.kt index 7fde1096..3de0a54c 100644 --- a/fetcher/http-fetcher/src/main/kotlin/it/skrape/fetcher/extensions.kt +++ b/fetcher/http-fetcher/src/main/kotlin/it/skrape/fetcher/extensions.kt @@ -1,3 +1,5 @@ +@file:Suppress("TooManyFunctions") + package it.skrape.fetcher import io.ktor.client.* @@ -14,7 +16,7 @@ import org.apache.http.conn.ssl.NoopHostnameVerifier import org.apache.http.conn.ssl.TrustSelfSignedStrategy import org.apache.http.ssl.SSLContextBuilder import java.net.Proxy - +import java.util.* internal fun Request.toHttpRequest(): HttpRequestBuilder { val request = this @@ -82,7 +84,7 @@ private fun String?.toExpires(): Expires { } } -private fun String?.toSameSite(): SameSite = when (this?.toLowerCase()) { +private fun String?.toSameSite(): SameSite = when (this?.lowercase(Locale.getDefault())) { "strict" -> SameSite.STRICT "lax" -> SameSite.LAX "none" -> SameSite.NONE @@ -96,7 +98,7 @@ internal fun HttpClientConfig.trustSelfSignedClient() { SSLContextBuilder .create() .loadTrustMaterial(TrustSelfSignedStrategy()) - .build() + .build(), ) setSSLHostnameVerifier(NoopHostnameVerifier()) } @@ -111,7 +113,7 @@ internal fun HttpResponse.toResult(): Result = headers = headers.flattenEntries() .associateBy({ item -> item.first }, { item -> headers[item.first]!! }), cookies = setCookie().map { cookie -> cookie.toDomainCookie(request.url.toString().urlOrigin) }, - baseUri = request.url.toString() + baseUri = request.url.toString(), ) private fun HttpResponse.toStatus() = Result.Status(this.status.value, this.status.description) diff --git a/gradle.properties b/gradle.properties index 7325aea4..2c6e7b7a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ kotlin.code.style=official -kotlin_version=1.6.21 +kotlin_version=1.9.10 org.gradle.jvmargs=-Dfile.encoding=UTF-8 release_version=1.3.0-alpha.1 @@ -9,10 +9,10 @@ kotlin.mpp.stability.nowarn=true # Enable build cache https://docs.gradle.org/current/userguide/build_cache.html org.gradle.caching=true # Enable configuration cache https://docs.gradle.org/current/userguide/configuration_cache.html -org.gradle.unsafe.configuration-cache=true +org.gradle.unsafe.configuration-cache=false org.gradle.unsafe.configuration-cache-problems=warn # Enable parallel subproject execution https://docs.gradle.org/current/userguide/performance.html#parallel_execution -#org.gradle.parallel=true +org.gradle.parallel=true # https://github.com/gradle/gradle/issues/20416 org.gradle.kotlin.dsl.precompiled.accessors.strict=true diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2ec77e51..fce403e4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/html-parser/src/main/kotlin/it/skrape/core/Parser.kt b/html-parser/src/main/kotlin/it/skrape/core/Parser.kt index 883dd1f5..47ff2962 100644 --- a/html-parser/src/main/kotlin/it/skrape/core/Parser.kt +++ b/html-parser/src/main/kotlin/it/skrape/core/Parser.kt @@ -16,16 +16,19 @@ internal class Parser( var html: String, val charset: Charset, val jsExecution: Boolean, - val baseUri: String + val baseUri: String, ) { fun parse(): Doc { return if (jsExecution) { checkBrowserFetcherIsPresent() jSoupParser(BrowserFetcher.render(html), baseUri).toDocWrapper() - } else jSoupParser(html, baseUri).toDocWrapper() + } else { + jSoupParser(html, baseUri).toDocWrapper() + } } + @Suppress("SwallowedException") private fun checkBrowserFetcherIsPresent() { try { Class.forName("it.skrape.fetcher.BrowserFetcherKt") @@ -35,7 +38,7 @@ internal class Parser( ⚠️ You need to add browser-fetcher dependency to execute Javascript. 💡 Please add 'it.skrape:skrapeit-browser-fetcher' dependency to your project. 🧐 Find overview of latest releases: https://search.maven.org/artifact/it.skrape/skrapeit-browser-fetcher - """.trimIndent() + """.trimIndent(), ) } } @@ -58,7 +61,7 @@ public fun htmlDocument( charset: Charset = Charsets.UTF_8, jsExecution: Boolean = false, baseUri: String = "", - init: Doc.() -> T + init: Doc.() -> T, ): T = htmlDocument(html, charset, jsExecution, baseUri).init() /** @@ -73,7 +76,7 @@ public fun htmlDocument( charset: Charset = Charsets.UTF_8, jsExecution: Boolean = false, baseUri: String = "", - init: Doc.() -> T + init: Doc.() -> T, ): T = htmlDocument(file, charset, jsExecution, baseUri).init() /** @@ -88,7 +91,7 @@ public fun htmlDocument( charset: Charset = Charsets.UTF_8, jsExecution: Boolean = false, baseUri: String = "", - init: Doc.() -> T + init: Doc.() -> T, ): T = htmlDocument(bytes, charset, jsExecution, baseUri).init() @SkrapeItDsl @@ -96,21 +99,21 @@ public fun htmlDocument( @Language("HTML") html: String, charset: Charset = Charsets.UTF_8, jsExecution: Boolean = false, - baseUri: String = "" + baseUri: String = "", ): Doc = Parser(html, charset, jsExecution, baseUri).parse() public fun htmlDocument( file: File, charset: Charset = Charsets.UTF_8, jsExecution: Boolean = false, - baseUri: String = "" + baseUri: String = "", ): Doc = htmlDocument(file.readText(charset), charset, jsExecution, baseUri) public fun htmlDocument( bytes: InputStream, charset: Charset = Charsets.UTF_8, jsExecution: Boolean = false, - baseUri: String = "" + baseUri: String = "", ): Doc = htmlDocument(bytes.bufferedReader().use(BufferedReader::readText), charset, jsExecution, baseUri) public val Result.document: Doc diff --git a/html-parser/src/main/kotlin/it/skrape/selects/CssSelectable.kt b/html-parser/src/main/kotlin/it/skrape/selects/CssSelectable.kt index bfa1d048..3b0df965 100644 --- a/html-parser/src/main/kotlin/it/skrape/selects/CssSelectable.kt +++ b/html-parser/src/main/kotlin/it/skrape/selects/CssSelectable.kt @@ -12,7 +12,7 @@ public abstract class CssSelectable { internal abstract fun applySelector(rawCssSelector: String): List public fun selection(cssSelector: String, init: CssSelector.() -> T): T = - CssSelector(rawCssSelector = cssSelector, doc = this).init() + CssSelector(rawCssSelector = cssSelector, doc = this).init() /** * Will create a CssSelector scope to calculate a css selector @@ -20,12 +20,13 @@ public abstract class CssSelectable { * @return T */ public operator fun String.invoke(init: CssSelector.() -> T): T = - this@CssSelectable.selection(this, init) + this@CssSelectable.selection(this, init) public open fun makeDefault(cssSelector: String, relaxed: Boolean = false): DocElement { return DocElement(Element("${UUID.randomUUID()}"), relaxed) } + @Suppress("MaxLineLength") /** * Will pick all occurrences of elements that are matching the CSS-Selector * @see Overview of CSS-Selectors for further information. @@ -33,20 +34,21 @@ public abstract class CssSelectable { * @return T */ public infix fun findAll(cssSelector: String): List = - this.applySelector(cssSelector) + this.applySelector(cssSelector) public fun findByIndex(index: Int, cssSelector: String = ""): DocElement = - findAll(cssSelector).getOrElse(index) { makeDefault(cssSelector) } + findAll(cssSelector).getOrElse(index) { makeDefault(cssSelector) } public operator fun Int.invoke(cssSelector: String = ""): DocElement = - findByIndex(this, cssSelector) + findByIndex(this, cssSelector) public fun findBySelectorMatching(regex: Regex): List = - this@CssSelectable.findAll("*").filter { it.ownCssSelector.matches(regex) } + this@CssSelectable.findAll("*").filter { it.ownCssSelector.matches(regex) } public operator fun Regex.invoke(): List = - findBySelectorMatching(this) + findBySelectorMatching(this) + @Suppress("MaxLineLength") /** * Will pick the first occurrence of an element that * is matching the CSS-Selector from a parsed document and invoke it to a lambda function. @@ -55,20 +57,21 @@ public abstract class CssSelectable { * @return T */ public infix fun findFirst(cssSelector: String): DocElement = - findByIndex(0, cssSelector) + findByIndex(0, cssSelector) public fun findSecond(cssSelector: String = ""): DocElement = - findByIndex(1, cssSelector) + findByIndex(1, cssSelector) public fun findThird(cssSelector: String = ""): DocElement = - findByIndex(2, cssSelector) + findByIndex(2, cssSelector) public fun findLast(cssSelector: String = ""): DocElement = - findAll(cssSelector).last() + findAll(cssSelector).last() public fun findSecondLast(cssSelector: String = ""): DocElement = - findAll(cssSelector).let { it.getOrElse(it.lastIndex -1) { makeDefault(cssSelector) } } + findAll(cssSelector).let { it.getOrElse(it.lastIndex - 1) { makeDefault(cssSelector) } } + @Suppress("MaxLineLength") /** * Will pick all occurrences of elements that are matching the CSS-Selector * @see Overview of CSS-Selectors for further information. @@ -76,20 +79,21 @@ public abstract class CssSelectable { * @return T */ public fun findAll(cssSelector: String = "", init: List.() -> T): T = - findAll(cssSelector).init() + findAll(cssSelector).init() public fun findByIndex(index: Int, cssSelector: String = "", init: DocElement.() -> T): T = - findByIndex(index, cssSelector).init() + findByIndex(index, cssSelector).init() public operator fun Int.invoke(cssSelector: String = "", init: DocElement.() -> T): T = - this(cssSelector).init() + this(cssSelector).init() public fun findBySelectorMatching(regex: Regex, init: List.() -> T): T = - findBySelectorMatching(regex).init() + findBySelectorMatching(regex).init() public operator fun Regex.invoke(init: List.() -> T): T = - this().init() + this().init() + @Suppress("MaxLineLength") /** * Will pick the first occurrence of an element that * is matching the CSS-Selector from a parsed document and invoke it to a lambda function. @@ -98,17 +102,17 @@ public abstract class CssSelectable { * @return T */ public fun findFirst(cssSelector: String = "", init: DocElement.() -> T): T = - findFirst(cssSelector).init() + findFirst(cssSelector).init() public fun findSecond(cssSelector: String = "", init: DocElement.() -> T): T = - findSecond(cssSelector).init() + findSecond(cssSelector).init() public fun findThird(cssSelector: String = "", init: DocElement.() -> T): T = - findThird(cssSelector).init() + findThird(cssSelector).init() public fun findLast(cssSelector: String = "", init: DocElement.() -> T): T = - findLast(cssSelector).init() + findLast(cssSelector).init() public fun findSecondLast(cssSelector: String = "", init: DocElement.() -> T): T = - findSecondLast(cssSelector).init() + findSecondLast(cssSelector).init() } diff --git a/html-parser/src/main/kotlin/it/skrape/selects/DocElement.kt b/html-parser/src/main/kotlin/it/skrape/selects/DocElement.kt index 1537b33d..f8a62094 100644 --- a/html-parser/src/main/kotlin/it/skrape/selects/DocElement.kt +++ b/html-parser/src/main/kotlin/it/skrape/selects/DocElement.kt @@ -2,12 +2,13 @@ package it.skrape.selects import it.skrape.SkrapeItDsl import org.jsoup.nodes.Element +import java.util.* @Suppress("TooManyFunctions") @SkrapeItDsl public class DocElement internal constructor( override val element: Element, - override val relaxed: Boolean + override val relaxed: Boolean, ) : DomTreeElement() { public constructor(element: Element) : this(element, false) @@ -39,7 +40,7 @@ public class DocElement internal constructor( public val wholeText: String by lazy { element.wholeText() } /** - * Get all of the element's attributes. + * Get all the element's attributes. * @return Map> of attribute key value pairs */ public val attributes: Map by lazy { element.attributes().associate { it.key to it.value } } @@ -71,25 +72,26 @@ public class DocElement internal constructor( public val dataAttributes: Map by lazy { attributes.filter { it.key.startsWith("data-") } } /** - * Gets the literal value of this element's "class" attribute, which may include multiple class names, space separated. + * Gets the literal value of this element's "class" attribute, + * which may include multiple class names, space separated. * (E.g. on
returns, "header gray") * @return String of the literal class attribute, or empty string if no class attribute set. */ public val className: String by lazy { attribute("class").trim() } /** - * Get all of the element's class names. E.g. on element
, + * Get all the element's class names. E.g. on element
, * returns a set of two elements "header", "gray". * @return Set distinct classnames, empty if no class attribute */ public val classNames: Set by lazy { className.split(" ").filter { it.isNotBlank() }.toSet() } /** - * Case insensitive check if this element has a class. + * Case-insensitive check if this element has a class. * @return Boolean */ public fun hasClass(className: String): Boolean = - classNames.map { it.toLowerCase() }.contains(className.toLowerCase()) + classNames.map { it.lowercase(Locale.getDefault()) }.contains(className.lowercase(Locale.getDefault())) /** * Gets the literal value of this element's "id" attribute. @@ -114,6 +116,7 @@ public class DocElement internal constructor( * Get this element's parent element. * @return DocElement */ + @Suppress("SwallowedException") public val parent: DocElement by lazy { try { parents.first() @@ -147,7 +150,8 @@ public class DocElement internal constructor( public val isPresent: Boolean by lazy { allElements.isNotEmpty() } /** - * Check if the element is NOT present thereby it will return true if the given node can not be found otherwise false. + * Check if the element is NOT present thereby it will + * return true if the given node can not be found otherwise false. * @return Boolean */ public val isNotPresent: Boolean by lazy { !isPresent } @@ -182,7 +186,7 @@ public class DocElement internal constructor( .filterNot { it.key == "class" } .filterNot { it.value.isBlank() } .toList(), - withAttributeKeys = attributes.filterValues { it.isBlank() }.map { it.key }.orNull() + withAttributeKeys = attributes.filterValues { it.isBlank() }.map { it.key }.orNull(), ).toString() } diff --git a/html-parser/src/main/kotlin/it/skrape/selects/html5/CustomTagSelectors.kt b/html-parser/src/main/kotlin/it/skrape/selects/html5/CustomTagSelectors.kt index adc861b9..e2029713 100644 --- a/html-parser/src/main/kotlin/it/skrape/selects/html5/CustomTagSelectors.kt +++ b/html-parser/src/main/kotlin/it/skrape/selects/html5/CustomTagSelectors.kt @@ -5,7 +5,7 @@ import it.skrape.selects.CssSelector /** * Will define a html5 custom tags css query selector. - * By default it will just be the specific tag-name. + * By default, it will just be the specific tag-name. * It is possible to define a more concrete selector by using the provided fields of the CssSelector object or * by passing a raw css query selector as parameter. * If a selector is passed as parameter as well as be defined via CssSelector fields, they will be merged. diff --git a/html-parser/src/main/kotlin/it/skrape/selects/html5/DemarcatingEditsSelectors.kt b/html-parser/src/main/kotlin/it/skrape/selects/html5/DemarcatingEditsSelectors.kt index 0f70efff..4e83798f 100644 --- a/html-parser/src/main/kotlin/it/skrape/selects/html5/DemarcatingEditsSelectors.kt +++ b/html-parser/src/main/kotlin/it/skrape/selects/html5/DemarcatingEditsSelectors.kt @@ -5,7 +5,7 @@ import it.skrape.selects.CssSelector /** * Will define a -tags css query selector. - * By default it will just be the specific tag-name. + * By default, it will just be the specific tag-name. * It is possible to define a more concrete selector by using the provided fields of the CssSelector object or * by passing a raw css query selector as parameter. * If a selector is passed as parameter as well as be defined via CssSelector fields, they will be merged. @@ -18,7 +18,7 @@ public fun CssSelectable.del(cssSelector: String = "", init: CssSelector.() /** * Will define a -tags css query selector. - * By default it will just be the specific tag-name. + * By default, it will just be the specific tag-name. * It is possible to define a more concrete selector by using the provided fields of the CssSelector object or * by passing a raw css query selector as parameter. * If a selector is passed as parameter as well as be defined via CssSelector fields, they will be merged. diff --git a/html-parser/src/main/kotlin/it/skrape/selects/html5/EmbeddedContentSelectors.kt b/html-parser/src/main/kotlin/it/skrape/selects/html5/EmbeddedContentSelectors.kt index 873dfd02..d5e96790 100644 --- a/html-parser/src/main/kotlin/it/skrape/selects/html5/EmbeddedContentSelectors.kt +++ b/html-parser/src/main/kotlin/it/skrape/selects/html5/EmbeddedContentSelectors.kt @@ -5,7 +5,7 @@ import it.skrape.selects.CssSelector /** * Will define a -tags css query selector. - * By default it will just be the specific tag-name. + * By default, it will just be the specific tag-name. * It is possible to define a more concrete selector by using the provided fields of the CssSelector object or * by passing a raw css query selector as parameter. * If a selector is passed as parameter as well as be defined via CssSelector fields, they will be merged. @@ -14,11 +14,11 @@ import it.skrape.selects.CssSelector * @return T */ public fun CssSelectable.applet(cssSelector: String = "", init: CssSelector.() -> T): T = - selection("applet$cssSelector", init) + selection("applet$cssSelector", init) /** * Will define a -tags css query selector. - * By default it will just be the specific tag-name. + * By default, it will just be the specific tag-name. * It is possible to define a more concrete selector by using the provided fields of the CssSelector object or * by passing a raw css query selector as parameter. * If a selector is passed as parameter as well as be defined via CssSelector fields, they will be merged. @@ -27,11 +27,11 @@ public fun CssSelectable.applet(cssSelector: String = "", init: CssSelector. * @return T */ public fun CssSelectable.embed(cssSelector: String = "", init: CssSelector.() -> T): T = - selection("embed$cssSelector", init) + selection("embed$cssSelector", init) /** * Will define a