From 44a4c071859a52e81784a9d7f99d3ccf2e2ef43d Mon Sep 17 00:00:00 2001 From: Julien BERNARD Date: Mon, 29 Nov 2021 10:32:17 -0500 Subject: [PATCH] Add support for IDNA 2008 (RFC 5891) when encoding URL (#819) --- buildSrc/src/main/kotlin/Constants.kt | 5 ++ fuel/build.gradle.kts | 1 + .../github/kittinunf/fuel/core/Encoding.kt | 36 ++++++++-- .../kittinunf/fuel/core/EncodingTest.kt | 72 +++++++++++++++++++ 4 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 fuel/src/test/kotlin/com/github/kittinunf/fuel/core/EncodingTest.kt diff --git a/buildSrc/src/main/kotlin/Constants.kt b/buildSrc/src/main/kotlin/Constants.kt index 9ef41e001..dc2ee141b 100644 --- a/buildSrc/src/main/kotlin/Constants.kt +++ b/buildSrc/src/main/kotlin/Constants.kt @@ -238,3 +238,8 @@ object Stetho { const val dependency = "com.facebook.stetho:stetho-urlconnection:$version" } } + +object ICU { + const val version = "70.1" + const val dependency = "com.ibm.icu:icu4j:$version" +} diff --git a/fuel/build.gradle.kts b/fuel/build.gradle.kts index bb620a936..d43d1e070 100644 --- a/fuel/build.gradle.kts +++ b/fuel/build.gradle.kts @@ -1,5 +1,6 @@ dependencies { api(Result.dependency) + implementation(ICU.dependency) testImplementation(project(Fuel.Test.name)) testImplementation(Json.dependency) diff --git a/fuel/src/main/kotlin/com/github/kittinunf/fuel/core/Encoding.kt b/fuel/src/main/kotlin/com/github/kittinunf/fuel/core/Encoding.kt index c7b0993b3..2c9e5a356 100644 --- a/fuel/src/main/kotlin/com/github/kittinunf/fuel/core/Encoding.kt +++ b/fuel/src/main/kotlin/com/github/kittinunf/fuel/core/Encoding.kt @@ -1,9 +1,9 @@ package com.github.kittinunf.fuel.core import com.github.kittinunf.fuel.core.requests.DefaultRequest +import com.ibm.icu.text.IDNA import java.net.MalformedURLException import java.net.URI -import java.net.URISyntaxException import java.net.URL class Encoding( @@ -13,6 +13,8 @@ class Encoding( val parameters: Parameters? = null ) : RequestFactory.RequestConvertible { + private val idna: IDNA = IDNA.getUTS46Instance(flags) + private val encoder: (Method, String, Parameters?) -> Request = { method, path, parameters -> DefaultRequest( method = method, @@ -36,13 +38,35 @@ class Encoding( } URL(base + if (path.startsWith('/') or path.isEmpty()) path else "/$path") } - val uri = try { - url.toURI() - } catch (e: URISyntaxException) { - URI(url.protocol, url.userInfo, url.host, url.port, url.path, url.query, url.ref) - } + + val uri = URI( + url.protocol, + url.userInfo, + // converts domain to A-Label (RFC 5891) + domainToAscii(url.host), + url.port, + url.path, + url.query, + url.ref + ) + return URL(uri.toASCIIString()) } + private fun domainToAscii(domain: String): String { + val info = IDNA.Info() + val sb = StringBuilder() + val domainAscii = idna.nameToASCII(domain, sb, info).toString() + if (info.hasErrors()) { + throw MalformedURLException(info.errors.toString()) + } + return domainAscii + } + private val defaultHeaders = Headers.from() + + companion object { + private const val flags = + IDNA.CHECK_BIDI or IDNA.CHECK_CONTEXTJ or IDNA.CHECK_CONTEXTO or IDNA.NONTRANSITIONAL_TO_ASCII or IDNA.USE_STD3_RULES + } } diff --git a/fuel/src/test/kotlin/com/github/kittinunf/fuel/core/EncodingTest.kt b/fuel/src/test/kotlin/com/github/kittinunf/fuel/core/EncodingTest.kt new file mode 100644 index 000000000..de9eedcec --- /dev/null +++ b/fuel/src/test/kotlin/com/github/kittinunf/fuel/core/EncodingTest.kt @@ -0,0 +1,72 @@ +package com.github.kittinunf.fuel.core + +import java.net.MalformedURLException +import org.hamcrest.CoreMatchers.equalTo +import org.hamcrest.MatcherAssert.assertThat +import org.junit.Test +import org.junit.experimental.runners.Enclosed +import org.junit.runner.RunWith +import org.junit.runners.Parameterized + +@RunWith(Enclosed::class) +internal class EncodingTest { + + @RunWith(Parameterized::class) + internal class Valid( + private val inputUrl: String, + private val encodedUrl: String + ) { + + companion object { + @JvmStatic + @Parameterized.Parameters(name = "validUrl") + fun data() = listOf( + arrayOf("https://github.com/kittinunf/fuel/", "https://github.com/kittinunf/fuel/"), + arrayOf( + "https://xn----f38am99bqvcd5liy1cxsg.test", + "https://xn----f38am99bqvcd5liy1cxsg.test" + ), + arrayOf("https://test.xn--rhqv96g", "https://test.xn--rhqv96g"), + arrayOf("https://test.شبك", "https://test.xn--ngbx0c"), + arrayOf("https://普遍接受-测试.top", "https://xn----f38am99bqvcd5liy1cxsg.top"), + arrayOf( + "https://मेल.डाटामेल.भारत", + "https://xn--r2bi6d.xn--c2bd4bq1db8d.xn--h2brj9c" + ), + arrayOf("http://fußball.de", "http://xn--fuball-cta.de"), + arrayOf("http://fußball.de", "http://xn--fuball-cta.de"), + ) + } + + @Test + fun testRequestURLIDNAEncoding() { + val encoding = Encoding( + httpMethod = Method.GET, + urlString = inputUrl, + ) + assertThat(encoding.request.url.toString(), equalTo(encodedUrl)) + } + } + + + @RunWith(Parameterized::class) + internal class Invalid( + private val inputUrl: String + ) { + + companion object { + @JvmStatic + @Parameterized.Parameters(name = "invalidUrl") + fun data() = listOf( + "https://in--valid", + "https://.test.top", + "https://\\u0557w.test" + ) + } + + @Test(expected = MalformedURLException::class) + fun testRequestInvalidURL() { + Encoding(httpMethod = Method.GET, urlString = inputUrl).request + } + } +}