Add support for IDNA 2008 (RFC 5891) when encoding URL (#819)

buildSrc/src/main/kotlin/Constants.kt

@@ -238,3 +238,8 @@ object Stetho {
         const val dependency = "com.facebook.stetho:stetho-urlconnection:$version"
     }
 }
+
+object ICU {
+    const val version = "70.1"
+    const val dependency = "com.ibm.icu:icu4j:$version"
+}

fuel/build.gradle.kts

@@ -1,5 +1,6 @@
 dependencies {
     api(Result.dependency)
+    implementation(ICU.dependency)
 
     testImplementation(project(Fuel.Test.name))
     testImplementation(Json.dependency)

fuel/src/main/kotlin/com/github/kittinunf/fuel/core/Encoding.kt

@@ -1,9 +1,9 @@
 package com.github.kittinunf.fuel.core
 
 import com.github.kittinunf.fuel.core.requests.DefaultRequest
+import com.ibm.icu.text.IDNA
 import java.net.MalformedURLException
 import java.net.URI
-import java.net.URISyntaxException
 import java.net.URL
 
 class Encoding(
@@ -13,6 +13,8 @@ class Encoding(
     val parameters: Parameters? = null
 ) : RequestFactory.RequestConvertible {
 
+    private val idna: IDNA = IDNA.getUTS46Instance(flags)
+
     private val encoder: (Method, String, Parameters?) -> Request = { method, path, parameters ->
         DefaultRequest(
             method = method,
@@ -36,13 +38,35 @@ class Encoding(
             }
             URL(base + if (path.startsWith('/') or path.isEmpty()) path else "/$path")
         }
-        val uri = try {
-            url.toURI()
-        } catch (e: URISyntaxException) {
-            URI(url.protocol, url.userInfo, url.host, url.port, url.path, url.query, url.ref)
-        }
+
+        val uri = URI(
+            url.protocol,
+            url.userInfo,
+            // converts domain to A-Label (RFC 5891)
+            domainToAscii(url.host),
+            url.port,
+            url.path,
+            url.query,
+            url.ref
+        )
+
         return URL(uri.toASCIIString())
     }
 
+    private fun domainToAscii(domain: String): String {
+        val info = IDNA.Info()
+        val sb = StringBuilder()
+        val domainAscii = idna.nameToASCII(domain, sb, info).toString()
+        if (info.hasErrors()) {
+            throw MalformedURLException(info.errors.toString())
+        }
+        return domainAscii
+    }
+
     private val defaultHeaders = Headers.from()
+
+    companion object {
+        private const val flags =
+            IDNA.CHECK_BIDI or IDNA.CHECK_CONTEXTJ or IDNA.CHECK_CONTEXTO or IDNA.NONTRANSITIONAL_TO_ASCII or IDNA.USE_STD3_RULES
+    }
 }

fuel/src/test/kotlin/com/github/kittinunf/fuel/core/EncodingTest.kt

@@ -0,0 +1,72 @@
+package com.github.kittinunf.fuel.core
+
+import java.net.MalformedURLException
+import org.hamcrest.CoreMatchers.equalTo
+import org.hamcrest.MatcherAssert.assertThat
+import org.junit.Test
+import org.junit.experimental.runners.Enclosed
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+@RunWith(Enclosed::class)
+internal class EncodingTest {
+
+    @RunWith(Parameterized::class)
+    internal class Valid(
+        private val inputUrl: String,
+        private val encodedUrl: String
+    ) {
+
+        companion object {
+            @JvmStatic
+            @Parameterized.Parameters(name = "validUrl")
+            fun data() = listOf(
+                arrayOf("https://github.com/kittinunf/fuel/", "https://github.com/kittinunf/fuel/"),
+                arrayOf(
+                    "https://xn----f38am99bqvcd5liy1cxsg.test",
+                    "https://xn----f38am99bqvcd5liy1cxsg.test"
+                ),
+                arrayOf("https://test.xn--rhqv96g", "https://test.xn--rhqv96g"),
+                arrayOf("https://test.شبك", "https://test.xn--ngbx0c"),
+                arrayOf("https://普遍接受-测试.top", "https://xn----f38am99bqvcd5liy1cxsg.top"),
+                arrayOf(
+                    "https://मेल.डाटामेल.भारत",
+                    "https://xn--r2bi6d.xn--c2bd4bq1db8d.xn--h2brj9c"
+                ),
+                arrayOf("http://fußball.de", "http://xn--fuball-cta.de"),
+                arrayOf("http://fußball.de", "http://xn--fuball-cta.de"),
+            )
+        }
+
+        @Test
+        fun testRequestURLIDNAEncoding() {
+            val encoding = Encoding(
+                httpMethod = Method.GET,
+                urlString = inputUrl,
+            )
+            assertThat(encoding.request.url.toString(), equalTo(encodedUrl))
+        }
+    }
+
+
+    @RunWith(Parameterized::class)
+    internal class Invalid(
+        private val inputUrl: String
+    ) {
+
+        companion object {
+            @JvmStatic
+            @Parameterized.Parameters(name = "invalidUrl")
+            fun data() = listOf(
+                "https://in--valid",
+                "https://.test.top",
+                "https://\\u0557w.test"
+            )
+        }
+
+        @Test(expected = MalformedURLException::class)
+        fun testRequestInvalidURL() {
+            Encoding(httpMethod = Method.GET, urlString = inputUrl).request
+        }
+    }
+}