From ed6d4b74173a10bbddabf1dbe6929c89a9b98df6 Mon Sep 17 00:00:00 2001
From: Isak Lindbeck <isak.lindbeck@outlook.com>
Date: Mon, 14 Oct 2024 15:05:42 +0200
Subject: [PATCH] AVRO-4062: Allow leading underscores for names in idl (#3178)

* AVRO-4062 [java] Fix broken test data

* AVRO-4062 [java] Make maven run unit test for IdlReader

The test is not run automatically unless the class naming convention is followed

* AVRO-4062 Allow leading underscore for names in idl

* AVRO-4062 [java] Add test for leading underscore in names

---------

Co-authored-by: Isak Lindbeck <isak.lindbeck@evinova.com>
---
 .../test/idl/input/leading_underscore.avdl    | 26 +++++++++++++++++++
 .../test/idl/output/leading_underscore.avpr   | 17 ++++++++++++
 lang/java/idl/src/test/idl/output/simple.avpr |  4 +--
 ...{IdlReaderTest.java => TestIdlReader.java} |  3 +--
 share/idl_grammar/org/apache/avro/idl/Idl.g4  |  4 +--
 5 files changed, 48 insertions(+), 6 deletions(-)
 create mode 100644 lang/java/idl/src/test/idl/input/leading_underscore.avdl
 create mode 100644 lang/java/idl/src/test/idl/output/leading_underscore.avpr
 rename lang/java/idl/src/test/java/org/apache/avro/idl/{IdlReaderTest.java => TestIdlReader.java} (99%)

diff --git a/lang/java/idl/src/test/idl/input/leading_underscore.avdl b/lang/java/idl/src/test/idl/input/leading_underscore.avdl
new file mode 100644
index 00000000000..cd5db70beb7
--- /dev/null
+++ b/lang/java/idl/src/test/idl/input/leading_underscore.avdl
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+Testing leading underscore for identifiers
+*/
+protocol _LeadingUnderscore {
+  record _TestRecord {
+    string _testField;
+  }
+}
diff --git a/lang/java/idl/src/test/idl/output/leading_underscore.avpr b/lang/java/idl/src/test/idl/output/leading_underscore.avpr
new file mode 100644
index 00000000000..af30efa3dc4
--- /dev/null
+++ b/lang/java/idl/src/test/idl/output/leading_underscore.avpr
@@ -0,0 +1,17 @@
+{
+  "protocol" : "_LeadingUnderscore",
+  "doc":"Testing leading underscore for identifiers",
+  "types" : [
+    {
+      "type" : "record",
+      "name" : "_TestRecord",
+      "fields" : [
+        {
+          "name" : "_testField",
+          "type" : "string"
+        }
+      ]
+    }
+  ],
+  "messages" : {}
+}
diff --git a/lang/java/idl/src/test/idl/output/simple.avpr b/lang/java/idl/src/test/idl/output/simple.avpr
index edcb68536e3..004e32480d9 100644
--- a/lang/java/idl/src/test/idl/output/simple.avpr
+++ b/lang/java/idl/src/test/idl/output/simple.avpr
@@ -80,11 +80,11 @@
       "default": null
     }, {
       "name": "numbers",
-      "type": {"type: array", "items": "int"},
+      "type": {"type": "array", "items": "int"},
       "default": []
     }, {
       "name": "dictionary",
-      "type": {"type: map", "items": "string"},
+      "type": {"type": "map", "values": "string"},
       "default": {}
     }],
     "my-property" : {
diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/IdlReaderTest.java b/lang/java/idl/src/test/java/org/apache/avro/idl/TestIdlReader.java
similarity index 99%
rename from lang/java/idl/src/test/java/org/apache/avro/idl/IdlReaderTest.java
rename to lang/java/idl/src/test/java/org/apache/avro/idl/TestIdlReader.java
index 9b503bcb2a5..96034cf3950 100644
--- a/lang/java/idl/src/test/java/org/apache/avro/idl/IdlReaderTest.java
+++ b/lang/java/idl/src/test/java/org/apache/avro/idl/TestIdlReader.java
@@ -23,7 +23,6 @@
 import org.apache.avro.Schema;
 import org.junit.Before;
 import org.junit.Test;
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
@@ -55,7 +54,7 @@
  * To make it simpler to write these tests, you can run ant -Dtestcase=TestIdl
  * -Dtest.idl.mode=write, which will *replace* all expected output.
  */
-public class IdlReaderTest {
+public class TestIdlReader {
   private static final File TEST_DIR = new File(System.getProperty("test.idl.dir", "src/test/idl"));
 
   private static final File TEST_INPUT_DIR = new File(TEST_DIR, "input").getAbsoluteFile();
diff --git a/share/idl_grammar/org/apache/avro/idl/Idl.g4 b/share/idl_grammar/org/apache/avro/idl/Idl.g4
index 81799bbb3ed..927962c5b73 100644
--- a/share/idl_grammar/org/apache/avro/idl/Idl.g4
+++ b/share/idl_grammar/org/apache/avro/idl/Idl.g4
@@ -252,6 +252,6 @@ fragment HexadecimalExponent: [pP] [+\-]? Digit+;
  * Also note that this token should *only* be used in the identifier grammar rule above.
  */
 IdentifierToken: ( '`' IdentifierPart '`' | IdentifierPart)([.-] ( '`' IdentifierPart '`' | IdentifierPart) )*;
-fragment IdentifierPart: [\p{XID_Start}] [\p{XID_Continue}]*;
+fragment IdentifierPart: [\p{XID_Start}_] [\p{XID_Continue}]*;
 // See discussion in AVRO-1022, AVRO-2659, AVRO-3115
-// fragment IdentifierPart: [A-Za-z] [A-Za-z0-9_]*
+// fragment IdentifierPart: [A-Za-z_] [A-Za-z0-9_]*