Use REPLACE for charset decoding errors

- replace malformed inputs and unmappable characters with "\uFFFD" - replace solo newline tokens with "#a" in compiled model map files Signed-off-by: jrte <[email protected]>
jrte · Aug 21, 2023 · 829a7d3 · 829a7d3
1 parent d012eae
commit 829a7d3
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 13 deletions.
diff --git a/src/com/characterforming/jrte/engine/Base.java b/src/com/characterforming/jrte/engine/Base.java
@@ -23,6 +23,7 @@
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
 import java.util.logging.FileHandler;
 import java.util.logging.Handler;
 import java.util.logging.Level;
@@ -118,7 +119,9 @@ public static void endLogging() {
 	 * @return a new CharsetDecoder insstance
 	 */
 	public static CharsetDecoder newCharsetDecoder() {
-		return Base.runtimeCharset.newDecoder();
+		return Base.runtimeCharset.newDecoder()
+			.onUnmappableCharacter(CodingErrorAction.REPLACE)
+			.onMalformedInput(CodingErrorAction.REPLACE);
 	}
 
 	/**

diff --git a/src/com/characterforming/jrte/engine/Token.java b/src/com/characterforming/jrte/engine/Token.java
@@ -35,7 +35,7 @@ final class Token implements IToken {
 	private final IToken.Type type;
 	private final byte[] literal;
 	private final byte[] symbol;
-	private int ordinal;
+	private final int ordinal;
 
 	/**
 	 * Assemble tokens from effector parameter bytes. For internal use
@@ -139,10 +139,11 @@ public int getSymbolOrdinal() {
 
 	@Override
 	public String toString() {
-		try {
+		if (this.literal.length == 1 && this.literal[0] == 0xa) {
+			return "#a";
+		} else try {
 			CharsetDecoder decoder = Base.newCharsetDecoder();
-			ByteBuffer buffer = ByteBuffer.wrap(getLiteralValue());
-			return decoder.decode(buffer).toString();
+			return decoder.decode(ByteBuffer.wrap(getLiteralValue())).toString();
 		} catch (Exception e) {
 			Bytes data = new Bytes(getLiteralValue());
 			return data.toHexString();

diff --git a/src/com/characterforming/jrte/engine/Transductor.java b/src/com/characterforming/jrte/engine/Transductor.java
@@ -747,8 +747,7 @@ public int invoke(final int parameterIndex) throws EffectorException {
 					selected.append(token.getLiteralValue());
 				} else {
 					throw new EffectorException(String.format("Invalid token `%1$s` for effector '%2$s'", 
-						Bytes.decode(Base.newCharsetDecoder(), token.getLiteralValue(), token.getLiteralValue().length), 
-						super.getName()));
+						token.toString(), super.getName()));
 				}
 			}
 			return IEffector.RTX_NONE;
@@ -905,6 +904,7 @@ public int invoke(final int parameterIndex) throws EffectorException {
 					IField field = getField(parameters[i].getSymbolOrdinal());
 					tokens[i] = (field != null) ? field.copyValue() : Bytes.EMPTY_BYTES;
 				} else {
+					assert parameters[i].getType() == IToken.Type.LITERAL;
 					tokens[i] = parameters[i].getLiteralValue();
 				}
 			}
@@ -934,6 +934,7 @@ public int invoke(final int parameterIndex) throws EffectorException {
 								super.target.output.write(field.getData(), 0, field.getLength());
 							}
 						} else {
+							assert token.getType() == IToken.Type.LITERAL;
 							byte[] data = token.getLiteralValue();
 							super.target.output.write(data, 0, data.length);
 						}
@@ -1000,9 +1001,8 @@ public int[] compileParameter(final IToken[] parameterList) throws TargetBinding
 				assert signalOrdinal >= SIGNUL;
 				return new int[] { count, signalOrdinal };
 			} else {
-				byte[] value = parameterList[1].getLiteralValue();
 				throw new TargetBindingException(String.format("%1$s.%2$s: invalid signal '%3$%s' for count effector",
-					super.getTarget().getName(), super.getName(), Bytes.decode(super.getDecoder(), value, value.length)));
+					super.getTarget().getName(), super.getName(), parameterList[1].toString()));
 			}
 		}
 

diff --git a/src/com/characterforming/ribose/base/BaseParameterizedEffector.java b/src/com/characterforming/ribose/base/BaseParameterizedEffector.java
@@ -72,10 +72,7 @@ public void setParameters(Object proxy) {
 	public String showParameterTokens(int parameterIndex) {
 		StringBuilder sb = new StringBuilder(256);
 		for (IToken token : getParameterTokens(parameterIndex)) {
-			byte[] name = token.getLiteralValue();
-			sb.append(sb.length() > 0 ? " `" : "`");
-			sb.append(Bytes.decode(super.getDecoder(), name, name.length));
-			sb.append("`");
+			sb.append(sb.length() == 0 ? "`" : " `").append(token.toString()).append("`");
 		}
 		return sb.toString();
 	}