From 7b1358c70809d88950bcfc3fc8cffa932c94e351 Mon Sep 17 00:00:00 2001 From: Edward Cooke Date: Mon, 18 Jul 2022 09:46:24 -0600 Subject: [PATCH] Multi-line scalars need to be indented. --- .../Serialization/DeserializerTest.cs | 112 +++++++++ YamlDotNet/Core/Scanner.cs | 216 ++++++++++++++++-- 2 files changed, 309 insertions(+), 19 deletions(-) diff --git a/YamlDotNet.Test/Serialization/DeserializerTest.cs b/YamlDotNet.Test/Serialization/DeserializerTest.cs index 30046df7..bbe4853e 100644 --- a/YamlDotNet.Test/Serialization/DeserializerTest.cs +++ b/YamlDotNet.Test/Serialization/DeserializerTest.cs @@ -24,6 +24,7 @@ using System.Linq; using FluentAssertions; using Xunit; +using YamlDotNet.Core; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -259,6 +260,117 @@ public void DeserializeScalarEdgeCases(IConvertible value, Type type) result.Should().Be(value); } + [Fact] + public void DeserializeMultiLineSingleQuoteRequiresCorrectFormatting() + { + var yaml = @"attach_workspace: &attach_workspace + attach_workspace: + at: '.' + +deploy_nonprod_job: &deploy_nonprod_job + working_directory: '~/build' + docker: [ {image: 'test.com'} ] + steps: + - *attach_workspace + - run: 'deploy-to-test' + +version: 2 + +jobs: + build_test_image: + working_directory: '~/build + docker: + - image: docker:17.05.0-ce-git + environment: + DOCKER_IMAGE_NAME: testApp + steps: + - run: + name: Install bash and curl + command: | + apk update + apk add -y curl + apk add -y bash + apk update + apk upgrade + + deploy_test: + environment: + HAL_TARGETS: '45597' + HAL_BUILD_FILE: '.hal_build_id_test' + +workflows: + version: 2 + pipeline: + jobs: + - build_test_image + - deploy_test +"; + var deserializer = new DeserializerBuilder().Build(); + var exception = Assert.Throws(() => + { + var o = deserializer.Deserialize(yaml, typeof(object)); + }); + Assert.Equal(16, exception.Start.Line); + Assert.Equal(28, exception.Start.Column); + Assert.Equal(338, exception.Start.Index); + Assert.Equal(16, exception.End.Line); + Assert.Equal(36, exception.End.Column); + Assert.Equal(346, exception.End.Index); + } + + [Fact] + public void DeserializeMultiLineSingleQuoteWorksWithCorrectFormatting() + { + var yaml = @"attach_workspace: &attach_workspace + attach_workspace: + at: '.' + +deploy_nonprod_job: &deploy_nonprod_job + working_directory: '~/build' + docker: [ {image: 'test.com'} ] + steps: + - *attach_workspace + - run: 'deploy-to-test' + +version: 2 + +jobs: + build_test_image: + working_directory: '~/build + hello' + docker: + - image: docker:17.05.0-ce-git + environment: + DOCKER_IMAGE_NAME: testApp + steps: + - run: + name: Install bash and curl + command: | + apk update + apk add -y curl + apk add -y bash + apk update + apk upgrade + + deploy_test: + environment: + HAL_TARGETS: '45597' + HAL_BUILD_FILE: '.hal_build_id_test' + +workflows: + version: 2 + pipeline: + jobs: + - build_test_image + - deploy_test +"; + var deserializer = new DeserializerBuilder().Build(); + var o = (IDictionary)deserializer.Deserialize(yaml, typeof(object)); + o = (IDictionary)o["jobs"]; + o = (IDictionary)o["build_test_image"]; + Assert.Equal("~/build hello", o["working_directory"]); + } + public class Test { public string Value { get; set; } diff --git a/YamlDotNet/Core/Scanner.cs b/YamlDotNet/Core/Scanner.cs index 081f4676..fe7aeceb 100644 --- a/YamlDotNet/Core/Scanner.cs +++ b/YamlDotNet/Core/Scanner.cs @@ -79,6 +79,7 @@ public class Scanner : IScanner private Token? previous; private Anchor? previousAnchor; private Scalar? lastScalar = null; + private Scalar? lastKeyScalar = null; private bool IsDocumentStart() => !analyzer.EndOfInput && @@ -426,6 +427,7 @@ private void FetchNextToken() { if (lastScalar != null) { + lastKeyScalar = lastScalar; lastScalar.IsKey = true; lastScalar = null; } @@ -1785,8 +1787,16 @@ private void FetchFlowScalar(bool isSingleQuoted) flowScalarFetched = true; // Create the SCALAR token and append it to the queue. - - tokens.Enqueue(ScanFlowScalar(isSingleQuoted)); + Token token; + if (isSingleQuoted) + { + token = ScanSingleQuotedFlowScalar(); + } + else + { + token = ScanDoubleQuotedFlowScalar(); + } + tokens.Enqueue(token); // Check if there is a comment subsequently after double-quoted scalar without space. @@ -1798,10 +1808,10 @@ private void FetchFlowScalar(bool isSingleQuoted) } /// - /// Scan a quoted scalar. + /// Scan a single quoted scalar. /// - private Token ScanFlowScalar(bool isSingleQuoted) + private Token ScanSingleQuotedFlowScalar() { // Eat the left quote. @@ -1841,11 +1851,6 @@ private Token ScanFlowScalar(bool isSingleQuoted) throw new SyntaxErrorException(start, cursor.Mark(), "While scanning a quoted scalar, found unexpected end of stream."); } - if (hasLeadingBlanks && !isSingleQuoted && indent >= cursor.LineOffset) - { - throw new SyntaxErrorException(start, cursor.Mark(), "While scanning a multi-line double-quoted scalar, found wrong indentation."); - } - hasLeadingBlanks = false; // Consume non-blank characters. @@ -1853,24 +1858,181 @@ private Token ScanFlowScalar(bool isSingleQuoted) while (!analyzer.IsWhiteBreakOrZero()) { // Check for an escaped single quote. - - if (isSingleQuoted && analyzer.Check('\'', 0) && analyzer.Check('\'', 1)) + if (analyzer.Check('\'', 0) && analyzer.Check('\'', 1)) { value.Append('\''); Skip(); Skip(); } + // right quote + else if (analyzer.Check('\'')) + { + break; + } + // It is a non-escaped non-blank character. + else + { + value.Append(ReadCurrentCharacter()); + } + } - // Check for the right quote. + // Check if we are at the end of the scalar. + if (analyzer.Check('\'')) + { + break; + } + + Mark? last = cursor.Mark(); + // Consume blank characters. + while (analyzer.IsWhite() || analyzer.IsBreak()) + { + if (analyzer.IsWhite()) + { + // Consume a space or a tab character. + if (!hasLeadingBlanks) + { + whitespaces.Append(ReadCurrentCharacter()); + } + else + { + Skip(); + } + } + else + { + // Check if it is a first line break. + if (!hasLeadingBlanks) + { + whitespaces.Length = 0; + leadingBreak.Append(ReadLine()); + hasLeadingBlanks = true; + } + else + { + trailingBreaks.Append(ReadLine()); + } + + // go to the current indent level + for (var i = 1; i <= indent; i++) + { + if (!analyzer.IsWhite()) + { + throw new SyntaxErrorException(start, last ?? cursor.Mark(), "Unexpected non-space character while parsing multi-line single quoted scalar."); + } + Skip(); + } + + // make sure we are indented by a space or new-lined when indented. + if (!analyzer.IsWhite() && !analyzer.IsBreak() && indent >= 0) + { + //single quotes need to have a white space character. Otherwise it is an invalid format. + throw new SyntaxErrorException(start, last ?? cursor.Mark(), "Unexpected line break while parsing single quoted scalar."); + } + } + } - else if (analyzer.Check(isSingleQuoted ? '\'' : '"')) + // Join the whitespaces or fold line breaks. + if (hasLeadingBlanks) + { + // Do we need to fold line breaks? + if (StartsWith(leadingBreak, '\n')) + { + if (trailingBreaks.Length == 0) + { + value.Append(' '); + } + else + { + value.Append(trailingBreaks.ToString()); + } + } + else + { + value.Append(leadingBreak.ToString()); + value.Append(trailingBreaks.ToString()); + } + leadingBreak.Length = 0; + trailingBreaks.Length = 0; + } + else + { + value.Append(whitespaces.ToString()); + whitespaces.Length = 0; + } + } + + // Eat the right quote. + + Skip(); + + return new Scalar(value.ToString(), ScalarStyle.SingleQuoted, start, cursor.Mark()); + } + + + /// + /// Scan a double quoted scalar. + /// + + private Token ScanDoubleQuotedFlowScalar() + { + // Eat the left quote. + + var start = cursor.Mark(); + + Skip(); + + // Consume the content of the quoted scalar. + + using var valueBuilder = StringBuilderPool.Rent(); + var value = valueBuilder.Builder; + + using var whitespacesBuilder = StringBuilderPool.Rent(); + var whitespaces = whitespacesBuilder.Builder; + + using var leadingBreakBuilder = StringBuilderPool.Rent(); + var leadingBreak = leadingBreakBuilder.Builder; + + using var trailingBreaksBuilder = StringBuilderPool.Rent(); + var trailingBreaks = trailingBreaksBuilder.Builder; + + var hasLeadingBlanks = false; + + while (true) + { + // Check that there are no document indicators at the beginning of the line. + + if (IsDocumentIndicator()) + { + throw new SyntaxErrorException(start, cursor.Mark(), "While scanning a quoted scalar, found unexpected document indicator."); + } + + // Check for EOF. + + if (analyzer.IsZero()) + { + throw new SyntaxErrorException(start, cursor.Mark(), "While scanning a quoted scalar, found unexpected end of stream."); + } + + if (hasLeadingBlanks && indent >= cursor.LineOffset) + { + throw new SyntaxErrorException(start, cursor.Mark(), "While scanning a multi-line double-quoted scalar, found wrong indentation."); + } + + hasLeadingBlanks = false; + + // Consume non-blank characters. + + while (!analyzer.IsWhiteBreakOrZero()) + { + // Check for the right quote. + if (analyzer.Check('"')) { break; } // Check for an escaped line break. - else if (!isSingleQuoted && analyzer.Check('\\') && analyzer.IsBreak(1)) + else if (analyzer.Check('\\') && analyzer.IsBreak(1)) { Skip(); SkipLine(); @@ -1880,7 +2042,7 @@ private Token ScanFlowScalar(bool isSingleQuoted) // Check for an escape sequence. - else if (!isSingleQuoted && analyzer.Check('\\')) + else if (analyzer.Check('\\')) { var codeLength = 0; @@ -1961,13 +2123,13 @@ private Token ScanFlowScalar(bool isSingleQuoted) // Check if we are at the end of the scalar. - if (analyzer.Check(isSingleQuoted ? '\'' : '"')) + if (analyzer.Check('"')) { break; } + Mark? last = cursor.Mark(); // Consume blank characters. - while (analyzer.IsWhite() || analyzer.IsBreak()) { if (analyzer.IsWhite()) @@ -1986,7 +2148,6 @@ private Token ScanFlowScalar(bool isSingleQuoted) else { // Check if it is a first line break. - if (!hasLeadingBlanks) { whitespaces.Length = 0; @@ -1997,6 +2158,23 @@ private Token ScanFlowScalar(bool isSingleQuoted) { trailingBreaks.Append(ReadLine()); } + + // go to the current indent level + for (var i = 1; i <= indent; i++) + { + if (!analyzer.IsWhite()) + { + throw new SyntaxErrorException(start, last ?? cursor.Mark(), "Unexpected character while parsing double quoted scalar."); + } + Skip(); + } + + // make sure we are indented by a space or new-lined when indented. + if (!analyzer.IsWhite() && !analyzer.IsBreak() && indent >= 0) + { + //single quotes need to have a white space character. Otherwise it is an invalid format. + throw new SyntaxErrorException(start, last ?? cursor.Mark(), "Unexpected line break while parsing double quoted scalar."); + } } } @@ -2036,7 +2214,7 @@ private Token ScanFlowScalar(bool isSingleQuoted) Skip(); - return new Scalar(value.ToString(), isSingleQuoted ? ScalarStyle.SingleQuoted : ScalarStyle.DoubleQuoted, start, cursor.Mark()); + return new Scalar(value.ToString(), ScalarStyle.DoubleQuoted, start, cursor.Mark()); } ///