From b8351112caa0455101e3801db87a23ebb80e245f Mon Sep 17 00:00:00 2001 From: Marek Aufart Date: Fri, 25 Aug 2023 10:44:35 +0200 Subject: [PATCH 1/6] Add XMLPublicid condition to buildtin provider Adding XML-like public-id condition supporting regular expression evaluation on the public-id attribute. Related to https://github.com/konveyor/windup-shim/pull/87 Related to https://github.com/konveyor/analyzer-lsp/issues/221 Signed-off-by: Marek Aufart --- provider/internal/builtin/provider.go | 7 ++ provider/internal/builtin/service_client.go | 73 +++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/provider/internal/builtin/provider.go b/provider/internal/builtin/provider.go index 3f243299..801a7d8c 100644 --- a/provider/internal/builtin/provider.go +++ b/provider/internal/builtin/provider.go @@ -54,6 +54,7 @@ type builtinCondition struct { Filecontent fileContentCondition `yaml:"filecontent"` File fileCondition `yaml:"file"` XML xmlCondition `yaml:"xml"` + XMLPublicid xmlPublicidCondition `yaml:"xmlPublicid"` JSON jsonCondition `yaml:"json"` HasTags []string `yaml:"hasTags"` provider.ProviderContext `yaml:",inline"` @@ -76,6 +77,12 @@ type xmlCondition struct { Filepaths []string `yaml:"filepaths"` } +type xmlPublicidCondition struct { + Regex string `yaml:"regex"` + Namespaces map[string]string `yaml:"namespaces"` + Filepaths []string `yaml:"filepaths"` +} + type jsonCondition struct { XPath string `yaml:'xpath'` Filepaths []string `yaml:"filepaths"` diff --git a/provider/internal/builtin/service_client.go b/provider/internal/builtin/service_client.go index f7a8a3d4..bd84052a 100644 --- a/provider/internal/builtin/service_client.go +++ b/provider/internal/builtin/service_client.go @@ -198,6 +198,79 @@ func (p *builtintServiceClient) Evaluate(cap string, conditionInfo []byte) (prov } } + return response, nil + case "xmlPublicid": + regex, err := regexp.Compile(cond.XMLPublicid.Regex) + if err != nil { + return response, fmt.Errorf("Could not parse provided public-id regex '%s': %v", cond.XMLPublicid.Regex, err) + } + query, err := xpath.CompileWithNS("//*[@public-id]", cond.XML.Namespaces) + if query == nil || err != nil { + return response, fmt.Errorf("Could not parse public-id xml query '%s': %v", cond.XML.XPath, err) + } + // TODO(fabianvf): how should we scope the files searched here? + var xmlFiles []string + patterns := []string{"*.xml", "*.xhtml"} + xmlFiles, err = provider.GetFiles(p.config.Location, cond.XML.Filepaths, patterns...) + if err != nil { + return response, fmt.Errorf("Unable to find files using pattern `%s`: %v", patterns, err) + } + + for _, file := range xmlFiles { + + f, err := os.Open(file) + if err != nil { + fmt.Printf("unable to open file '%s': %v\n", file, err) + continue + } + // TODO This should start working if/when this merges and releases: https://github.com/golang/go/pull/56848 + var doc *xmlquery.Node + doc, err = xmlquery.ParseWithOptions(f, xmlquery.ParserOptions{Decoder: &xmlquery.DecoderOptions{Strict: false}}) + if err != nil { + if err.Error() == "xml: unsupported version \"1.1\"; only version 1.0 is supported" { + // TODO HACK just pretend 1.1 xml documents are 1.0 for now while we wait for golang to support 1.1 + b, err := os.ReadFile(file) + if err != nil { + fmt.Printf("unable to parse xml file '%s': %v\n", file, err) + continue + } + docString := strings.Replace(string(b), " Date: Wed, 22 Nov 2023 14:34:52 +0100 Subject: [PATCH 2/6] Update XMLPublicID name Signed-off-by: Marek Aufart --- provider/internal/builtin/provider.go | 4 ++-- provider/internal/builtin/service_client.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/provider/internal/builtin/provider.go b/provider/internal/builtin/provider.go index 801a7d8c..9d0c308a 100644 --- a/provider/internal/builtin/provider.go +++ b/provider/internal/builtin/provider.go @@ -54,7 +54,7 @@ type builtinCondition struct { Filecontent fileContentCondition `yaml:"filecontent"` File fileCondition `yaml:"file"` XML xmlCondition `yaml:"xml"` - XMLPublicid xmlPublicidCondition `yaml:"xmlPublicid"` + XMLPublicID xmlPublicIDCondition `yaml:"xmlPublicID"` JSON jsonCondition `yaml:"json"` HasTags []string `yaml:"hasTags"` provider.ProviderContext `yaml:",inline"` @@ -77,7 +77,7 @@ type xmlCondition struct { Filepaths []string `yaml:"filepaths"` } -type xmlPublicidCondition struct { +type xmlPublicIDCondition struct { Regex string `yaml:"regex"` Namespaces map[string]string `yaml:"namespaces"` Filepaths []string `yaml:"filepaths"` diff --git a/provider/internal/builtin/service_client.go b/provider/internal/builtin/service_client.go index bd84052a..46b6632c 100644 --- a/provider/internal/builtin/service_client.go +++ b/provider/internal/builtin/service_client.go @@ -200,9 +200,9 @@ func (p *builtintServiceClient) Evaluate(cap string, conditionInfo []byte) (prov return response, nil case "xmlPublicid": - regex, err := regexp.Compile(cond.XMLPublicid.Regex) + regex, err := regexp.Compile(cond.XMLPublicID.Regex) if err != nil { - return response, fmt.Errorf("Could not parse provided public-id regex '%s': %v", cond.XMLPublicid.Regex, err) + return response, fmt.Errorf("Could not parse provided public-id regex '%s': %v", cond.XMLPublicID.Regex, err) } query, err := xpath.CompileWithNS("//*[@public-id]", cond.XML.Namespaces) if query == nil || err != nil { From 13ff6e349a479f4f7660bcd2e84aac266135df6b Mon Sep 17 00:00:00 2001 From: Marek Aufart Date: Wed, 22 Nov 2023 15:06:57 +0100 Subject: [PATCH 3/6] Separate XML files setup to methods Signed-off-by: Marek Aufart --- provider/internal/builtin/service_client.go | 118 +++++++++----------- 1 file changed, 52 insertions(+), 66 deletions(-) diff --git a/provider/internal/builtin/service_client.go b/provider/internal/builtin/service_client.go index 46b6632c..9d7af6f9 100644 --- a/provider/internal/builtin/service_client.go +++ b/provider/internal/builtin/service_client.go @@ -141,47 +141,19 @@ func (p *builtintServiceClient) Evaluate(cap string, conditionInfo []byte) (prov if query == nil || err != nil { return response, fmt.Errorf("Could not parse provided xpath query '%s': %v", cond.XML.XPath, err) } - //TODO(fabianvf): how should we scope the files searched here? - var xmlFiles []string - patterns := []string{"*.xml", "*.xhtml"} - xmlFiles, err = provider.GetFiles(p.config.Location, cond.XML.Filepaths, patterns...) + xmlFiles, err := findXMLFiles(p.config.Location, cond.XMLPublicID.Filepaths) if err != nil { - return response, fmt.Errorf("Unable to find files using pattern `%s`: %v", patterns, err) + return response, fmt.Errorf("Unable to find XML files: %v", err) } for _, file := range xmlFiles { - - f, err := os.Open(file) + nodes, err := queryXMLFile(file, query) if err != nil { - fmt.Printf("unable to open file '%s': %v\n", file, err) continue } - // TODO This should start working if/when this merges and releases: https://github.com/golang/go/pull/56848 - var doc *xmlquery.Node - doc, err = xmlquery.ParseWithOptions(f, xmlquery.ParserOptions{Decoder: &xmlquery.DecoderOptions{Strict: false}}) - if err != nil { - if err.Error() == "xml: unsupported version \"1.1\"; only version 1.0 is supported" { - // TODO HACK just pretend 1.1 xml documents are 1.0 for now while we wait for golang to support 1.1 - b, err := os.ReadFile(file) - if err != nil { - fmt.Printf("unable to parse xml file '%s': %v\n", file, err) - continue - } - docString := strings.Replace(string(b), " Date: Wed, 29 Nov 2023 10:36:18 +0100 Subject: [PATCH 4/6] Try add demo rule-example Signed-off-by: Marek Aufart --- rule-example.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rule-example.yaml b/rule-example.yaml index 1ce511a1..b817303d 100644 --- a/rule-example.yaml +++ b/rule-example.yaml @@ -173,6 +173,11 @@ builtin.xml: xpath: invalid-query: "test" +- message: "Found jboss in XML public-id for eap7/eap5" + ruleID: jboss-eap5-7-xml-02000 + when: + builtin.xmlPublicID: + regex: .*JBoss.+DTD Java EE.+5.* - message: "Tags {{tags}} found, creating message and new tag both" ruleID: multiple-actions-001 tag: From 488c3cb2dcea378bd808996efcd41b01d5e9fa54 Mon Sep 17 00:00:00 2001 From: Marek Aufart Date: Thu, 30 Nov 2023 10:49:34 +0100 Subject: [PATCH 5/6] Fix sample demo analysis Signed-off-by: Marek Aufart --- demo-output.yaml | 10 ++++++++++ examples/java/jboss-app.xml | 8 ++++++++ provider/internal/builtin/provider.go | 4 ++++ provider/internal/builtin/service_client.go | 2 +- rule-example.yaml | 2 +- 5 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 examples/java/jboss-app.xml diff --git a/demo-output.yaml b/demo-output.yaml index 12864c19..55460685 100644 --- a/demo-output.yaml +++ b/demo-output.yaml @@ -309,6 +309,16 @@ variables: name: junit.junit version: "4.11" + jboss-eap5-7-xml-02000: + description: "" + category: potential + incidents: + - uri: file:///analyzer-lsp/examples/java/jboss-app.xml + message: JBoss 5.x EAR descriptor (jboss-app.xml) was found with public-id + variables: + data: module + innerText: "\n jboss-example-service\n " + matchingXML: jboss-example-service lang-ref-001: description: "" category: potential diff --git a/examples/java/jboss-app.xml b/examples/java/jboss-app.xml new file mode 100644 index 00000000..57ab37f7 --- /dev/null +++ b/examples/java/jboss-app.xml @@ -0,0 +1,8 @@ + + + + + + jboss-example-service + + diff --git a/provider/internal/builtin/provider.go b/provider/internal/builtin/provider.go index 9c414935..3bb03814 100644 --- a/provider/internal/builtin/provider.go +++ b/provider/internal/builtin/provider.go @@ -40,6 +40,10 @@ var capabilities = []provider.Capability{ Name: "xml", TemplateContext: openapi3.SchemaRef{}, }, + { + Name: "xmlPublicID", + TemplateContext: openapi3.SchemaRef{}, + }, { Name: "json", TemplateContext: openapi3.SchemaRef{}, diff --git a/provider/internal/builtin/service_client.go b/provider/internal/builtin/service_client.go index 64ceafcc..6efe2c84 100644 --- a/provider/internal/builtin/service_client.go +++ b/provider/internal/builtin/service_client.go @@ -161,7 +161,7 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi } return response, nil - case "xmlPublicid": + case "xmlPublicID": regex, err := regexp.Compile(cond.XMLPublicID.Regex) if err != nil { return response, fmt.Errorf("Could not parse provided public-id regex '%s': %v", cond.XMLPublicID.Regex, err) diff --git a/rule-example.yaml b/rule-example.yaml index b817303d..8f4ceaea 100644 --- a/rule-example.yaml +++ b/rule-example.yaml @@ -173,7 +173,7 @@ builtin.xml: xpath: invalid-query: "test" -- message: "Found jboss in XML public-id for eap7/eap5" +- message: "JBoss 5.x EAR descriptor (jboss-app.xml) was found with public-id" ruleID: jboss-eap5-7-xml-02000 when: builtin.xmlPublicID: From 57d16ccb56f31047582b0f3c7a8c128c66c1181a Mon Sep 17 00:00:00 2001 From: Marek Aufart Date: Wed, 6 Dec 2023 11:05:11 +0100 Subject: [PATCH 6/6] Remove naked returns Signed-off-by: Marek Aufart --- provider/internal/builtin/service_client.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/provider/internal/builtin/service_client.go b/provider/internal/builtin/service_client.go index 7548f23d..e9fa2f3c 100644 --- a/provider/internal/builtin/service_client.go +++ b/provider/internal/builtin/service_client.go @@ -392,18 +392,18 @@ func findFilesMatchingPattern(root, pattern string) ([]string, error) { return matches, err } -func findXMLFiles(baseLocation string, filePaths []string) (xmlFiles []string, err error) { +func findXMLFiles(baseLocation string, filePaths []string) ([]string, error) { patterns := []string{"*.xml", "*.xhtml"} // TODO(fabianvf): how should we scope the files searched here? - xmlFiles, err = provider.GetFiles(baseLocation, filePaths, patterns...) - return + xmlFiles, err := provider.GetFiles(baseLocation, filePaths, patterns...) + return xmlFiles, err } -func queryXMLFile(filePath string, query *xpath.Expr) (nodes []*xmlquery.Node, err error) { +func queryXMLFile(filePath string, query *xpath.Expr) ([]*xmlquery.Node, error) { f, err := os.Open(filePath) if err != nil { fmt.Printf("unable to open file '%s': %v\n", filePath, err) - return + return nil, err } defer f.Close() // TODO This should start working if/when this merges and releases: https://github.com/golang/go/pull/56848 @@ -416,19 +416,19 @@ func queryXMLFile(filePath string, query *xpath.Expr) (nodes []*xmlquery.Node, e b, err = os.ReadFile(filePath) if err != nil { fmt.Printf("unable to parse xml file '%s': %v\n", filePath, err) - return + return nil, err } docString := strings.Replace(string(b), "