Skip to content

Commit

Permalink
rollup: split rules for plain/tagged metrics (for perfomance)
Browse files Browse the repository at this point in the history
  • Loading branch information
@msaf1980
msaf1980 committed May 26, 2021
1 parent bbb3a62 commit 86d8d55
Show file tree
Hide file tree
Showing 10 changed files with 909 additions and 56 deletions.
39 changes: 20 additions & 19 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,24 +129,25 @@ var knownDataTableContext = map[string]bool{

// DataTable configs
type DataTable struct {
Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"`
TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"`
RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
ContextMap map[string]bool `toml:"-" json:"-"`
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"`
TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"`
RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
RollupRuleTypeAutodetect bool `toml:"rollup-rule-type-autodetect" json:"rollup-rule-type-autodetect"`
Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
ContextMap map[string]bool `toml:"-" json:"-"`
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
}

// Debug config
Expand Down Expand Up @@ -453,7 +454,7 @@ func (c *Config) ProcessDataTables() (err error) {
} else if c.DataTable[i].RollupConf == "none" {
c.DataTable[i].Rollup, err = rollup.NewDefault(rdp, rdf)
} else {
c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf)
c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf, c.DataTable[i].RollupRuleTypeAutodetect)
}

if err != nil {
Expand Down
29 changes: 27 additions & 2 deletions helper/rollup/compact.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@ compact form of rollup rules for tests
regexp;function;age:precision,age:precision,...
*/

func parseCompact(body string) (*Rules, error) {
func parseCompact(body string, auto bool) (*Rules, error) {
lines := strings.Split(body, "\n")
patterns := make([]Pattern, 0)

for _, line := range lines {
var ruleType RuleType
if strings.TrimSpace(line) == "" {
continue
}
Expand All @@ -29,6 +30,30 @@ func parseCompact(body string) (*Rules, error) {
return nil, fmt.Errorf("can't parse line: %#v", line)
}
regexp := strings.TrimSpace(line[:p1])
if len(regexp) > 8 && regexp[0] == '<' && regexp[1] == '!' && regexp[7] == '>' {
typeStr := regexp[1:7]
switch typeStr {
case "!ALL_T":
ruleType = RuleAll
case "!PLAIN":
ruleType = RulePlain
case "!TAG_R":
ruleType = RuleTaggedRegex
//case "!TAG_T":
// ruleType = RuleTagged
default:
return nil, fmt.Errorf("not realised rule type for line: %#v", line)
}
regexp = regexp[8:]
} else {
if ruleType == RuleAuto {
if auto && len(regexp) > 0 {
ruleType = AutoDetectRuleType(regexp)
} else {
ruleType = RuleAll
}
}
}
function := strings.TrimSpace(line[p1+1 : p2])
retention := make([]Retention, 0)

Expand All @@ -55,7 +80,7 @@ func parseCompact(body string) (*Rules, error) {
}
}

patterns = append(patterns, Pattern{Regexp: regexp, Function: function, Retention: retention})
patterns = append(patterns, Pattern{RuleType: ruleType, Regexp: regexp, Function: function, Retention: retention})
}

return (&Rules{Pattern: patterns}).compile()
Expand Down
183 changes: 179 additions & 4 deletions helper/rollup/compact_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,186 @@ import (
func TestParseCompact(t *testing.T) {
config := `
click_cost;any;0:3600,86400:60
;max;0:60,3600:300,86400:3600`
<!PLAIN>\.max$;max;0:3600,86400:60
<!TAG_R>\.max\?;max;0:3600
\.min$;min;0:3600,86400:60
\.min\?;min;0:3600
env=cloud;avg;0:3600
;avg;0:60,3600:300,86400:3600`

expected, _ := (&Rules{
Pattern: []Pattern{
Pattern{Regexp: "click_cost", Function: "any", Retention: []Retention{
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{Regexp: "", Function: "max", Retention: []Retention{
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

expectedPlain, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

expectedTagged, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

assert := assert.New(t)
r, err := parseCompact(config, false)
assert.NoError(err)
assert.Equal(expected, r)

assert.Equal(len(expected.patternPlain), 6)
assert.Equal(expectedPlain.Pattern, r.patternPlain)

assert.Equal(len(expected.patternTagged), 6)
assert.Equal(expectedTagged.Pattern, r.patternTagged)
}

func TestParseCompactAutoDetect(t *testing.T) {
config := `
click_cost;any;0:3600,86400:60
<!PLAIN>\.max$;max;0:3600,86400:60
<!TAG_R>\.max\?;max;0:3600
\.min$;min;0:3600,86400:60
\.min\?;min;0:3600
env=cloud;avg;0:3600
;avg;0:60,3600:300,86400:3600`

expected, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

expectedPlain, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
Retention{Age: 86400, Precision: 60},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
}},
},
}).compile()

expectedTagged, _ := (&Rules{
Pattern: []Pattern{
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 3600},
}},
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
Retention{Age: 0, Precision: 60},
Retention{Age: 3600, Precision: 300},
Retention{Age: 86400, Precision: 3600},
Expand All @@ -26,7 +197,11 @@ func TestParseCompact(t *testing.T) {
}).compile()

assert := assert.New(t)
r, err := parseCompact(config)
r, err := parseCompact(config, true)
assert.NoError(err)
assert.Equal(expected, r)

assert.Equal(expectedPlain.Pattern, r.patternPlain)

assert.Equal(expectedTagged.Pattern, r.patternTagged)
}
16 changes: 11 additions & 5 deletions helper/rollup/remote.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ import (
)

type rollupRulesResponseRecord struct {
Regexp string `json:"regexp"`
Function string `json:"function"`
Age string `json:"age"`
Precision string `json:"precision"`
IsDefault int `json:"is_default"`
RuleType RuleType `json:"type"`
Regexp string `json:"regexp"`
Function string `json:"function"`
Age string `json:"age"`
Precision string `json:"precision"`
IsDefault int `json:"is_default"`
}
type rollupRulesResponse struct {
Data []rollupRulesResponseRecord `json:"data"`
Expand Down Expand Up @@ -74,7 +75,11 @@ func parseJson(body []byte) (*Rules, error) {
}
} else {
if last() == nil || last().Regexp != d.Regexp || last().Function != d.Function {
if d.RuleType == RuleAuto {
d.RuleType = RuleAll // for remote rules - no auto-detect rule type
}
r.Pattern = append(r.Pattern, Pattern{
RuleType: d.RuleType,
Retention: make([]Retention, 0),
Regexp: d.Regexp,
Function: d.Function,
Expand All @@ -92,6 +97,7 @@ func parseJson(body []byte) (*Rules, error) {

if defaultFunction != "" || len(defaultRetention) != 0 {
r.Pattern = append(r.Pattern, Pattern{
RuleType: RuleAll,
Regexp: "",
Function: defaultFunction,
Retention: defaultRetention,
Expand Down
24 changes: 21 additions & 3 deletions helper/rollup/remote_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,25 @@ func TestParseJson(t *testing.T) {
"precision": "0",
"is_default": 0
},
{
"type": "plain",
"regexp": "\\.min$",
"function": "min",
"age": "0",
"precision": "3600",
"is_default": 0
},
{
"type": "tagged_regex",
"regexp": "\\.min\\?",
"function": "min",
"age": "0",
"precision": "3600",
"is_default": 0
},
{
"regexp": "",
"function": "max",
"function": "avg",
"age": "0",
"precision": "60",
"is_default": 1
Expand All @@ -112,11 +128,13 @@ func TestParseJson(t *testing.T) {
total$;sum;
min$;min;
max$;max;
;max;0:60
<!PLAIN>\.min$;min;0:3600
<!TAG_R>\.min\?;min;0:3600
;avg;0:60
`

assert := assert.New(t)
expected, err := parseCompact(compact)
expected, err := parseCompact(compact, false)
assert.NoError(err)

r, err := parseJson([]byte(response))
Expand Down
Loading

0 comments on commit 86d8d55

Please sign in to comment.