Skip to content

Commit

Permalink
[query] Add Graphite support for ** with metric path selectors (#3020)
Browse files Browse the repository at this point in the history
  • Loading branch information
@robskillington
robskillington authored Dec 29, 2020
1 parent 59c9667 commit 468ab1c
Show file tree
Hide file tree
Showing 7 changed files with 90 additions and 21 deletions.
0 ...ntegration-tests/carbon/expected/a.b.json → ...integration-tests/carbon/expected/ab.json
View file
File renamed without changes.
0 ...tegration-tests/carbon/expected/a.ba.json → ...ntegration-tests/carbon/expected/aba.json
View file
File renamed without changes.
0 ...egration-tests/carbon/expected/a.b.c.json → ...ntegration-tests/carbon/expected/abc.json
View file
File renamed without changes.
0 ...ration-tests/carbon/expected/a.b.c.d.json → ...tegration-tests/carbon/expected/abcd.json
View file
File renamed without changes.
56 changes: 35 additions & 21 deletions scripts/docker-integration-tests/carbon/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ t=$(date +%s)
echo "foo.min.aggregate.baz 41 $t" | nc 0.0.0.0 7204
echo "foo.min.aggregate.baz 42 $t" | nc 0.0.0.0 7204
echo "Attempting to read min aggregated carbon metric"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.aggregate.baz 41
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.aggregate.baz' 41"

echo "Writing out a carbon metric that should not be aggregated"
t=$(date +%s)
Expand All @@ -64,25 +64,39 @@ t=$(date +%s)
echo "foo.min.already-aggregated.baz 42 $t" | nc 0.0.0.0 7204
echo "foo.min.already-aggregated.baz 43 $t" | nc 0.0.0.0 7204
echo "Attempting to read unaggregated carbon metric"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.already-aggregated.baz 43
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.already-aggregated.baz' 43"

echo "Writing out a carbon metric that should should use the default mean aggregation"
t=$(date +%s)
# Mean of 10 and 20 is 15. Same comment as the min aggregation test above.
echo "foo.min.catch-all.baz 10 $t" | nc 0.0.0.0 7204
echo "foo.min.catch-all.baz 20 $t" | nc 0.0.0.0 7204
echo "Attempting to read mean aggregated carbon metric"
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon foo.min.catch-all.baz 15
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.catch-all.baz' 15"

# Test writing and reading IDs with colons in them.
t=$(date +%s)
echo "foo.bar:baz.qux 42 $t" | nc 0.0.0.0 7204
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon 'foo.bar:*.*' 42
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.bar:*.*' 42"

# Test writing and reading IDs with a single element.
t=$(date +%s)
echo "quail 42 $t" | nc 0.0.0.0 7204
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff read_carbon 'quail' 42
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'quail' 42"

# Test using "**" in queries
t=$(date +%s)
echo "qux.pos1-a.pos2-0 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-a.pos2-1 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-b.pos2-0 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-b.pos2-1 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-c.pos2-0 1 $t" | nc 0.0.0.0 7204
echo "qux.pos1-c.pos2-1 1 $t" | nc 0.0.0.0 7204
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(qux**)' 6"
ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(qux.pos1-a**)' 2"
ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos1-a**)' 2"
ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos2-1**)' 3"
ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos2-1)' 3"

t=$(date +%s)

Expand All @@ -107,19 +121,19 @@ echo "g.bar.baz.. 0 $t" | nc 0.0.0.0 7204
echo "h.bar@@baz 0 $t" | nc 0.0.0.0 7204
echo "i.bar!!baz 0 $t" | nc 0.0.0.0 7204

ATTEMPTS=10 TIMEOUT=1 retry_with_backoff find_carbon a* a.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b* a.b.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.ba[rg] a.ba.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b*.c* a.b.c.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.b*.caw.* a.b.c.d.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon x none.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon a.d none.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon *.*.*.*.* none.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon c:* cbar.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon c:bar.* cbaz.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon d.bar.* dbaz.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon e.bar.* ebaz.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon f.bar.* fbaz.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon g.bar.* gbaz.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon h.bar* hbarbaz.json
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff find_carbon i.bar* ibarbaz.json
ATTEMPTS=10 TIMEOUT=1 retry_with_backoff "find_carbon 'a*' a.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*' ab.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.ba[rg]' aba.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*.c*' abc.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*.caw.*' abcd.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'x' none.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.d' none.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon '*.*.*.*.*' none.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'c:*' cbar.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'c:bar.*' cbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'd.bar.*' dbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'e.bar.*' ebaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'f.bar.*' fbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'g.bar.*' gbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'h.bar*' hbarbaz.json"
ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'i.bar*' ibarbaz.json"
27 changes: 27 additions & 0 deletions src/query/graphite/storage/m3_wrapper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,11 @@ import (
"errors"
"fmt"
"math"
"strings"
"sync"
"time"

"github.com/m3db/m3/src/m3ninx/doc"
"github.com/m3db/m3/src/query/block"
xctx "github.com/m3db/m3/src/query/graphite/context"
"github.com/m3db/m3/src/query/graphite/graphite"
Expand Down Expand Up @@ -94,6 +96,31 @@ func NewM3WrappedStorage(
func TranslateQueryToMatchersWithTerminator(
query string,
) (models.Matchers, error) {
if strings.Contains(query, "**") {
// First add matcher to ensure it's a graphite metric with __g0__ tag.
hasFirstPathMatcher, err := convertMetricPartToMatcher(0, wildcard)
if err != nil {
return nil, err
}
// Need to regexp on the entire ID since ** matches over different
// graphite path dimensions.
globOpts := graphite.GlobOptions{
AllowMatchAll: true,
}
idRegexp, _, err := graphite.ExtendedGlobToRegexPattern(query, globOpts)
if err != nil {
return nil, err
}
return models.Matchers{
hasFirstPathMatcher,
models.Matcher{
Type: models.MatchRegexp,
Name: doc.IDReservedFieldName,
Value: idRegexp,
},
}, nil
}

metricLength := graphite.CountMetricParts(query)
// Add space for a terminator character.
matchersLength := metricLength + 1
Expand Down
28 changes: 28 additions & 0 deletions src/query/graphite/storage/m3_wrapper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"testing"
"time"

"github.com/m3db/m3/src/m3ninx/doc"
"github.com/m3db/m3/src/query/block"
xctx "github.com/m3db/m3/src/query/graphite/context"
"github.com/m3db/m3/src/query/graphite/graphite"
Expand Down Expand Up @@ -81,6 +82,33 @@ func TestTranslateQuery(t *testing.T) {
assert.Equal(t, expected, matchers)
}

func TestTranslateQueryStarStar(t *testing.T) {
query := `foo**bar`
end := time.Now()
start := end.Add(time.Hour * -2)
opts := FetchOptions{
StartTime: start,
EndTime: end,
DataOptions: DataOptions{
Timeout: time.Minute,
},
}

translated, err := translateQuery(query, opts, M3WrappedStorageOptions{})
assert.NoError(t, err)
assert.Equal(t, end, translated.End)
assert.Equal(t, start, translated.Start)
assert.Equal(t, time.Duration(0), translated.Interval)
assert.Equal(t, query, translated.Raw)
matchers := translated.TagMatchers
expected := models.Matchers{
{Type: models.MatchRegexp, Name: graphite.TagName(0), Value: []byte(".*")},
{Type: models.MatchRegexp, Name: doc.IDReservedFieldName, Value: []byte("foo.*bar")},
}

assert.Equal(t, expected, matchers)
}

func TestTranslateQueryTrailingDot(t *testing.T) {
query := `foo.`
end := time.Now()
Expand Down

0 comments on commit 468ab1c

Please sign in to comment.