Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Quest for distributed #58

Merged
merged 5 commits into from
Apr 23, 2024
Merged

Quest for distributed #58

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
167a353
quest for distributed/standalone deployments of parseable
nikhilsinhaparseable Apr 22, 2024
9f52596
changes for initialising ingestor client for distributed
nikhilsinhaparseable Apr 22, 2024
aeb847d
fixed quest test for distributed
nikhilsinhaparseable Apr 22, 2024
67c474c
integrity test updated for distributed
nikhilsinhaparseable Apr 22, 2024
219da6e
fix in integrity test and quest test files for distributed and standa…
nikhilsinhaparseable Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 21 additions & 9 deletions integrity_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,21 +139,33 @@ func TestIntegrity(t *testing.T) {
require.Equal(t, actualFlog, expectedFlog)
}

DeleteStream(t, NewGlob.Client, NewGlob.Stream)
DeleteStream(t, NewGlob.QueryClient, NewGlob.Stream)
}

func ingestFlogs(flogs []Flog, stream string) error {
payload, _ := json.Marshal(flogs)
if NewGlob.IngestorUrl.String() == "" {
req, _ := NewGlob.QueryClient.NewRequest(http.MethodPost, "ingest", bytes.NewBuffer(payload))
req.Header.Add("X-P-Stream", stream)
response, err := NewGlob.QueryClient.Do(req)
if err != nil {
return err
}

req, _ := NewGlob.Client.NewRequest(http.MethodPost, "ingest", bytes.NewBuffer(payload))
req.Header.Add("X-P-Stream", stream)
response, err := NewGlob.Client.Do(req)
if err != nil {
return err
}
if response.StatusCode != http.StatusOK {
return fmt.Errorf("couldn't ingest logs, status code = %d", response.StatusCode)
}
} else {
req, _ := NewGlob.IngestorClient.NewRequest(http.MethodPost, "ingest", bytes.NewBuffer(payload))
req.Header.Add("X-P-Stream", stream)
response, err := NewGlob.QueryClient.Do(req)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be IngestorClient here?

if err != nil {
return err
}

if response.StatusCode != http.StatusOK {
return fmt.Errorf("couldn't ingest logs, status code = %d", response.StatusCode)
if response.StatusCode != http.StatusOK {
return fmt.Errorf("couldn't ingest logs, status code = %d", response.StatusCode)
}
}

return nil
Expand Down
98 changes: 70 additions & 28 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,16 @@ func main() {
}

type Glob struct {
Url url.URL
Username string
Password string
Stream string
Client HTTPClient
Mode string
QueryUrl url.URL
QueryUsername string
QueryPassword string
IngestorUrl url.URL
IngestorUsername string
IngestorPassword string
Stream string
QueryClient HTTPClient
IngestorClient HTTPClient
Mode string
MinIoConfig
}

Expand All @@ -45,9 +49,14 @@ type MinIoConfig struct {

var NewGlob = func() Glob {
testing.Init()
var targetUrl string
var username string
var password string
var targetQueryUrl string
var queryUsername string
var queryPassword string

var targetIngestorUrl string
var ingestorUsername string
var ingestorPassword string

var stream string
var mode string
// XXX
Expand All @@ -56,9 +65,14 @@ var NewGlob = func() Glob {
var minioPass string
var minioBucket string

flag.StringVar(&targetUrl, "url", "http://localhost:8000", "Specify url. Default is root")
flag.StringVar(&username, "user", "admin", "Specify username. Default is admin")
flag.StringVar(&password, "pass", "admin", "Specify pass. Default is admin")
flag.StringVar(&targetQueryUrl, "query-url", "http://localhost:8000", "Specify url. Default is root")
flag.StringVar(&queryUsername, "query-user", "admin", "Specify username. Default is admin")
flag.StringVar(&queryPassword, "query-pass", "admin", "Specify pass. Default is admin")

flag.StringVar(&targetIngestorUrl, "ingestor-url", "", "Specify url. Default is root")
flag.StringVar(&ingestorUsername, "ingestor-user", "admin", "Specify username. Default is admin")
flag.StringVar(&ingestorPassword, "ingestor-pass", "admin", "Specify pass. Default is admin")

flag.StringVar(&stream, "stream", "app", "Specify stream. Default is app")
flag.StringVar(&mode, "mode", "smoke", "Specify mode. Default is smoke")

Expand All @@ -69,25 +83,53 @@ var NewGlob = func() Glob {

flag.Parse()

parsedTargetUrl, err := url.Parse(targetUrl)
parsedQueryTargetUrl, err := url.Parse(targetQueryUrl)
if err != nil {
panic("Could not parse url")
}

client := DefaultClient(*parsedTargetUrl, username, password)

return Glob{
Url: *parsedTargetUrl,
Username: username,
Password: password,
Stream: stream,
Client: client,
Mode: mode,
MinIoConfig: MinIoConfig{
Url: minioUrl,
User: minioUser,
Pass: minioPass,
Bucket: minioBucket,
},
queryClient := DefaultClient(*parsedQueryTargetUrl, queryUsername, queryPassword)

if targetIngestorUrl != "" {
parsedIngestorTargetUrl, err := url.Parse(targetIngestorUrl)
if err != nil {
panic("Could not parse url")
}

ingestorClient := DefaultClient(*parsedIngestorTargetUrl, ingestorUsername, ingestorPassword)
return Glob{
QueryUrl: *parsedQueryTargetUrl,
QueryUsername: queryUsername,
QueryPassword: queryPassword,
QueryClient: queryClient,
IngestorUrl: *parsedIngestorTargetUrl,
IngestorUsername: ingestorUsername,
IngestorPassword: ingestorPassword,
IngestorClient: ingestorClient,
Stream: stream,
Mode: mode,
MinIoConfig: MinIoConfig{
Url: minioUrl,
User: minioUser,
Pass: minioPass,
Bucket: minioBucket,
},
}
} else {
return Glob{
QueryUrl: *parsedQueryTargetUrl,
QueryUsername: queryUsername,
QueryPassword: queryPassword,
QueryClient: queryClient,
Stream: stream,
Mode: mode,
MinIoConfig: MinIoConfig{
Url: minioUrl,
User: minioUser,
Pass: minioPass,
Bucket: minioBucket,
},
}
}

}()
6 changes: 4 additions & 2 deletions main.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,13 @@ minio_secret_key=${10}
: "${minio_secret_key:="minioadmin"}"
minio_bucket=${11}
: "${minio_bucket:="parseable"}"

ingestor_endpoint=${12}
ingestor_username=${13}
ingestor_password=${14}
stream_name=$(head /dev/urandom | tr -dc a-z | head -c10)

run () {
./quest.test -test.v -mode="$mode" -url="$endpoint" -stream="$stream_name" -user="$username" -pass="$password" -minio-url="$minio_url" -minio-user="$minio_access_key" -minio-pass="$minio_secret_key" -minio-bucket="$minio_bucket"
./quest.test -test.v -mode="$mode" -query-url="$endpoint" -stream="$stream_name" -query-user="$username" -query-pass="$password" -minio-url="$minio_url" -minio-user="$minio_access_key" -minio-pass="$minio_secret_key" -minio-bucket="$minio_bucket" -ingestor-url="$ingestor_endpoint" -ingestor-user="$ingestor_username" -ingestor-pass="$ingestor_password"
return $?
}

Expand Down
Loading
Loading