From 6658fe29d8476ee39d9abcf42904ee3b5a343021 Mon Sep 17 00:00:00 2001 From: "haoran.yanghr" Date: Fri, 31 Mar 2017 20:02:01 +0800 Subject: [PATCH 1/5] 1. Implement the Create/DeleteProject. 2. Use v0.6.0 API. 3. Add the request id in Error struct. --- config.go | 6 ++-- example/log_config_sample.go | 2 ++ log_project.go | 68 +++++++++++------------------------- log_store.go | 24 ++++++------- machine_group.go | 2 +- request.go | 40 +++++++++++++-------- signature.go | 2 +- signature_test.go | 16 ++++----- 8 files changed, 73 insertions(+), 87 deletions(-) diff --git a/config.go b/config.go index 2d196d68..0152040c 100644 --- a/config.go +++ b/config.go @@ -1,7 +1,7 @@ package sls const ( - version = "0.5.0" // SDK version + version = "0.6.0" // SDK version signatureMethod = "hmac-sha1" // Signature method // OffsetNewest stands for the log head offset, i.e. the offset that will be @@ -12,8 +12,8 @@ const ( OffsetOldest = "begin" // ProgressHeader stands for the progress header in GetLogs response - ProgressHeader = "X-Sls-Progress" + ProgressHeader = "X-Log-Progress" // GetLogsCountHeader stands for the count header in GetLogs response - GetLogsCountHeader = "X-Sls-Count" + GetLogsCountHeader = "X-Log-Count" ) diff --git a/example/log_config_sample.go b/example/log_config_sample.go index d5c27d7a..1c3f5a22 100644 --- a/example/log_config_sample.go +++ b/example/log_config_sample.go @@ -1,5 +1,6 @@ package main +/* import ( "fmt" sls "github.com/galaxydi/go-loghub" @@ -119,3 +120,4 @@ func createConfig(configName string, projectName string, logstore string, servic } return nil } +*/ diff --git a/log_project.go b/log_project.go index b0fe2856..02aa732b 100644 --- a/log_project.go +++ b/log_project.go @@ -7,32 +7,6 @@ import ( "net/http" ) -// Error defines sls error -type Error struct { - Code string `json:"errorCode"` - Message string `json:"errorMessage"` -} - -// NewClientError new client error -func NewClientError(message string) *Error { - err := new(Error) - err.Code = "ClientError" - err.Message = message - return err -} - -func (e Error) String() string { - b, err := json.MarshalIndent(e, "", " ") - if err != nil { - return "" - } - return string(b) -} - -func (e Error) Error() string { - return e.String() -} - // LogProject defines log project type LogProject struct { Name string // Project name @@ -62,7 +36,7 @@ func (p *LogProject) WithToken(token string) (*LogProject, error) { // ListLogStore returns all logstore names of project p. func (p *LogProject) ListLogStore() ([]string, error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } uri := fmt.Sprintf("/logstores") @@ -91,7 +65,7 @@ func (p *LogProject) ListLogStore() ([]string, error) { // GetLogStore returns logstore according by logstore name. func (p *LogProject) GetLogStore(name string) (*LogStore, error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } r, err := request(p, "GET", "/logstores/"+name, h, nil) @@ -133,7 +107,7 @@ func (p *LogProject) CreateLogStore(name string, ttl, shardCnt int) error { } h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), "Content-Type": "application/json", "Accept-Encoding": "deflate", // TODO: support lz4 } @@ -155,7 +129,7 @@ func (p *LogProject) CreateLogStore(name string, ttl, shardCnt int) error { // DeleteLogStore deletes a logstore according by logstore name. func (p *LogProject) DeleteLogStore(name string) (err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } r, err := request(p, "DELETE", "/logstores/"+name, h, nil) @@ -191,7 +165,7 @@ func (p *LogProject) UpdateLogStore(name string, ttl, shardCnt int) (err error) } h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), "Content-Type": "application/json", "Accept-Encoding": "deflate", // TODO: support lz4 } @@ -213,7 +187,7 @@ func (p *LogProject) UpdateLogStore(name string, ttl, shardCnt int) (err error) // The offset starts from 0 and the size is the max number of machine groups could be returned. func (p *LogProject) ListMachineGroup(offset, size int) (m []string, total int, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } if size <= 0 { size = 500 @@ -246,7 +220,7 @@ func (p *LogProject) ListMachineGroup(offset, size int) (m []string, total int, // CheckMachineGroupExist check machine group exist or not func (p *LogProject) CheckMachineGroupExist(name string) (exist bool, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } r, err := request(p, "GET", "/machinegroups/"+name, h, nil) if err != nil { @@ -270,7 +244,7 @@ func (p *LogProject) CheckMachineGroupExist(name string) (exist bool, err error) // GetMachineGroup retruns machine group according by machine group name. func (p *LogProject) GetMachineGroup(name string) (m *MachineGroup, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } resp, err := request(p, "GET", "/machinegroups/"+name, h, nil) if err != nil { @@ -298,7 +272,7 @@ func (p *LogProject) CreateMachineGroup(m *MachineGroup) error { } h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), "Content-Type": "application/json", "Accept-Encoding": "deflate", // TODO: support lz4 } @@ -324,7 +298,7 @@ func (p *LogProject) UpdateMachineGroup(m *MachineGroup) (err error) { } h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), "Content-Type": "application/json", "Accept-Encoding": "deflate", // TODO: support lz4 } @@ -345,7 +319,7 @@ func (p *LogProject) UpdateMachineGroup(m *MachineGroup) (err error) { // DeleteMachineGroup deletes machine group according machine group name. func (p *LogProject) DeleteMachineGroup(name string) (err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } r, err := request(p, "DELETE", "/machinegroups/"+name, h, nil) if err != nil { @@ -366,7 +340,7 @@ func (p *LogProject) DeleteMachineGroup(name string) (err error) { // The offset starts from 0 and the size is the max number of configs could be returned. func (p *LogProject) ListConfig(offset, size int) (cfgNames []string, total int, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } if size <= 0 { size = 100 @@ -398,7 +372,7 @@ func (p *LogProject) ListConfig(offset, size int) (cfgNames []string, total int, // CheckConfigExist check config exist or not func (p *LogProject) CheckConfigExist(name string) (exist bool, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } r, err := request(p, "GET", "/configs/"+name, h, nil) if err != nil { @@ -423,7 +397,7 @@ func (p *LogProject) CheckConfigExist(name string) (exist bool, err error) { // GetConfig returns config according by config name. func (p *LogProject) GetConfig(name string) (c *LogConfig, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } r, err := request(p, "GET", "/configs/"+name, h, nil) if err != nil { @@ -451,7 +425,7 @@ func (p *LogProject) UpdateConfig(c *LogConfig) (err error) { } h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), "Content-Type": "application/json", "Accept-Encoding": "deflate", // TODO: support lz4 } @@ -477,7 +451,7 @@ func (p *LogProject) CreateConfig(c *LogConfig) (err error) { } h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), "Content-Type": "application/json", "Accept-Encoding": "deflate", // TODO: support lz4 } @@ -498,7 +472,7 @@ func (p *LogProject) CreateConfig(c *LogConfig) (err error) { // DeleteConfig deletes a config according by config name. func (p *LogProject) DeleteConfig(name string) (err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } r, err := request(p, "DELETE", "/configs/"+name, h, nil) if err != nil { @@ -518,7 +492,7 @@ func (p *LogProject) DeleteConfig(name string) (err error) { // GetAppliedMachineGroups returns applied machine group names list according config name. func (p *LogProject) GetAppliedMachineGroups(confName string) (groupNames []string, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } uri := fmt.Sprintf("/configs/%v/machinegroups", confName) r, err := request(p, "GET", uri, h, nil) @@ -546,7 +520,7 @@ func (p *LogProject) GetAppliedMachineGroups(confName string) (groupNames []stri // GetAppliedConfigs returns applied config names list according machine group name groupName. func (p *LogProject) GetAppliedConfigs(groupName string) (confNames []string, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } uri := fmt.Sprintf("/machinegroups/%v/configs", groupName) r, err := request(p, "GET", uri, h, nil) @@ -574,7 +548,7 @@ func (p *LogProject) GetAppliedConfigs(groupName string) (confNames []string, er // ApplyConfigToMachineGroup applies config to machine group. func (p *LogProject) ApplyConfigToMachineGroup(confName, groupName string) (err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } uri := fmt.Sprintf("/machinegroups/%v/configs/%v", groupName, confName) r, err := request(p, "PUT", uri, h, nil) @@ -595,7 +569,7 @@ func (p *LogProject) ApplyConfigToMachineGroup(confName, groupName string) (err // RemoveConfigFromMachineGroup removes config from machine group. func (p *LogProject) RemoveConfigFromMachineGroup(confName, groupName string) (err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } uri := fmt.Sprintf("/machinegroups/%v/configs/%v", groupName, confName) r, err := request(p, "DELETE", uri, h, nil) diff --git a/log_store.go b/log_store.go index 17082a2d..ad25997e 100644 --- a/log_store.go +++ b/log_store.go @@ -33,7 +33,7 @@ type Shard struct { // ListShards returns shard id list of this logstore. func (s *LogStore) ListShards() (shardIDs []int, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } uri := fmt.Sprintf("/logstores/%v/shards", s.Name) r, err := request(s.project, "GET", uri, h, nil) @@ -76,8 +76,8 @@ func (s *LogStore) PutLogs(lg *LogGroup) (err error) { } h := map[string]string{ - "x-sls-compresstype": "lz4", - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), + "x-log-compresstype": "lz4", + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), "Content-Type": "application/x-protobuf", } @@ -101,7 +101,7 @@ func (s *LogStore) PutLogs(lg *LogGroup) (err error) { // For more detail please read: http://gitlab.alibaba-inc.com/sls/doc/blob/master/api/shard.md#logstore func (s *LogStore) GetCursor(shardID int, from string) (cursor string, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } uri := fmt.Sprintf("/logstores/%v/shards/%v?type=cursor&from=%v", s.Name, shardID, from) @@ -149,7 +149,7 @@ func (s *LogStore) GetCursor(shardID int, from string) (cursor string, err error func (s *LogStore) GetLogsBytes(shardID int, cursor, endCursor string, logGroupMaxCount int) (out []byte, nextCursor string, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", "Accept": "application/x-protobuf", "Accept-Encoding": "lz4", } @@ -181,9 +181,9 @@ func (s *LogStore) GetLogsBytes(shardID int, cursor, endCursor string, err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) return } - v, ok := r.Header["X-Sls-Compresstype"] + v, ok := r.Header["X-Log-Compresstype"] if !ok || len(v) == 0 { - err = fmt.Errorf("can't find 'x-sls-compresstype' header") + err = fmt.Errorf("can't find 'x-log-compresstype' header") return } if v[0] != "lz4" { @@ -191,16 +191,16 @@ func (s *LogStore) GetLogsBytes(shardID int, cursor, endCursor string, return } - v, ok = r.Header["X-Sls-Cursor"] + v, ok = r.Header["X-Log-Cursor"] if !ok || len(v) == 0 { - err = fmt.Errorf("can't find 'x-sls-cursor' header") + err = fmt.Errorf("can't find 'x-log-cursor' header") return } nextCursor = v[0] - v, ok = r.Header["X-Sls-Bodyrawsize"] + v, ok = r.Header["X-Log-Bodyrawsize"] if !ok || len(v) == 0 { - err = fmt.Errorf("can't find 'x-sls-bodyrawsize' header") + err = fmt.Errorf("can't find 'x-log-bodyrawsize' header") return } bodyRawSize, err := strconv.Atoi(v[0]) @@ -253,7 +253,7 @@ func (s *LogStore) GetLogs(topic string, from int64, to int64, queryExp string, maxLineNum int64, offset int64, reverse bool) (*GetLogsResponse, error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", "Accept": "application/json", } diff --git a/machine_group.go b/machine_group.go index ff7e07c9..1c96b28c 100644 --- a/machine_group.go +++ b/machine_group.go @@ -47,7 +47,7 @@ type MachineList struct { // ListMachines returns machine list of this machine group. func (m *MachineGroup) ListMachines() (ms []*Machine, total int, err error) { h := map[string]string{ - "x-sls-bodyrawsize": "0", + "x-log-bodyrawsize": "0", } uri := fmt.Sprintf("/machinegroups/%v/machines", m.Name) diff --git a/request.go b/request.go index 653f8499..1b6f381e 100644 --- a/request.go +++ b/request.go @@ -7,24 +7,26 @@ import ( "net/http" "net/http/httputil" + "encoding/json" + "io/ioutil" + "github.com/golang/glog" ) // request sends a request to SLS. func request(project *LogProject, method, uri string, headers map[string]string, - body []byte) (resp *http.Response, err error) { + body []byte) (*http.Response, error) { - // The caller should provide 'x-sls-bodyrawsize' header - if _, ok := headers["x-sls-bodyrawsize"]; !ok { - err = fmt.Errorf("Can't find 'x-sls-bodyrawsize' header") - return + // The caller should provide 'x-log-bodyrawsize' header + if _, ok := headers["x-log-bodyrawsize"]; !ok { + return nil, fmt.Errorf("Can't find 'x-log-bodyrawsize' header") } // SLS public request headers headers["Host"] = project.Name + "." + project.Endpoint headers["Date"] = nowRFC1123() - headers["x-sls-apiversion"] = version - headers["x-sls-signaturemethod"] = signatureMethod + headers["x-log-apiversion"] = version + headers["x-log-signaturemethod"] = signatureMethod // Access with token if project.SessionToken != "" { @@ -35,8 +37,7 @@ func request(project *LogProject, method, uri string, headers map[string]string, bodyMD5 := fmt.Sprintf("%X", md5.Sum(body)) headers["Content-MD5"] = bodyMD5 if _, ok := headers["Content-Type"]; !ok { - err = fmt.Errorf("Can't find 'Content-Type' header") - return + return nil, fmt.Errorf("Can't find 'Content-Type' header") } } @@ -44,17 +45,17 @@ func request(project *LogProject, method, uri string, headers map[string]string, // Authorization = "SLS :" digest, err := signature(project, method, uri, headers) if err != nil { - return + return nil, err } auth := fmt.Sprintf("SLS %v:%v", project.AccessKeyID, digest) headers["Authorization"] = auth // Initialize http request reader := bytes.NewReader(body) - urlStr := fmt.Sprintf("http://%v.%v%v", project.Name, project.Endpoint, uri) + urlStr := fmt.Sprintf("https://%v.%v%v", project.Name, project.Endpoint, uri) req, err := http.NewRequest(method, urlStr, reader) if err != nil { - return + return nil, err } for k, v := range headers { req.Header.Add(k, v) @@ -69,9 +70,18 @@ func request(project *LogProject, method, uri string, headers map[string]string, } // Get ready to do request - resp, err = http.DefaultClient.Do(req) + resp, err := http.DefaultClient.Do(req) if err != nil { - return + return nil, err + } + + // Parse the sls error from body. + buf, _ := ioutil.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + err := &Error{} + json.Unmarshal(buf, err) + err.RequestID = resp.Header.Get("x-log-requestid") + return nil, err } if glog.V(1) { @@ -81,5 +91,5 @@ func request(project *LogProject, method, uri string, headers map[string]string, } glog.Infof("HTTP Response:\n%v", string(dump)) } - return + return resp, nil } diff --git a/signature.go b/signature.go index 012cabf0..4d17a736 100644 --- a/signature.go +++ b/signature.go @@ -44,7 +44,7 @@ func signature(project *LogProject, method, uri string, slsHeaders := make(map[string]string, len(headers)) for k, v := range headers { l := strings.TrimSpace(strings.ToLower(k)) - if strings.HasPrefix(l, "x-sls-") || strings.HasPrefix(l, "x-acs-") { + if strings.HasPrefix(l, "x-log-") || strings.HasPrefix(l, "x-acs-") { slsHeaders[l] = strings.TrimSpace(v) slsHeaderKeys = append(slsHeaderKeys, l) } diff --git a/signature_test.go b/signature_test.go index 01a9cfcd..dc523e2b 100644 --- a/signature_test.go +++ b/signature_test.go @@ -19,12 +19,12 @@ var project = &LogProject{ func TestSignatureGet(t *testing.T) { defer glog.Flush() h := map[string]string{ - "x-sls-apiversion": "0.4.0", - "x-sls-signaturemethod": "hmac-sha1", - "x-sls-bodyrawsize": "0", + "x-log-apiversion": "0.6.0", + "x-log-signaturemethod": "hmac-sha1", + "x-log-bodyrawsize": "0", "Date": "Mon, 3 Jan 2010 08:33:47 GMT", } - digest := "3DGtV4yTxzzqTCxfXTPGKWmHX8M=" + digest := "Rwm6cTKzoti4HWoe+GKcb6Kv07E=" s, err := signature(project, "GET", "/logstores", h) if err != nil { t.Fatal(err) @@ -76,16 +76,16 @@ func TestSignaturePost(t *testing.T) { t.Fatal(err) } h := map[string]string{ - "x-sls-apiversion": "0.4.0", - "x-sls-signaturemethod": "hmac-sha1", - "x-sls-bodyrawsize": "50", + "x-log-apiversion": "0.6.0", + "x-log-signaturemethod": "hmac-sha1", + "x-log-bodyrawsize": "50", "Content-MD5": md5Sum, "Content-Type": "application/x-protobuf", "Content-Length": "50", "Date": "Mon, 3 Jan 2010 08:33:47 GMT", } - digest := "WgfedxpxXG9q2r27d1ex/bHy+tY=" + digest := "87xQWqFaOSewqRIma8kPjGYlXHc=" s, err := signature(project, "GET", "/logstores/app_log", h) if err != nil { t.Fatal(err) From acbd86d031d73ab87f41f08dd330b6c7c0a0506f Mon Sep 17 00:00:00 2001 From: "haoran.yanghr" Date: Fri, 31 Mar 2017 20:04:40 +0800 Subject: [PATCH 2/5] Add the missing files --- client.go | 96 +++++++++++++++++++++++++++++++++++++++++++++++++ project_test.go | 33 +++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 client.go create mode 100644 project_test.go diff --git a/client.go b/client.go new file mode 100644 index 00000000..9743f6b0 --- /dev/null +++ b/client.go @@ -0,0 +1,96 @@ +package sls + +import ( + "encoding/json" + "fmt" +) + +// Error defines sls error +type Error struct { + Code string `json:"errorCode"` + Message string `json:"errorMessage"` + RequestID string `json:"requestID"` +} + +// NewClientError new client error +func NewClientError(message string) *Error { + err := new(Error) + err.Code = "ClientError" + err.Message = message + return err +} + +func (e Error) String() string { + b, err := json.MarshalIndent(e, "", " ") + if err != nil { + return "" + } + return string(b) +} + +func (e Error) Error() string { + return e.String() +} + +// Client ... +type Client struct { + Endpoint string // IP or hostname of SLS endpoint + AccessKeyID string + AccessKeySecret string + SessionToken string +} + +func convert(c *Client, projName string) *LogProject { + return &LogProject{ + Name: projName, + Endpoint: c.Endpoint, + AccessKeyID: c.AccessKeyID, + AccessKeySecret: c.AccessKeySecret, + SessionToken: c.SessionToken, + } +} + +// CreateProject create a new loghub project. +func (c *Client) CreateProject(name, description string) (*LogProject, error) { + type Body struct { + ProjectName string `json:"projectName"` + Description string `json:"description"` + } + body, err := json.Marshal(Body{ + ProjectName: name, + Description: description, + }) + if err != nil { + return nil, err + } + + h := map[string]string{ + "x-log-bodyrawsize": fmt.Sprintf("%d", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + uri := "/" + proj := convert(c, name) + _, err = request(proj, "POST", uri, h, body) + if err != nil { + return nil, err + } + + return proj, nil +} + +func (c *Client) DeleteProject(name string) error { + h := map[string]string{ + "x-log-bodyrawsize": "0", + } + + proj := convert(c, name) + uri := "/" + _, err := request(proj, "DELETE", uri, h, nil) + if err != nil { + return err + } + + return nil +} diff --git a/project_test.go b/project_test.go new file mode 100644 index 00000000..a6393826 --- /dev/null +++ b/project_test.go @@ -0,0 +1,33 @@ +package sls + +import ( + "fmt" + "testing" + "time" +) + +func TestCreateDeleteProject(t *testing.T) { + c := Client{ + Endpoint: "cn-shanghai.log.aliyuncs.com", + AccessKeyID: "LTAIJ4gv93WLYyA4", + AccessKeySecret: "xYv3JoZ8jeFmRIVzIiRUghEwVHnVdQ", + } + + for i := 0; i < 2; i++ { + var err error + name := fmt.Sprintf("faint%d", i) + _, err = c.CreateProject(name, "this is faint") + if err != nil { + fmt.Printf("Create project error: %v\n", err) + return + } + + time.Sleep(5 * time.Second) + + err = c.DeleteProject(name) + if err != nil { + fmt.Printf("Delete project error: %v\n", err) + return + } + } +} From cb1abe58305bf2f0f7c77f16e7b9078eb3d79a20 Mon Sep 17 00:00:00 2001 From: "haoran.yanghr" Date: Thu, 6 Apr 2017 06:51:13 +0800 Subject: [PATCH 3/5] Support creating index. Still ongoing. --- log_store.go | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++ model.go | 10 +++++ 2 files changed, 114 insertions(+) diff --git a/log_store.go b/log_store.go index ad25997e..7b6b7bcb 100644 --- a/log_store.go +++ b/log_store.go @@ -290,3 +290,107 @@ func (s *LogStore) GetLogs(topic string, from int64, to int64, queryExp string, return &getLogsResponse, nil } + +func (s *LogStore) CreateIndex() error { + type Index struct { + Name string `json:"logstoreName"` + TTL int `json:"ttl"` + ShardCount int `json:"shardCount"` + } + index := &Index{ + Name: name, + TTL: ttl, + ShardCount: shardCnt, + } + body, err := json.Marshal(index) + if err != nil { + return err + } + + h := map[string]string{ + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + uri := fmt.Sprintf("/logstores/%s/index", s.Name) + _, err = request(s.project, "POST", uri, h, body) + return err +} + +func (s *LogStore) UpdateIndex() error { + type Index struct { + Name string `json:"logstoreName"` + TTL int `json:"ttl"` + ShardCount int `json:"shardCount"` + } + index := &Index{ + Name: name, + TTL: ttl, + ShardCount: shardCnt, + } + body, err := json.Marshal(index) + if err != nil { + return err + } + + h := map[string]string{ + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + uri := fmt.Sprintf("/logstores/%s/index", s.Name) + _, err = request(s.project, "PUT", uri, h, body) + return err +} + +func (s *LogStore) DeleteIndex() error { + type Body struct { + project string `json:"projectName"` + store string `json:logstoreName` + } + + body, err := json.Marshal(Body{ + project: s.project.Name, + store: s.Name, + }) + if err != nil { + return err + } + + h := map[string]string{ + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + uri := fmt.Sprintf("/logstores/%s/index", s.Name) + _, err = request(s.project, "DELETE", uri, h, body) + return err +} + +func (s *LogStore) GetIndex() error { + type Body struct { + project string `json:"projectName"` + store string `json:logstoreName` + } + + body, err := json.Marshal(Body{ + project: s.project.Name, + store: s.Name, + }) + if err != nil { + return err + } + + h := map[string]string{ + "x-log-bodyrawsize": fmt.Sprintf("%v", len(body)), + "Content-Type": "application/json", + "Accept-Encoding": "deflate", // TODO: support lz4 + } + + uri := fmt.Sprintf("/logstores/%s/index", s.Name) + _, err = request(s.project, "GET", uri, h, body) + return err +} diff --git a/model.go b/model.go index 30b733bd..4d4c0154 100644 --- a/model.go +++ b/model.go @@ -6,3 +6,13 @@ type GetLogsResponse struct { Count int64 `json:"count"` Logs []map[string]string `json:"logs"` } + +// IndexKey ... +type IndexKey struct { + Tokens []string // tokens that split the log line. + CaseSensitive bool + Type string // text, long, double +} + +type Index struct { +} From b4ae30885cfc29eff3330aaed75a7bfb7205161c Mon Sep 17 00:00:00 2001 From: Haoran Yang Date: Fri, 7 Apr 2017 07:20:13 +0800 Subject: [PATCH 4/5] update --- client.go | 15 +++++++++++++++ log_project.go | 1 + log_store.go | 25 +++---------------------- model.go | 17 ++++++++++++++--- project_test.go | 38 +++++++++++++++++++++++++++++++++----- 5 files changed, 66 insertions(+), 30 deletions(-) diff --git a/client.go b/client.go index 9743f6b0..3e7b8425 100644 --- a/client.go +++ b/client.go @@ -80,6 +80,21 @@ func (c *Client) CreateProject(name, description string) (*LogProject, error) { return proj, nil } +func (c *Client) GetProject(name string) (*LogProject, error) { + h := map[string]string{ + "x-log-bodyrawsize": "0", + } + + uri := "/" + proj := convert(c, name) + _, err := request(proj, "GET", uri, h, nil) + if err != nil { + return nil, err + } + + return proj, nil +} + func (c *Client) DeleteProject(name string) error { h := map[string]string{ "x-log-bodyrawsize": "0", diff --git a/log_project.go b/log_project.go index 02aa732b..031b191d 100644 --- a/log_project.go +++ b/log_project.go @@ -82,6 +82,7 @@ func (p *LogProject) GetLogStore(name string) (*LogStore, error) { s := &LogStore{} json.Unmarshal(buf, s) + s.Name = name s.project = p return s, nil } diff --git a/log_store.go b/log_store.go index 7b6b7bcb..bf2dce7b 100644 --- a/log_store.go +++ b/log_store.go @@ -291,17 +291,7 @@ func (s *LogStore) GetLogs(topic string, from int64, to int64, queryExp string, return &getLogsResponse, nil } -func (s *LogStore) CreateIndex() error { - type Index struct { - Name string `json:"logstoreName"` - TTL int `json:"ttl"` - ShardCount int `json:"shardCount"` - } - index := &Index{ - Name: name, - TTL: ttl, - ShardCount: shardCnt, - } +func (s *LogStore) CreateIndex(index Index) error { body, err := json.Marshal(index) if err != nil { return err @@ -314,21 +304,12 @@ func (s *LogStore) CreateIndex() error { } uri := fmt.Sprintf("/logstores/%s/index", s.Name) + println(string(body)) _, err = request(s.project, "POST", uri, h, body) return err } -func (s *LogStore) UpdateIndex() error { - type Index struct { - Name string `json:"logstoreName"` - TTL int `json:"ttl"` - ShardCount int `json:"shardCount"` - } - index := &Index{ - Name: name, - TTL: ttl, - ShardCount: shardCnt, - } +func (s *LogStore) UpdateIndex(index Index) error { body, err := json.Marshal(index) if err != nil { return err diff --git a/model.go b/model.go index 4d4c0154..cfec09ad 100644 --- a/model.go +++ b/model.go @@ -9,10 +9,21 @@ type GetLogsResponse struct { // IndexKey ... type IndexKey struct { - Tokens []string // tokens that split the log line. - CaseSensitive bool - Type string // text, long, double + Token []string `json:"token"` // tokens that split the log line. + CaseSensitive bool `json:"caseSensitive"` + Type string `json:"type"` // text, long, double } +type IndexLine struct { + Token []string `json:"token"` + CaseSensitive bool `json:"caseSensitive"` + IncludeKeys []string `json:"include_keys,omitempty"` + ExcludeKeys []string `json:"exclude_keys,omitempty"` +} + +// Index is an index config for a log store. type Index struct { + TTL int `json:"ttl"` + Keys map[string]IndexKey `json:"keys,omitempty"` + Line *IndexLine `json:"line,omitempty"` } diff --git a/project_test.go b/project_test.go index a6393826..e00f0f82 100644 --- a/project_test.go +++ b/project_test.go @@ -9,21 +9,49 @@ import ( func TestCreateDeleteProject(t *testing.T) { c := Client{ Endpoint: "cn-shanghai.log.aliyuncs.com", - AccessKeyID: "LTAIJ4gv93WLYyA4", - AccessKeySecret: "xYv3JoZ8jeFmRIVzIiRUghEwVHnVdQ", + AccessKeyID: "", + AccessKeySecret: "", } - for i := 0; i < 2; i++ { + for i := 0; i < 1; i++ { var err error name := fmt.Sprintf("faint%d", i) - _, err = c.CreateProject(name, "this is faint") + //proj, err := c.CreateProject(name, "this is faint") + proj, err := c.GetProject(name) if err != nil { fmt.Printf("Create project error: %v\n", err) return } - time.Sleep(5 * time.Second) + /* + err = proj.CreateLogStore("test", 7, 1) + if err != nil { + fmt.Printf("Create store error: %v\n", err) + return + } + */ + + store, err := proj.GetLogStore("test") + if err != nil { + fmt.Printf("Get store error: %v\n", err) + return + } + err = store.CreateIndex(Index{ + TTL: 7, + Keys: map[string]IndexKey{ + "function": IndexKey{ + Token: []string{"\n", "\t", ";", ","}, + CaseSensitive: false, + Type: "text", + }, + }, + }) + if err != nil { + fmt.Printf("Create index error: %v\n", err) + return + } + err = c.DeleteProject(name) if err != nil { fmt.Printf("Delete project error: %v\n", err) From 8cfc1497221fa95c9b3408f129364c02f2f73856 Mon Sep 17 00:00:00 2001 From: Haoran Yang Date: Sun, 9 Apr 2017 11:19:47 +0800 Subject: [PATCH 5/5] Fix the tiny issues. Remove the manual test. --- client.go | 4 ++-- log_project.go | 4 ++-- log_store.go | 23 +++++++++++++------ project_test.go | 61 ------------------------------------------------- request.go | 6 ++--- 5 files changed, 23 insertions(+), 75 deletions(-) delete mode 100644 project_test.go diff --git a/client.go b/client.go index 3e7b8425..b70f614e 100644 --- a/client.go +++ b/client.go @@ -37,7 +37,7 @@ type Client struct { Endpoint string // IP or hostname of SLS endpoint AccessKeyID string AccessKeySecret string - SessionToken string + SecurityToken string } func convert(c *Client, projName string) *LogProject { @@ -46,7 +46,7 @@ func convert(c *Client, projName string) *LogProject { Endpoint: c.Endpoint, AccessKeyID: c.AccessKeyID, AccessKeySecret: c.AccessKeySecret, - SessionToken: c.SessionToken, + SecurityToken: c.SecurityToken, } } diff --git a/log_project.go b/log_project.go index 031b191d..e7964d15 100644 --- a/log_project.go +++ b/log_project.go @@ -13,7 +13,7 @@ type LogProject struct { Endpoint string // IP or hostname of SLS endpoint AccessKeyID string AccessKeySecret string - SessionToken string + SecurityToken string } // NewLogProject creates a new SLS project. @@ -29,7 +29,7 @@ func NewLogProject(name, endpoint, accessKeyID, accessKeySecret string) (p *LogP // WithToken add token parameter func (p *LogProject) WithToken(token string) (*LogProject, error) { - p.SessionToken = token + p.SecurityToken = token return p, nil } diff --git a/log_store.go b/log_store.go index bf2dce7b..7cf963c7 100644 --- a/log_store.go +++ b/log_store.go @@ -304,7 +304,6 @@ func (s *LogStore) CreateIndex(index Index) error { } uri := fmt.Sprintf("/logstores/%s/index", s.Name) - println(string(body)) _, err = request(s.project, "POST", uri, h, body) return err } @@ -329,7 +328,7 @@ func (s *LogStore) UpdateIndex(index Index) error { func (s *LogStore) DeleteIndex() error { type Body struct { project string `json:"projectName"` - store string `json:logstoreName` + store string `json:"logstoreName"` } body, err := json.Marshal(Body{ @@ -351,10 +350,10 @@ func (s *LogStore) DeleteIndex() error { return err } -func (s *LogStore) GetIndex() error { +func (s *LogStore) GetIndex() (*Index, error) { type Body struct { project string `json:"projectName"` - store string `json:logstoreName` + store string `json:"logstoreName"` } body, err := json.Marshal(Body{ @@ -362,7 +361,7 @@ func (s *LogStore) GetIndex() error { store: s.Name, }) if err != nil { - return err + return nil, err } h := map[string]string{ @@ -372,6 +371,16 @@ func (s *LogStore) GetIndex() error { } uri := fmt.Sprintf("/logstores/%s/index", s.Name) - _, err = request(s.project, "GET", uri, h, body) - return err + resp, err := request(s.project, "GET", uri, h, body) + if err != nil { + } + + index := &Index{} + data, _ := ioutil.ReadAll(resp.Body) + err = json.Unmarshal(data, index) + if err != nil { + return nil, err + } + + return index, err } diff --git a/project_test.go b/project_test.go deleted file mode 100644 index e00f0f82..00000000 --- a/project_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package sls - -import ( - "fmt" - "testing" - "time" -) - -func TestCreateDeleteProject(t *testing.T) { - c := Client{ - Endpoint: "cn-shanghai.log.aliyuncs.com", - AccessKeyID: "", - AccessKeySecret: "", - } - - for i := 0; i < 1; i++ { - var err error - name := fmt.Sprintf("faint%d", i) - //proj, err := c.CreateProject(name, "this is faint") - proj, err := c.GetProject(name) - if err != nil { - fmt.Printf("Create project error: %v\n", err) - return - } - time.Sleep(5 * time.Second) - - /* - err = proj.CreateLogStore("test", 7, 1) - if err != nil { - fmt.Printf("Create store error: %v\n", err) - return - } - */ - - store, err := proj.GetLogStore("test") - if err != nil { - fmt.Printf("Get store error: %v\n", err) - return - } - err = store.CreateIndex(Index{ - TTL: 7, - Keys: map[string]IndexKey{ - "function": IndexKey{ - Token: []string{"\n", "\t", ";", ","}, - CaseSensitive: false, - Type: "text", - }, - }, - }) - if err != nil { - fmt.Printf("Create index error: %v\n", err) - return - } - - err = c.DeleteProject(name) - if err != nil { - fmt.Printf("Delete project error: %v\n", err) - return - } - } -} diff --git a/request.go b/request.go index 1b6f381e..279a6d4b 100644 --- a/request.go +++ b/request.go @@ -29,8 +29,8 @@ func request(project *LogProject, method, uri string, headers map[string]string, headers["x-log-signaturemethod"] = signatureMethod // Access with token - if project.SessionToken != "" { - headers["x-acs-security-token"] = project.SessionToken + if project.SecurityToken != "" { + headers["x-acs-security-token"] = project.SecurityToken } if body != nil { @@ -76,9 +76,9 @@ func request(project *LogProject, method, uri string, headers map[string]string, } // Parse the sls error from body. - buf, _ := ioutil.ReadAll(resp.Body) if resp.StatusCode != http.StatusOK { err := &Error{} + buf, _ := ioutil.ReadAll(resp.Body) json.Unmarshal(buf, err) err.RequestID = resp.Header.Get("x-log-requestid") return nil, err