Skip to content
This repository has been archived by the owner on Aug 31, 2022. It is now read-only.

Add Virtual Database Client + Unit Tests #18

Open
wants to merge 22 commits into
base: master
Choose a base branch
from
Open

Add Virtual Database Client + Unit Tests #18

Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
43a8fcd
Add feature to support query for PFC watchdog
chaliu Sep 5, 2018
3908994
fix typos;rewrite log messeages; replace hard-coded : with database s…
chaliu Sep 5, 2018
cca4c30
replace hardcoded table separator |
chaliu Sep 6, 2018
750133e
Implement new virtual path for pfcwd
chaliu Sep 26, 2018
b2c6c5e
Unit tests for virtual database target SONiC, rename new_sonic_data_c…
randygaulmsft Mar 25, 2019
9ea0465
Merge branch 'master' into new-pfcwd
randygaulmsft Mar 26, 2019
0fcf9a5
remove testing directory imports
randygaulmsft Mar 26, 2019
f0e0fdb
silence unnecessary printfs
randygaulmsft Apr 10, 2019
bd4af90
remove more debug printf's
randygaulmsft Apr 10, 2019
da43c69
Attempt to fixup import paths and remove old unnecessary debug format…
randygaulmsft Apr 25, 2019
6dbe27c
Merge branch 'new-pfcwd' of https://github.com/randygaulmsft/sonic-te…
randygaulmsft Apr 25, 2019
d16da4c
get tests in a good state
randygaulmsft May 5, 2019
b414595
Got a basic subscribe test working.
randygaulmsft May 6, 2019
e29cfa5
Figured out way to subscribe to target SONiC_DB
randygaulmsft May 6, 2019
e597358
Wrapped up stream queries.
randygaulmsft May 7, 2019
90d4437
gofmt on all code under folder.
randygaulmsft May 15, 2019
ede7a94
Remove dead code (comments).
randygaulmsft May 15, 2019
f01f39c
SONiC_DB to SONIC_DB
randygaulmsft May 15, 2019
d33fca6
Removed dead code.
randygaulmsft May 17, 2019
9f94324
Remove more dead code.
randygaulmsft May 17, 2019
6f2b220
sdc to vdc (use correct path for setting TCP redis settings)
randygaulmsft May 17, 2019
53c0e70
Remove dead code from sonic_data_client.
randygaulmsft May 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,6 @@ There is also a test program dialout_server_cli for collecting data from dial-ou

The binaries will be installed under $GOPATH/bin/, they may be copied to any SONiC switch and run there.

You can also build a debian package and install it:
randygaulmsft marked this conversation as resolved.
Show resolved Hide resolved

git clone https://github.com/Azure/sonic-telemetry.git
pushd sonic-telemetry
dpkg-buildpackage -rfakeroot -b -us -uc
popd

### Running
* See [SONiC gRPC telemetry](./doc/grpc_telemetry.md) for how to run dial-in mode system telemetry server
* See [SONiC telemetry in dial-out mode](./doc/dialout.md) for how to run dial-out mode system telemetry client
Expand Down
8 changes: 5 additions & 3 deletions dialout/dialout_client/dialout_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package telemetry_dialout

import (
// "encoding/json"
"crypto/tls"
"errors"
"fmt"

spb "github.com/Azure/sonic-telemetry/proto"

"net"

sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
"github.com/go-redis/redis"
log "github.com/golang/glog"
Expand All @@ -15,8 +18,7 @@ import (
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"net"
//"reflect"

"strconv"
"strings"
"sync"
Expand Down
12 changes: 5 additions & 7 deletions dialout/dialout_client/dialout_client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,16 @@ package telemetry_dialout
import (
"crypto/tls"
"encoding/json"

"github.com/go-redis/redis"
//"github.com/golang/protobuf/proto"

testcert "github.com/Azure/sonic-telemetry/testdata/tls"

//"github.com/kylelemons/godebug/pretty"
//"github.com/openconfig/gnmi/client"
pb "github.com/openconfig/gnmi/proto/gnmi"
"github.com/openconfig/gnmi/value"
"golang.org/x/net/context"
"google.golang.org/grpc"
//"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
//"google.golang.org/grpc/status"
//"fmt"

"io/ioutil"
"os"
"os/exec"
Expand All @@ -28,6 +24,8 @@ import (
"testing"
"time"

"google.golang.org/grpc/credentials"

sds "github.com/Azure/sonic-telemetry/dialout/dialout_server"
spb "github.com/Azure/sonic-telemetry/proto"
sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
Expand Down
10 changes: 9 additions & 1 deletion gnmi_server/client_subscribe.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/codes"

//spb "github.com/Azure/sonic-telemetry/proto"
randygaulmsft marked this conversation as resolved.
Show resolved Hide resolved
sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
vdc "github.com/Azure/sonic-telemetry/virtual_database_client"

gnmipb "github.com/openconfig/gnmi/proto/gnmi"
)

Expand Down Expand Up @@ -119,6 +120,8 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
var dc sdc.Client
if target == "OTHERS" {
dc, err = sdc.NewNonDbClient(paths, prefix)
} else if target == "SONIC_DB" {
dc, err = vdc.NewDbClient(paths, prefix)
} else {
dc, err = sdc.NewDbClient(paths, prefix)
}
Expand Down Expand Up @@ -226,6 +229,11 @@ func (c *Client) send(stream gnmipb.GNMI_SubscribeServer) error {
c.errors++
return err
}
case vdc.Value:
if resp, err = vdc.ValToResp(v); err != nil {
c.errors++
return err
}
default:
log.V(1).Infof("Unknown data type %v for %s in queue", items[0], c)
c.errors++
Expand Down
13 changes: 13 additions & 0 deletions gnmi_server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import (
"google.golang.org/grpc/status"

sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
vdc "github.com/Azure/sonic-telemetry/virtual_database_client"

gnmipb "github.com/openconfig/gnmi/proto/gnmi"
)

Expand Down Expand Up @@ -77,6 +79,11 @@ func (srv *Server) Serve() error {
return srv.s.Serve(srv.lis)
}

func (srv *Server) Stop() {
s := srv.s
s.Stop()
}

// Address returns the port the Server is listening to.
func (srv *Server) Address() string {
addr := srv.lis.Addr().String()
Expand Down Expand Up @@ -174,6 +181,8 @@ func (s *Server) Get(ctx context.Context, req *gnmipb.GetRequest) (*gnmipb.GetRe
var dc sdc.Client
if target == "OTHERS" {
dc, err = sdc.NewNonDbClient(paths, prefix)
} else if target == "SONIC_DB" {
dc, err = vdc.NewDbClient(paths, prefix)
} else {
dc, err = sdc.NewDbClient(paths, prefix)
}
Expand All @@ -186,6 +195,10 @@ func (s *Server) Get(ctx context.Context, req *gnmipb.GetRequest) (*gnmipb.GetRe
return nil, status.Error(codes.NotFound, err.Error())
}

if target == "SONIC_DB" {
notifications = make([]*gnmipb.Notification, len(spbValues))
}

for index, spbValue := range spbValues {
update := &gnmipb.Update{
Path: spbValue.GetPath(),
Expand Down
16 changes: 10 additions & 6 deletions gnmi_server/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,18 @@ package gnmi
import (
"crypto/tls"
"encoding/json"

testcert "github.com/Azure/sonic-telemetry/testdata/tls"
"github.com/go-redis/redis"
"github.com/golang/protobuf/proto"

"io/ioutil"
"os"
"os/exec"
"reflect"
"testing"
"time"

"github.com/kylelemons/godebug/pretty"
"github.com/openconfig/gnmi/client"
pb "github.com/openconfig/gnmi/proto/gnmi"
Expand All @@ -19,15 +27,11 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/status"
"io/ioutil"
"os"
"os/exec"
"reflect"
"testing"
"time"

// Register supported client types.
spb "github.com/Azure/sonic-telemetry/proto"
sdc "github.com/Azure/sonic-telemetry/sonic_data_client"

gclient "github.com/jipanyang/gnmi/client/gnmi"
)

Expand Down
25 changes: 23 additions & 2 deletions sonic_data_client/db_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ func (val Value) Compare(other queue.Item) int {
type DbClient struct {
prefix *gnmipb.Path
pathG2S map[*gnmipb.Path][]tablePath

q *queue.PriorityQueue
channel chan struct{}

Expand All @@ -112,6 +113,14 @@ func NewDbClient(paths []*gnmipb.Path, prefix *gnmipb.Path) (Client, error) {
if UseRedisLocalTcpPort {
useRedisTcpClient()
}

// TODO: Remove debug log
//for _, _path := range paths {
// fmt.Printf("single path: %v\n", _path)
//}
//
//fmt.Printf("prefix: %v\n", prefix)

if prefix.GetTarget() == "COUNTERS_DB" {
err = initCountersPortNameMap()
if err != nil {
Expand Down Expand Up @@ -145,7 +154,7 @@ func NewDbClient(paths []*gnmipb.Path, prefix *gnmipb.Path) (Client, error) {
// String returns the target the client is querying.
func (c *DbClient) String() string {
// TODO: print gnmiPaths of this DbClient
return fmt.Sprintf("DbClient Prefix %v sendMsg %v, recvMsg %v",
return fmt.Sprintf("DbClient Prefix %v sendMsg %v, recvMsg %v",
c.prefix.GetTarget(), c.sendMsg, c.recvMsg)
}

Expand Down Expand Up @@ -243,6 +252,7 @@ func (c *DbClient) Get(w *sync.WaitGroup) ([]*spb.Value, error) {
ts := time.Now()
for gnmiPath, tblPaths := range c.pathG2S {
val, err := tableData2TypedValue(tblPaths, nil)
//log.V(5).Infof("Val: %v\n", val)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -496,6 +506,12 @@ func makeJSON_redis(msi *map[string]interface{}, key *string, op *string, mfv ma
for f, v := range mfv {
(*msi)[f] = v
}
// Debug log
//log.V(5).Infof("msi: %v\n", msi)
//log.V(5).Infof("key: %v\n", key)
//log.V(5).Infof("op: %v\n", op)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove these

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They don't appear to have been removed.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah these were in multiple files. I removed them from virtual_database_client but this one was under sonic_data_client.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove if you have not. Thanks!

//log.V(5).Infof("fv: %v\n", mfv)

return nil
}

Expand All @@ -515,12 +531,12 @@ func makeJSON_redis(msi *map[string]interface{}, key *string, op *string, mfv ma
of[*op] = fp
(*msi)[*key] = of
}

return nil
}

// emitJSON marshalls map[string]interface{} to JSON byte stream.
func emitJSON(v *map[string]interface{}) ([]byte, error) {
//j, err := json.MarshalIndent(*v, "", indentString)
j, err := json.Marshal(*v)
if err != nil {
return nil, fmt.Errorf("JSON marshalling error: %v", err)
Expand Down Expand Up @@ -589,6 +605,7 @@ func tableData2Msi(tblPath *tablePath, useKey bool, op *string, msi *map[string]
// Split dbkey string into two parts and second part is key in table
keys := strings.SplitN(dbkey, tblPath.delimitor, 2)
key = keys[1]

err = makeJSON_redis(msi, &key, op, fv)
}
if err != nil {
Expand All @@ -597,6 +614,7 @@ func tableData2Msi(tblPath *tablePath, useKey bool, op *string, msi *map[string]
}
log.V(6).Infof("Added idex %v fv %v ", idx, fv)
}

return nil
}

Expand Down Expand Up @@ -644,6 +662,9 @@ func tableData2TypedValue(tblPaths []tablePath, op *string) (*gnmipb.TypedValue,
}
}

// Debug logging
log.V(5).Infof("tblPath: %v\n", tblPath)

err := tableData2Msi(&tblPath, useKey, nil, &msi)
if err != nil {
return nil, err
Expand Down
3 changes: 2 additions & 1 deletion sonic_data_client/virtual_db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ package client

import (
"fmt"
log "github.com/golang/glog"
"strings"

log "github.com/golang/glog"
)

// virtual db is to Handle
Expand Down
1 change: 1 addition & 0 deletions telemetry/telemetry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"google.golang.org/grpc/credentials"

gnmi "github.com/Azure/sonic-telemetry/gnmi_server"

testcert "github.com/Azure/sonic-telemetry/testdata/tls"
)

Expand Down
84 changes: 84 additions & 0 deletions testdata/Interfaces_Port_name_Ethernet68_1_BaseCounter.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
{
"SAI_PORT_STAT_ETHER_IN_PKTS_1024_TO_1518_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_128_TO_255_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_1519_TO_2047_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_2048_TO_4095_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_256_TO_511_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_4096_TO_9216_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_512_TO_1023_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_64_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_65_TO_127_OCTETS": "0",
"SAI_PORT_STAT_ETHER_IN_PKTS_9217_TO_16383_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_1024_TO_1518_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_128_TO_255_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_1519_TO_2047_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_2048_TO_4095_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_256_TO_511_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_4096_TO_9216_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_512_TO_1023_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_64_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_65_TO_127_OCTETS": "0",
"SAI_PORT_STAT_ETHER_OUT_PKTS_9217_TO_16383_OCTETS": "0",
"SAI_PORT_STAT_ETHER_RX_OVERSIZE_PKTS": "0",
"SAI_PORT_STAT_ETHER_STATS_BROADCAST_PKTS": "0",
"SAI_PORT_STAT_ETHER_STATS_COLLISIONS": "0",
"SAI_PORT_STAT_ETHER_STATS_CRC_ALIGN_ERRORS": "0",
"SAI_PORT_STAT_ETHER_STATS_DROP_EVENTS": "0",
"SAI_PORT_STAT_ETHER_STATS_FRAGMENTS": "0",
"SAI_PORT_STAT_ETHER_STATS_JABBERS": "0",
"SAI_PORT_STAT_ETHER_STATS_MULTICAST_PKTS": "0",
"SAI_PORT_STAT_ETHER_STATS_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_OVERSIZE_PKTS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_1024_TO_1518_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_128_TO_255_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_1519_TO_2047_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_2048_TO_4095_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_256_TO_511_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_4096_TO_9216_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_512_TO_1023_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_64_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_65_TO_127_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_PKTS_9217_TO_16383_OCTETS": "0",
"SAI_PORT_STAT_ETHER_STATS_RX_NO_ERRORS": "0",
"SAI_PORT_STAT_ETHER_STATS_TX_NO_ERRORS": "0",
"SAI_PORT_STAT_ETHER_STATS_UNDERSIZE_PKTS": "0",
"SAI_PORT_STAT_ETHER_TX_OVERSIZE_PKTS": "0",
"SAI_PORT_STAT_IF_IN_BROADCAST_PKTS": "0",
"SAI_PORT_STAT_IF_IN_DISCARDS": "0",
"SAI_PORT_STAT_IF_IN_ERRORS": "0",
"SAI_PORT_STAT_IF_IN_MULTICAST_PKTS": "0",
"SAI_PORT_STAT_IF_IN_NON_UCAST_PKTS": "0",
"SAI_PORT_STAT_IF_IN_OCTETS": "0",
"SAI_PORT_STAT_IF_IN_UCAST_PKTS": "0",
"SAI_PORT_STAT_IF_IN_UNKNOWN_PROTOS": "0",
"SAI_PORT_STAT_IF_IN_VLAN_DISCARDS": "0",
"SAI_PORT_STAT_IF_OUT_BROADCAST_PKTS": "0",
"SAI_PORT_STAT_IF_OUT_DISCARDS": "0",
"SAI_PORT_STAT_IF_OUT_ERRORS": "0",
"SAI_PORT_STAT_IF_OUT_MULTICAST_PKTS": "0",
"SAI_PORT_STAT_IF_OUT_NON_UCAST_PKTS": "0",
"SAI_PORT_STAT_IF_OUT_OCTETS": "0",
"SAI_PORT_STAT_IF_OUT_QLEN": "0",
"SAI_PORT_STAT_IF_OUT_UCAST_PKTS": "0",
"SAI_PORT_STAT_IPV6_IN_DISCARDS": "0",
"SAI_PORT_STAT_IPV6_IN_MCAST_PKTS": "0",
"SAI_PORT_STAT_IPV6_IN_NON_UCAST_PKTS": "0",
"SAI_PORT_STAT_IPV6_IN_OCTETS": "0",
"SAI_PORT_STAT_IPV6_IN_RECEIVES": "0",
"SAI_PORT_STAT_IPV6_IN_UCAST_PKTS": "0",
"SAI_PORT_STAT_IPV6_OUT_DISCARDS": "0",
"SAI_PORT_STAT_IPV6_OUT_MCAST_PKTS": "0",
"SAI_PORT_STAT_IPV6_OUT_NON_UCAST_PKTS": "0",
"SAI_PORT_STAT_IPV6_OUT_OCTETS": "0",
"SAI_PORT_STAT_IPV6_OUT_UCAST_PKTS": "0",
"SAI_PORT_STAT_IP_IN_DISCARDS": "0",
"SAI_PORT_STAT_IP_IN_NON_UCAST_PKTS": "0",
"SAI_PORT_STAT_IP_IN_OCTETS": "0",
"SAI_PORT_STAT_IP_IN_RECEIVES": "0",
"SAI_PORT_STAT_IP_IN_UCAST_PKTS": "0",
"SAI_PORT_STAT_IP_OUT_DISCARDS": "0",
"SAI_PORT_STAT_IP_OUT_NON_UCAST_PKTS": "0",
"SAI_PORT_STAT_IP_OUT_OCTETS": "0",
"SAI_PORT_STAT_IP_OUT_UCAST_PKTS": "0"
}
18 changes: 18 additions & 0 deletions testdata/Interfaces_Port_name_Ethernet68_1_PfcCounter.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"SAI_PORT_STAT_PFC_0_RX_PKTS": "0",
"SAI_PORT_STAT_PFC_0_TX_PKTS": "0",
"SAI_PORT_STAT_PFC_1_RX_PKTS": "0",
"SAI_PORT_STAT_PFC_1_TX_PKTS": "0",
"SAI_PORT_STAT_PFC_2_RX_PKTS": "0",
"SAI_PORT_STAT_PFC_2_TX_PKTS": "0",
"SAI_PORT_STAT_PFC_3_RX_PKTS": "0",
"SAI_PORT_STAT_PFC_3_TX_PKTS": "0",
"SAI_PORT_STAT_PFC_4_RX_PKTS": "0",
"SAI_PORT_STAT_PFC_4_TX_PKTS": "0",
"SAI_PORT_STAT_PFC_5_RX_PKTS": "0",
"SAI_PORT_STAT_PFC_5_TX_PKTS": "0",
"SAI_PORT_STAT_PFC_6_RX_PKTS": "0",
"SAI_PORT_STAT_PFC_6_TX_PKTS": "0",
"SAI_PORT_STAT_PFC_7_RX_PKTS": "2",
"SAI_PORT_STAT_PFC_7_TX_PKTS": "0"
}
Loading