Update to SDK.v2

Update integration tests

Fixed build with v2

Signed-off-by: Juanjo Alvarez <[email protected]>

.travis.yml

@@ -7,7 +7,7 @@ services:
   - docker
 
 before_script:
-  - go get -v gopkg.in/bblfsh/sdk.v1/...
+  - go get -v gopkg.in/bblfsh/sdk.v2/...
   - bblfsh-sdk prepare-build .
   - go get -v -t ./driver/...
 

Dockerfile.build.tpl

@@ -3,6 +3,6 @@ FROM alpine:3.7
 RUN mkdir -p /opt/driver/src && \
     adduser $BUILD_USER -u $BUILD_UID -D -h /opt/driver/src
 
-RUN apk add --no-cache --update python python3 git make
+RUN apk add --no-cache --update python python3 py-pip py2-pip git make
 
 WORKDIR /opt/driver/src

Dockerfile.tpl

@@ -3,7 +3,7 @@ MAINTAINER source{d}
 
 ARG DEVDEPS=native/dev_deps
 ARG CONTAINER_DEVDEPS=/tmp/dev_deps
-ARG PYDETECTOR_VER=0.14.2
+ARG PYDETECTOR_VER=0.14.3
 
 RUN apk add --no-cache --update python python3 py-pip py2-pip git
 

Makefile

@@ -2,15 +2,16 @@ DEV_DEPS ?= native/dev_deps
 
 test-native-internal:
 	pip3 install --user ${DEV_DEPS}/python-pydetector/ || pip3 install --user pydetector-bblfsh
+	pip2 install --user ${DEV_DEPS}/python-pydetector/ || pip2 install --user pydetector-bblfsh
 	cd native/python_package/test && \
 	python3 -m unittest discover
 
 build-native-internal:
 	pip3 install --user ${DEV_DEPS}/python-pydetector/ || pip3 install --user pydetector-bblfsh
-	cd native/python_package/ && \
-	pip3 install -U --user .
+	pip2 install --user ${DEV_DEPS}/python-pydetector/ || pip2 install --user pydetector-bblfsh
+	cd native/python_package/ && pip3 install -U --user .
 	cp native/sh/native.sh $(BUILD_PATH)/bin/native;
 	chmod +x $(BUILD_PATH)/bin/native
-
+	cat $(BUILD_PATH)/bin/native
 
 include .sdk/Makefile

README.md

@@ -7,7 +7,7 @@ Development Environment
 
 Requirements:
 - `docker`
-- [`bblfsh-sdk`](https://github.com/bblfsh/sdk) _(go get -u gopkg.in/bblfsh/sdk.v1/...)_
+- [`bblfsh-sdk`](https://github.com/bblfsh/sdk) _(go get -u gopkg.in/bblfsh/sdk.v2/...)_
 - UAST converter dependencies _(go get -t -v ./...)_
 
 To initialize the build system execute: `bblfsh-sdk prepare-build`, at the root of the project. This will install the SDK at `.sdk` for this driver.

driver/fixtures/fixtures_test.go

@@ -5,8 +5,8 @@ import (
 	"testing"
 
 	"github.com/bblfsh/python-driver/driver/normalizer"
-	"gopkg.in/bblfsh/sdk.v1/sdk/driver"
-	"gopkg.in/bblfsh/sdk.v1/sdk/driver/fixtures"
+	"gopkg.in/bblfsh/sdk.v2/sdk/driver"
+	"gopkg.in/bblfsh/sdk.v2/sdk/driver/fixtures"
 )
 
 const projectRoot = "../../"

driver/impl/impl.go

@@ -1,6 +1,6 @@
 package impl
 
-import "gopkg.in/bblfsh/sdk.v1/sdk/driver"
+import "gopkg.in/bblfsh/sdk.v2/sdk/driver"
 
 func init() {
 	// Can be overridden to link a native driver into a Go driver server.

driver/main.go

@@ -4,7 +4,7 @@ import (
 	_ "github.com/bblfsh/python-driver/driver/impl"
 	"github.com/bblfsh/python-driver/driver/normalizer"
 
-	"gopkg.in/bblfsh/sdk.v1/sdk/driver"
+	"gopkg.in/bblfsh/sdk.v2/sdk/driver"
 )
 
 func main() {

driver/normalizer/annotation.go

@@ -1,10 +1,10 @@
 package normalizer
 
 import (
-	"gopkg.in/bblfsh/sdk.v1/uast"
-	"gopkg.in/bblfsh/sdk.v1/uast/role"
-	. "gopkg.in/bblfsh/sdk.v1/uast/transformer"
-	"gopkg.in/bblfsh/sdk.v1/uast/transformer/positioner"
+	"gopkg.in/bblfsh/sdk.v2/uast"
+	"gopkg.in/bblfsh/sdk.v2/uast/role"
+	. "gopkg.in/bblfsh/sdk.v2/uast/transformer"
+	"gopkg.in/bblfsh/sdk.v2/uast/transformer/positioner"
 )
 
 var Native = Transformers([][]Transformer{
@@ -22,7 +22,7 @@ var Code = []CodeTransformer{
 }
 
 func annotateTypeToken(typ, token string, roles ...role.Role) Mapping {
-	return AnnotateTypeFields(typ,
+	return AnnotateType(typ,
 		FieldRoles{
 			uast.KeyToken: {Add: true, Op: String(token)},
 		}, roles...)
@@ -185,9 +185,9 @@ var Annotations = []Mapping{
 	annotateTypeToken("Pass", "pass", role.Noop, role.Statement),
 	annotateTypeToken("Assert", "assert", role.Assert, role.Statement),
 
-	AnnotateTypeFields("Name", FieldRoles{"id": {Rename: uast.KeyToken}},
+	AnnotateType("Name", FieldRoles{"id": {Rename: uast.KeyToken}},
 		role.Identifier, role.Expression),
-	AnnotateTypeFields("Attribute", FieldRoles{"attr": {Rename: uast.KeyToken}},
+	AnnotateType("Attribute", FieldRoles{"attr": {Rename: uast.KeyToken}},
 		role.Identifier, role.Expression),
 
 	// Binary Expressions
@@ -198,20 +198,20 @@ var Annotations = []Mapping{
 	}, role.Expression, role.Binary),
 
 	// Primitive Literals
-	AnnotateTypeFields("Str", FieldRoles{"s": {Rename: uast.KeyToken}},
+	AnnotateType("Str", FieldRoles{"s": {Rename: uast.KeyToken}},
 		role.Literal, role.String, role.Expression, role.Primitive),
-	AnnotateTypeFields("Bytes", FieldRoles{"s": {Rename: uast.KeyToken}},
+	AnnotateType("Bytes", FieldRoles{"s": {Rename: uast.KeyToken}},
 		role.Literal, role.ByteString, role.Expression, role.Primitive),
-	AnnotateTypeFields("StringLiteral", FieldRoles{"s": {Rename: uast.KeyToken}},
+	AnnotateType("StringLiteral", FieldRoles{"s": {Rename: uast.KeyToken}},
 		role.Literal, role.String, role.Expression, role.Primitive),
-	AnnotateTypeFields("BoolLiteral", FieldRoles{"LiteralValue": {Rename: uast.KeyToken}},
+	AnnotateType("BoolLiteral", FieldRoles{"LiteralValue": {Rename: uast.KeyToken}},
 		role.Literal, role.Boolean, role.Expression, role.Primitive),
 	annotateTypeToken("NoneLiteral", "None", role.Literal, role.Null, role.Expression, role.Primitive),
-	AnnotateTypeFields("Num", FieldRoles{"n": {Rename: uast.KeyToken}},
+	AnnotateType("Num", FieldRoles{"n": {Rename: uast.KeyToken}},
 		role.Expression, role.Literal, role.Number, role.Primitive),
-	AnnotateTypeFields("BoolLiteral", FieldRoles{"LiteralValue": {Rename: uast.KeyToken}},
+	AnnotateType("BoolLiteral", FieldRoles{"LiteralValue": {Rename: uast.KeyToken}},
 		role.Expression, role.Literal, role.Boolean, role.Primitive),
-	AnnotateTypeFields("Dict", FieldRoles{
+	AnnotateType("Dict", FieldRoles{
 		"keys":   {Arr: true, Roles: role.Roles{role.Map, role.Key}},
 		"values": {Arr: true, Roles: role.Roles{role.Map, role.Value}},
 	}, role.Expression, role.Literal, role.Primitive, role.Map),
@@ -225,7 +225,7 @@ var Annotations = []Mapping{
 	//		targets[] => Left
 	//		value	  => Right
 	//
-	AnnotateTypeFields("Assign", FieldRoles{
+	AnnotateType("Assign", FieldRoles{
 		"targets": {Arr: true, Roles: role.Roles{role.Left}},
 		"value":   {Roles: role.Roles{role.Right}},
 	}, role.Binary, role.Expression, role.Assignment),
@@ -271,12 +271,12 @@ var Annotations = []Mapping{
 
 	// python 2 exception handling
 	AnnotateType("TryExcept", nil, role.Try, role.Catch, role.Statement),
-	AnnotateTypeFields("ExceptHandler", FieldRoles{"name": {Rename: uast.KeyToken}},
+	AnnotateType("ExceptHandler", FieldRoles{"name": {Rename: uast.KeyToken}},
 		role.Try, role.Catch, role.Identifier),
 	AnnotateType("TryFinally", nil, role.Try, role.Finally, role.Statement),
 	AnnotateType("Raise", nil, role.Throw),
 
-	AnnotateTypeFields("Raise",
+	AnnotateType("Raise",
 		FieldRoles{
 			"exc":         {Opt: true, Roles: role.Roles{role.Call}},
 			uast.KeyToken: {Add: true, Op: String("raise")},
@@ -296,7 +296,7 @@ var Annotations = []Mapping{
 	AnnotateType("SetComp", nil, role.Set, role.For, role.Expression),
 
 	// FIXME: once we have an async Role we should interpret the is_async property
-	AnnotateTypeFields("comprehension", FieldRoles{
+	AnnotateType("comprehension", FieldRoles{
 		"ifs":    {Arr: true, Roles: role.Roles{role.If, role.Condition}},
 		"iter":   {Roles: role.Roles{role.For, role.Update, role.Statement}},
 		"target": {Roles: role.Roles{role.For, role.Expression}},
@@ -327,7 +327,7 @@ var Annotations = []Mapping{
 
 	// Formal Arguments
 	// FIXME: opt: true + arr: true seems to cause a crash in the SDK
-	AnnotateTypeFields("arguments", FieldRoles{
+	AnnotateType("arguments", FieldRoles{
 		"args":     {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.Argument, role.Name, role.Identifier}},
 		"defaults": {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Value, role.Default}},
 		// Default arguments: Python's AST puts default arguments on a sibling list to the one of
@@ -341,48 +341,48 @@ var Annotations = []Mapping{
 		"vararg":      {Opt: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Name, role.Identifier}},
 	}, role.Function, role.Declaration, role.Argument, role.Incomplete),
 
-	AnnotateTypeFields("arguments", FieldRoles{
+	AnnotateType("arguments", FieldRoles{
 		"args":     {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.Argument, role.Name, role.Identifier}},
 		"defaults": {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Value, role.Default}},
 		"kwarg":    {Opt: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Map, role.Name, role.Identifier}},
 		"vararg":   {Opt: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Name, role.Identifier}},
 	}, role.Function, role.Declaration, role.Argument, role.Incomplete),
 
 	// Function Calls
-	AnnotateTypeFields("Call", FieldRoles{
+	AnnotateType("Call", FieldRoles{
 		"args":     {Arr: true, Roles: role.Roles{role.Function, role.Call, role.Positional, role.Argument, role.Name}},
 		"func":     {Roles: role.Roles{role.Call, role.Callee}},
 		"keywords": {Arr: true, Roles: role.Roles{role.Function, role.Call, role.Argument}},
 	}, role.Function, role.Call, role.Expression),
 
 	// Keywords are additionally annotated in FunctionDef and ClassDef
-	AnnotateTypeFields("keyword", FieldRoles{
+	AnnotateType("keyword", FieldRoles{
 		"value": {Roles: role.Roles{role.Argument, role.Value}},
 		"arg":   {Rename: uast.KeyToken},
 	}, role.Name),
 
 	// Comments and non significative whitespace
 	AnnotateType("SameLineNoops", nil, role.Comment),
 
-	AnnotateTypeFields("PreviousNoops", FieldRoles{
+	AnnotateType("PreviousNoops", FieldRoles{
 		"lines": {Arr: true, Roles: role.Roles{role.Noop}},
 	}, role.Noop),
 
-	AnnotateTypeFields("RemainderNoops", FieldRoles{
+	AnnotateType("RemainderNoops", FieldRoles{
 		"lines": {Arr: true, Roles: role.Roles{role.Noop}},
 	}, role.Noop),
 
-	AnnotateTypeFields("NoopLine", FieldRoles{
+	AnnotateType("NoopLine", FieldRoles{
 		"noop_line": {Rename: uast.KeyToken},
 	}, role.Noop, role.Comment),
 
-	AnnotateTypeFields("NoopSameLine", FieldRoles{
+	AnnotateType("NoopSameLine", FieldRoles{
 		"s": {Rename: uast.KeyToken},
 	}, role.Noop, role.Comment),
 
 	// Qualified Identifiers
 	// a.b.c ("a" and "b" will be Qualified+Identifier, "c" will be just Identifier)
-	AnnotateTypeFields("Attribute", FieldRoles{
+	AnnotateType("Attribute", FieldRoles{
 		"value": {Arr: true, Roles: role.Roles{role.Qualified}},
 	}),
 
@@ -458,20 +458,20 @@ var Annotations = []Mapping{
 		},
 	}, role.Type, role.Declaration, role.Identifier, role.Statement),
 
-	AnnotateTypeFields("ClassDef", FieldRoles{
+	AnnotateType("ClassDef", FieldRoles{
 		"keywords": {Arr: true, Roles: role.Roles{role.Incomplete}},
 	}),
 
 	// These two (exec & print) are AST nodes in Python2 but we convert them to functions
 	// in the UAST like they are in Python3
-	AnnotateTypeFields("Exec", FieldRoles{
+	AnnotateType("Exec", FieldRoles{
 		"body":        {Roles: role.Roles{role.Call, role.Argument, role.Positional}},
 		"globals":     {Roles: role.Roles{role.Call, role.Argument, role.Positional}},
 		"locals":      {Roles: role.Roles{role.Call, role.Argument, role.Positional}},
 		uast.KeyToken: {Add: true, Op: String("exec")},
 	}, role.Function, role.Call, role.Expression),
 
-	AnnotateTypeFields("Print", FieldRoles{
+	AnnotateType("Print", FieldRoles{
 		"values":      {Arr: true, Roles: role.Roles{role.Call, role.Argument, role.Positional}},
 		uast.KeyToken: {Add: true, Op: String("print")},
 	}, role.Function, role.Call, role.Callee, role.Identifier, role.Expression),

fixtures/issue62.py.uast

@@ -7,7 +7,6 @@ Module {
 .  Children: {
 .  .  0: ImportFrom {
 .  .  .  Roles: Declaration,Import,Statement
-.  .  .  TOKEN "import from"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 0
 .  .  .  .  Line: 1
@@ -23,14 +22,14 @@ Module {
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: ImportFrom.level {
-.  .  .  .  .  Roles: Identifier,Import,Incomplete
+.  .  .  .  .  Roles: Import,Incomplete
 .  .  .  .  .  TOKEN "0"
 .  .  .  .  .  Properties: {
 .  .  .  .  .  .  internalRole: level
 .  .  .  .  .  }
 .  .  .  .  }
 .  .  .  .  1: ImportFrom.module {
-.  .  .  .  .  Roles: Identifier,Import
+.  .  .  .  .  Roles: Identifier,Import,Pathname
 .  .  .  .  .  TOKEN "os"
 .  .  .  .  .  Properties: {
 .  .  .  .  .  .  internalRole: module

fixtures/issue62_b.py.uast

@@ -7,7 +7,6 @@ Module {
 .  Children: {
 .  .  0: ImportFrom {
 .  .  .  Roles: Declaration,Import,Statement
-.  .  .  TOKEN "import from"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 0
 .  .  .  .  Line: 1
@@ -23,14 +22,14 @@ Module {
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: ImportFrom.level {
-.  .  .  .  .  Roles: Identifier,Import,Incomplete
+.  .  .  .  .  Roles: Import,Incomplete
 .  .  .  .  .  TOKEN "0"
 .  .  .  .  .  Properties: {
 .  .  .  .  .  .  internalRole: level
 .  .  .  .  .  }
 .  .  .  .  }
 .  .  .  .  1: ImportFrom.module {
-.  .  .  .  .  Roles: Identifier,Import
+.  .  .  .  .  Roles: Identifier,Import,Pathname
 .  .  .  .  .  TOKEN "collections"
 .  .  .  .  .  Properties: {
 .  .  .  .  .  .  internalRole: module
@@ -63,7 +62,6 @@ Module {
 .  .  }
 .  .  1: ImportFrom {
 .  .  .  Roles: Declaration,Import,Statement
-.  .  .  TOKEN "import from"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 33
 .  .  .  .  Line: 3
@@ -74,14 +72,14 @@ Module {
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: ImportFrom.level {
-.  .  .  .  .  Roles: Identifier,Import,Incomplete
+.  .  .  .  .  Roles: Import,Incomplete
 .  .  .  .  .  TOKEN "0"
 .  .  .  .  .  Properties: {
 .  .  .  .  .  .  internalRole: level
 .  .  .  .  .  }
 .  .  .  .  }
 .  .  .  .  1: ImportFrom.module {
-.  .  .  .  .  Roles: Identifier,Import
+.  .  .  .  .  Roles: Identifier,Import,Pathname
 .  .  .  .  .  TOKEN "ast2vec.bblfsh_roles"
 .  .  .  .  .  Properties: {
 .  .  .  .  .  .  internalRole: module
@@ -130,7 +128,6 @@ Module {
 .  .  }
 .  .  2: ImportFrom {
 .  .  .  Roles: Declaration,Import,Statement
-.  .  .  TOKEN "import from"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 84
 .  .  .  .  Line: 4
@@ -141,14 +138,14 @@ Module {
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: ImportFrom.level {
-.  .  .  .  .  Roles: Identifier,Import,Incomplete
+.  .  .  .  .  Roles: Import,Incomplete
 .  .  .  .  .  TOKEN "0"
 .  .  .  .  .  Properties: {
 .  .  .  .  .  .  internalRole: level
 .  .  .  .  .  }
 .  .  .  .  }
 .  .  .  .  1: ImportFrom.module {
-.  .  .  .  .  Roles: Identifier,Import
+.  .  .  .  .  Roles: Identifier,Import,Pathname
 .  .  .  .  .  TOKEN "ast2vec.repo2.base"
 .  .  .  .  .  Properties: {
 .  .  .  .  .  .  internalRole: module