Process positional arguments

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Function.java

@@ -3,17 +3,21 @@
 import static org.eclipse.core.runtime.Platform.getLog;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Objects;
+import java.util.Set;
 
 import org.eclipse.core.runtime.ILog;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.jface.text.BadLocationException;
 import org.eclipse.jface.text.IDocument;
+import org.python.pydev.ast.codecompletion.revisited.visitors.Definition;
 import org.python.pydev.core.IPythonNature;
 import org.python.pydev.core.docutils.PySelection;
 import org.python.pydev.parser.jython.ast.Attribute;
 import org.python.pydev.parser.jython.ast.Call;
 import org.python.pydev.parser.jython.ast.FunctionDef;
+import org.python.pydev.parser.jython.ast.Name;
 import org.python.pydev.parser.jython.ast.NameTok;
 import org.python.pydev.parser.jython.ast.argumentsType;
 import org.python.pydev.parser.jython.ast.decoratorsType;
@@ -41,24 +45,54 @@ public class Function extends RefactorableProgramEntity {
 	 */
 	public class HybridizationParameters {
 
+		/**
+		 * Available in TF version [2.4,2.11].
+		 */
 		private static final String EXPERIMENTAL_FOLLOW_TYPE_HINTS = "experimental_follow_type_hints";
 
+		/**
+		 * Available in TF version [2.0,2.11].
+		 */
 		private static final String EXPERIMENTAL_AUTOGRAPH_OPTIONS = "experimental_autograph_options";
 
+		/**
+		 * Available in TF version [2.1,2.11].
+		 */
 		private static final String EXPERIMENTAL_IMPLEMENTS = "experimental_implements";
 
+		/**
+		 * Available in TF version [2.9,2.11]. Previously, experimental_relax_shapes which is available TF version [2.0,2.11].
+		 */
 		private static final String REDUCE_RETRACING = "reduce_retracing";
 
+		/**
+		 * Available in TF version [2.0,2.11].
+		 */
 		private static final String EXPERIMENTAL_RELAX_SHAPES = "experimental_relax_shapes";
 
+		/**
+		 * Available in TF version [2.1,2.11].
+		 */
 		private static final String EXPERIMENTAL_COMPILE = "experimental_compile";
 
+		/**
+		 * Available in TF version [2.5,2.11]. Previously, experimental_compile which is available TF version [2.1,2.11].
+		 */
 		private static final String JIT_COMPILE = "jit_compile";
 
+		/**
+		 * Available in TF version [2.0,2.11].
+		 */
 		private static final String AUTOGRAPH = "autograph";
 
+		/**
+		 * Available in TF version [2.0,2.11].
+		 */
 		private static final String INPUT_SIGNATURE = "input_signature";
 
+		/**
+		 * Available in TF version [2.0,2.11].
+		 */
 		private static final String FUNC = "func";
 
 		/**
@@ -108,6 +142,10 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep
 			// Will contain the last tf.function decorator
 			decoratorsType tfFunctionDecorator = null;
 
+			// Declaring definitions of the decorator, if it contains multiple definitions there might be more than one in this set. Since
+			// we are dealing with tf.function, we expect only one.
+			Definition declaringDefinition = null;
+
 			// Iterate through the decorators of the function
 			for (decoratorsType decorator : decoratorArray) {
 				IDocument document = Function.this.getContainingDocument();
@@ -116,19 +154,72 @@ public HybridizationParameters(IProgressMonitor monitor) throws BadLocationExcep
 				// Save the hybrid decorator
 				try {
 					if (Function.isHybrid(decorator, Function.this.containingModuleName, Function.this.containingFile, selection,
-							Function.this.nature, monitor)) // TODO: Cache this from a previous call (#118).
+							Function.this.nature, monitor)) { // TODO: Cache this from a previous call (#118).
 						tfFunctionDecorator = decorator;
+						// Returns the set of potential declaring definitions of the selection.
+						Set<Definition> potentialDeclaringDefitinion = Util.getDeclaringDefinition(selection,
+								Function.this.containingModuleName, Function.this.containingFile, Function.this.nature, monitor);
+						if (potentialDeclaringDefitinion.size() == 1)
+							declaringDefinition = potentialDeclaringDefitinion.iterator().next();
+					}
 				} catch (AmbiguousDeclaringModuleException e) {
 					throw new IllegalStateException("Can't determine whether decorator: " + decorator + " is hybrid.", e);
 				}
 			} // We expect to have the last tf.function decorator in tfFunctionDecorator
 
+			// Getting tf.functions Python definition arguments.
+			ArrayList<String> argumentIdDeclaringDefintion = getTfFunctionPythonDefinitionArguments(declaringDefinition);
+
 			if (tfFunctionDecorator != null)
 				// tfFunctionDecorator must be an instance of Call, because that's the only way we have parameters.
 				if (tfFunctionDecorator.func instanceof Call) {
 					Call callFunction = (Call) tfFunctionDecorator.func;
-					// We only care about the actual keywords for now.
-					// TODO: Parse positional arguments (#108).
+
+					// Processing positional arguments for tf.function
+					exprType[] arguments = callFunction.args;
+					// We iterate over the tf.function's parameters positions.
+					for (int i = 0; i < arguments.length; i++) {
+						// From the position i, we use the tf.function's definition to verify which parameter we are analyzing.
+						String argumentDeclaringDefinition = argumentIdDeclaringDefintion.get(i);
+
+						// Matching the arguments from the definition and the arguments from the code being analyzed.
+						if (argumentDeclaringDefinition.equals(FUNC))
+							// Found parameter func
+							this.funcParamExists = true;
+						else if (argumentDeclaringDefinition.equals(INPUT_SIGNATURE))
+							// Found parameter input_signature
+							this.inputSignatureParamExists = true;
+						else if (argumentDeclaringDefinition.equals(AUTOGRAPH))
+							// Found parameter autograph
+							this.autoGraphParamExists = true;
+						// In our accepter interval version ([2.0,2.11]) of the API allows parameter names jit_compile and
+						// deprecated name experimental_compile.
+						else if (argumentDeclaringDefinition.equals(JIT_COMPILE)
+								|| argumentDeclaringDefinition.equals(EXPERIMENTAL_COMPILE))
+							// Found parameter jit_compile/experimental_compile
+							this.jitCompileParamExists = true;
+						// In our accepter interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing
+						// and deprecated name experimental_relax_shapes.
+						else if (argumentDeclaringDefinition.equals(REDUCE_RETRACING))
+							// Found parameter reduce_retracing
+							this.reduceRetracingParamExists = true;
+						else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_RELAX_SHAPES))
+							// Found parameter experimental_relax_shapes
+							this.reduceRetracingParamExists = true;
+						else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_IMPLEMENTS))
+							// Found parameter experimental_implements
+							this.experimentalImplementsParamExists = true;
+						else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_AUTOGRAPH_OPTIONS))
+							// Found parameter experimental_autograph_options
+							this.experimentalAutographOptionsParamExists = true;
+						else if (argumentDeclaringDefinition.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS))
+							// Found parameter experimental_follow_type_hints
+							this.experimentaFollowTypeHintsParamExists = true;
+					}
+
+					// Processing keywords arguments
+					// If we have keyword parameter, afterwards, we cannot have positional parameters because it would result in invalid
+					// Python code. This is why we check the keywords last.
 					keywordType[] keywords = callFunction.keywords;
 					for (keywordType keyword : keywords) {
 						if (keyword.arg instanceof NameTok) {
@@ -142,15 +233,13 @@ else if (name.id.equals(INPUT_SIGNATURE))
 							else if (name.id.equals(AUTOGRAPH))
 								// Found parameter autograph
 								this.autoGraphParamExists = true;
-							// The version of the API we are using allows
-							// parameter names jit_compile and
-							// deprecated name experimental_compile
+							// In our accepter interval version ([2.0,2.11]) of the API allows parameter names jit_compile and
+							// deprecated name experimental_compile.
 							else if (name.id.equals(JIT_COMPILE) || name.id.equals(EXPERIMENTAL_COMPILE))
 								// Found parameter jit_compile/experimental_compile
 								this.jitCompileParamExists = true;
-							// The version of the API we are using allows
-							// parameter names reduce_retracing
-							// and deprecated name experimental_relax_shapes
+							// In our accepter interval version ([2.0,2.11]) of the API allows parameter names reduce_retracing
+							// and deprecated name experimental_relax_shapes.
 							else if (name.id.equals(REDUCE_RETRACING) || name.id.equals(EXPERIMENTAL_RELAX_SHAPES))
 								// Found parameter reduce_retracing
 								// or experimental_relax_shapes
@@ -169,6 +258,41 @@ else if (name.id.equals(EXPERIMENTAL_FOLLOW_TYPE_HINTS))
 				} // else, tf.function is used without parameters.
 		}
 
+		/**
+		 * Get the tf.function parameter names from the {@link Definition}.
+		 *
+		 * @param declaringDefinition The Definition to use.
+		 * @return An array with the names of the arguments given by {@link Definition}.
+		 */
+		ArrayList<String> getTfFunctionPythonDefinitionArguments(Definition declaringDefinition) {
+			// Python source arguments from the declaring definition
+			exprType[] declaringArguments = null;
+
+			// Getting the arguments from TensorFlow source
+			if (declaringDefinition != null) {
+				if (declaringDefinition.ast instanceof FunctionDef) {
+					FunctionDef declaringFunctionDefinition = (FunctionDef) declaringDefinition.ast;
+					argumentsType declaringArgumentTypes = declaringFunctionDefinition.args;
+					declaringArguments = declaringArgumentTypes.args;
+				}
+			}
+
+			// Python source arguments from the declaring definition
+			ArrayList<String> argumentIdDeclaringDefintion = new ArrayList<>();
+
+			// Getting the arguments from the definition
+			if (declaringArguments != null) {
+				for (exprType declaredArgument : declaringArguments) {
+					if (declaredArgument instanceof Name) {
+						Name argumentName = (Name) declaredArgument;
+						argumentIdDeclaringDefintion.add(argumentName.id);
+					}
+				}
+			}
+
+			return argumentIdDeclaringDefintion;
+		}
+
 		/**
 		 * True iff this {@link Function}'s {@link decoratorsType} has parameter autograph.
 		 *

edu.cuny.hunter.hybridize.core/src/edu/cuny/hunter/hybridize/core/analysis/Util.java

@@ -35,24 +35,19 @@ public class Util {
 	private static final ILog LOG = getLog(Util.class);
 
 	/**
-	 * Get the name of the module defining the entity described in the given {@link PySelection}.
+	 * Get the set of potential declaring definitions of the entity described in the given {@link PySelection}.
 	 *
 	 * @param selection The {@link PySelection} in question.
 	 * @param containingModName The name of the module containing the {@link PySelection}.
 	 * @param containingFile The {@link File} containing the module.
 	 * @param nature The {@link IPythonNature} to use.
 	 * @param monitor The IProgressMonitor to use.
-	 * @return The name of the module defining the given {@link PySelection}.
+	 * @return The definition of {@link PySelection}.
 	 * @throws AmbiguousDeclaringModuleException On ambiguous definitions found.
 	 * @throws BadLocationException On a parsing error.
 	 */
-	public static String getDeclaringModuleName(PySelection selection, String containingModName, File containingFile, IPythonNature nature,
-			IProgressMonitor monitor) throws BadLocationException, AmbiguousDeclaringModuleException {
-		monitor.beginTask("Getting declaring module name.", 1);
-
-		LOG.info(String.format("Getting declaring module name for selection: %s in line: %s, module: %s, file: %s, and project: %s.",
-				selection.getSelectedText(), selection.getLineWithoutCommentsOrLiterals().strip(), containingModName, containingFile,
-				nature.getProject()));
+	public static Set<Definition> getDeclaringDefinition(PySelection selection, String containingModName, File containingFile,
+			IPythonNature nature, IProgressMonitor monitor) throws BadLocationException, AmbiguousDeclaringModuleException {
 
 		RefactoringRequest request = new RefactoringRequest(containingFile, selection, nature);
 
@@ -77,14 +72,49 @@ public static String getDeclaringModuleName(PySelection selection, String contai
 							selection.getSelectedText(), selection.getLineWithoutCommentsOrLiterals().strip(), containingModName,
 							containingFile.getName(), nature.getProject()));
 
-		// Collect the potential declaring module names.
-		Set<String> potentialDeclaringModuleNames = new HashSet<>();
+		// Collect the potential declaring definition names.
+		Set<Definition> potentialDeclaringDefinitions = new HashSet<>();
 
 		// for each match.
 		for (ItemPointer itemPointer : pointers) {
 			Definition definition = itemPointer.definition;
 			LOG.info("Found definition: " + definition + ".");
 
+			// add it to the set of found definition names.
+			potentialDeclaringDefinitions.add(definition);
+		}
+
+		return potentialDeclaringDefinitions;
+	}
+
+	/**
+	 * Get the name of the module defining the entity described in the given {@link PySelection}.
+	 *
+	 * @param selection The {@link PySelection} in question.
+	 * @param containingModName The name of the module containing the {@link PySelection}.
+	 * @param containingFile The {@link File} containing the module.
+	 * @param nature The {@link IPythonNature} to use.
+	 * @param monitor The IProgressMonitor to use.
+	 * @return The name of the module defining the given {@link PySelection}.
+	 * @throws AmbiguousDeclaringModuleException On ambiguous definitions found.
+	 * @throws BadLocationException On a parsing error.
+	 */
+	public static String getDeclaringModuleName(PySelection selection, String containingModName, File containingFile, IPythonNature nature,
+			IProgressMonitor monitor) throws BadLocationException, AmbiguousDeclaringModuleException {
+		monitor.beginTask("Getting declaring module name.", 1);
+
+		LOG.info(String.format("Getting declaring module name for selection: %s in line: %s, module: %s, file: %s, and project: %s.",
+				selection.getSelectedText(), selection.getLineWithoutCommentsOrLiterals().strip(), containingModName, containingFile,
+				nature.getProject()));
+
+		Set<Definition> definitions = getDeclaringDefinition(selection, containingModName, containingFile, nature, monitor);
+
+		// Collect the potential declaring module names.
+		Set<String> potentialDeclaringModuleNames = new HashSet<>();
+
+		// for each definition.
+		for (Definition definition : definitions) {
+
 			IModule module = definition.module;
 			LOG.info(String.format("Found module: %s.", module));
 

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/A.py

@@ -0,0 +1,10 @@
+import tensorflow as tf
+
+
+@tf.function
+def test():
+    pass
+
+
+if __name__ == '__main__':
+    test()

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters/in/requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.9.3

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/A.py

@@ -0,0 +1,14 @@
+import tensorflow as tf
+
+# experimental_compile cannot be True or False because you get the following error ValueError: Cannot specify both 'experimental_compile' and 'jit_compile'.
+# That is why I put the value as None.
+
+
+@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None)
+def test(x):
+    return x
+
+
+if __name__ == '__main__':
+    number = tf.constant([1.0])
+    print(test(number))

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters10/in/requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.9.3

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/A.py

@@ -0,0 +1,9 @@
+import tensorflow as tf
+
+@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, True, "google.matmul_low_rank_matrix", tf.autograph.experimental.Feature.EQUALITY_OPERATORS, True, None, False)
+def test(x: tf.Tensor):
+    return x
+
+if __name__ == '__main__':
+  number = tf.constant([1.0, 1.0])
+  print(test(number))

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters11/in/requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.9.3

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/A.py

@@ -0,0 +1,10 @@
+import tensorflow as tf
+
+
+@tf.function(None, None, True, None, False, None, None, None, None, None)
+def test():
+    pass
+
+
+if __name__ == '__main__':
+    test()

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters12/in/requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.9.3

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/A.py

@@ -0,0 +1,11 @@
+import tensorflow as tf
+
+
+@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), False, True, "google.matmul_low_rank_matrix")
+def test(x):
+    return x
+
+
+if __name__ == '__main__':
+  number = tf.constant([1.0, 1.0])
+  test(number)

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters13/in/requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.8.0

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/A.py

@@ -0,0 +1,11 @@
+import tensorflow as tf
+
+
+@tf.function(None, (tf.TensorSpec(shape=[None], dtype=tf.float32),), autograph=True)
+def test(x):
+    return x
+
+
+if __name__ == '__main__':
+  number = tf.constant([1.0, 1.0])
+  test(number)

edu.cuny.hunter.hybridize.tests/resources/HybridizeFunction/testPositionalParameters14/in/requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.9.3