[devtools] Got pyannotate running again

Had to download the tarballs from PyPI, because pip (and venv) don't
work in our Python 2 build.  (And PyPI for Python 2 might not work
either.)

Fix some basic type errors.

The shell script is able to apply types, but I haven't done that yet.

devtools/types-old/run.sh → devtools/pyann.sh

@@ -85,18 +85,98 @@ add-imports() {
 # This has a bug
 #pyannotate() { ~/.local/bin/pyannotate "$@"; }
 
-readonly PYANN_REPO=~/git/oilshell/pyannotate/
+readonly PYANN_REPO=~/git/oils-for-unix/pyannotate/
+
+VENV=_tmp/pyann-venv
+
+make-venv() {
+  python3 -m venv $VENV
+}
+
+install-deps() {
+  . $VENV/bin/activate
+  python3 -m pip install -r $PYANN_REPO/requirements.txt
+}
 
 pyann-patched() {
+  . $VENV/bin/activate
   local tool=$PYANN_REPO/pyannotate_tools/annotations
-  export PYTHONPATH=$PYANN_REPO
+  #export PYTHONPATH=$PYANN_REPO:vendor
+
   # --dump can help
-  python $tool "$@"
+  python3 $tool "$@"
+}
+
+
+#
+# Second try
+#
+
+VENV2=_tmp/pyann-venv2
+
+make-venv2() {
+  python3 -m venv $VENV2
+}
+
+install2() {
+  . $VENV2/bin/activate
+  python3 -m pip install pyannotate
 }
 
+tool-demo2() {
+  . $VENV2/bin/activate
+  python3 -m pyannotate_tools.annotations
+}
+
+lib-demo2() {
+  . $VENV2/bin/activate
+  #echo $PYTHONPATH
+
+  # DOES NOT WORK - this is Python 2 code!!!
+  python3 devtools/pyann_driver.py "$@"
+
+  ls -l type_info.json
+  wc -l type_info.json
+}
+
+#
+# Third try - the problem is python2
+#
+
+deps3() {
+  # Gah my custom python2 build doesn't have pip or venv!
+  python2 -m pip install -r $PYANN_REPO/requirements.txt
+}
+
+# September 2019
+PYANN_URL='https://files.pythonhosted.org/packages/0d/26/2f68c02fae0b88d9cefdbc632edad190d61621b5660c72c920be1e52631e/pyannotate-1.2.0.tar.gz'
+
+# October 2019
+MYPY_EXT_URL='https://files.pythonhosted.org/packages/63/60/0582ce2eaced55f65a4406fc97beba256de4b7a95a0034c6576458c6519f/mypy_extensions-0.4.3.tar.gz'
+
+# December 2024
+SIX_URL='https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz'
+
+download-tarballs() {
+  wget --no-clobber --directory _tmp \
+    $PYANN_URL $MYPY_EXT_URL $SIX_URL
+}
+
+PY_PATH_2='.:vendor:_tmp/pyannotate-1.2.0:_tmp/mypy_extensions-0.4.3:_tmp/six-1.17.0'
+
 collect-types() {
-  export PYTHONPATH=".:$PYANN_REPO"
-  types/pyann_driver.py "$@"
+  # syntax error?
+  # Now it requires python3
+  # I think we need an old release
+  # https://pypi.org/project/mypy-extensions/
+  # https://github.com/python/mypy_extensions
+  # TypedDict
+  # https://github.com/python/mypy_extensions/commit/e0c6670e05a87507d59b7d3a0aa2eec88e9813b0
+
+  #local ext=~/git/oils-for-unix/mypy_extensions
+  #export PYTHONPATH=".:$PYANN_REPO:$ext"
+
+  PYTHONPATH=$PY_PATH_2 devtools/pyann_driver.py "$@"
 
   ls -l type_info.json
   wc -l type_info.json
@@ -133,11 +213,17 @@ peek-type-info() {
 apply-types() {
   local json=${1:-type_info.json}
   shift
-  local -a files=(osh/builtin_comp.py core/completion.py)
+  #local -a files=(osh/builtin_comp.py core/completion.py)
+  local -a files=(lazylex/*.py doctools/*.py)
 
   #local -a files=( $(cat _tmp/osh-parse-src.txt | grep -v syntax_asdl.py ) )
 
-  pyann-patched --type-info $json "${files[@]}" "$@"
+  # Use -w to write files
+  set -x
+  PYTHONPATH=$PY_PATH_2 \
+    python2 -m pyannotate_tools.annotations --type-info $json "${files[@]}" "$@"
+
+  #pyann-patched --type-info $json "${files[@]}" "$@"
 }
 
 apply-many() {

devtools/types-old/pyann_driver.py → devtools/pyann_driver.py

@@ -58,9 +58,10 @@ def main():
   loader = unittest.TestLoader()
 
   g = glob.glob
-  py = g('frontend/*_test.py') + g('osh/*_test.py') + g('core/*_test.py') + g('')
+  py = g('lazylex/*_test.py') + g('doctools/*_test.py')
+  #py = g('frontend/*_test.py') + g('osh/*_test.py') + g('core/*_test.py') + g('')
   # hangs
-  py.remove('core/process_test.py')
+  #py.remove('core/process_test.py')
 
   modules = []
   for p in py:

doctools/cmark.py

@@ -10,7 +10,8 @@
 try:
     from HTMLParser import HTMLParser
 except ImportError:
-    from html.parser import HTMLParser  # python3
+    # python3
+    from html.parser import HTMLParser  # type: ignore
 import json
 import optparse
 import os

doctools/oils_doc.py

@@ -15,7 +15,8 @@
 try:
     from cStringIO import StringIO
 except ImportError:
-    from io import StringIO  # python3
+    # for python3
+    from io import StringIO  # type: ignore
 import re
 import sys
 

doctools/ul_table.py

@@ -4,7 +4,7 @@
 try:
     from cStringIO import StringIO
 except ImportError:
-    from io import StringIO
+    from io import StringIO  # type: ignore
 import re
 import sys
 

lazylex/html.py

@@ -9,7 +9,8 @@
 try:
     from cStringIO import StringIO
 except ImportError:
-    from io import StringIO  # python3
+    # for python3
+    from io import StringIO  # type: ignore
 import re
 import sys
 
@@ -188,7 +189,7 @@ def MakeLexer(rules):
     (r'&', Tok.BadAmpersand),
 ]
 
-LEXER = CHAR_LEX + [
+HTM8_LEX = CHAR_LEX + [
     (r'<!--', Tok.CommentBegin),
 
     # Processing instruction are used for the XML header:
@@ -251,7 +252,7 @@ def MakeLexer(rules):
 #(r'<!-- [\s\S]*? -->', Tok.Comment),
 #(r'<!-- (?:.|[\n])*? -->', Tok.Comment),
 
-LEXER = MakeLexer(LEXER)
+HTM8_LEX_COMPILED = MakeLexer(HTM8_LEX)
 
 
 class Lexer(object):
@@ -301,7 +302,7 @@ def _Peek(self):
         # Note: frontend/match.py uses _LongestMatch(), which is different!
         # TODO: reconcile them.  This lexer should be expressible in re2c.
 
-        for pat, tok_id in LEXER:
+        for pat, tok_id in HTM8_LEX_COMPILED:
             m = pat.match(self.s, self.pos)
             if m:
                 if tok_id in (Tok.StartTag, Tok.EndTag, Tok.StartEndTag):
@@ -354,14 +355,14 @@ def TagNameEquals(self, expected):
         return expected == self.CanonicalTagName()
 
     def _LiteralTagName(self):
-        # type: () -> None
+        # type: () -> str
         assert self.tag_pos_left != -1, self.tag_pos_left
         assert self.tag_pos_right != -1, self.tag_pos_right
 
         return self.s[self.tag_pos_left:self.tag_pos_right]
 
     def CanonicalTagName(self):
-        # type: () -> None
+        # type: () -> str
         tag_name = self._LiteralTagName()
         # Most tags are already lower case, so avoid allocation with this conditional
         # TODO: this could go in the mycpp runtime?