From 3b244f9b9f0002c6860ad329d63e36514e9f2205 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 20 Jul 2024 10:56:57 -0400 Subject: [PATCH 001/506] [sourcehut] Renamed repo to 'oils' Update build YAML --- .builds/dummy_orig.yml_disabled | 4 ++-- .builds/worker1.yml_disabled | 26 +++++++++++++------------- .builds/worker2.yml | 22 +++++++++++----------- .builds/worker3.yml | 18 +++++++++--------- .builds/worker4.yml_disabled | 14 +++++++------- .builds/worker5.yml | 6 +++--- .builds/worker6.yml | 6 +++--- 7 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.builds/dummy_orig.yml_disabled b/.builds/dummy_orig.yml_disabled index 655a525a54..d49f582bbd 100644 --- a/.builds/dummy_orig.yml_disabled +++ b/.builds/dummy_orig.yml_disabled @@ -8,8 +8,8 @@ secrets: - 2678474d-b22b-449f-a19a-16cb403c94cd tasks: - dummy: | - cd oil + cd oils soil/worker.sh JOB-dummy - publish-html: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key diff --git a/.builds/worker1.yml_disabled b/.builds/worker1.yml_disabled index 594bead79a..f9b66fbe11 100644 --- a/.builds/worker1.yml_disabled +++ b/.builds/worker1.yml_disabled @@ -20,56 +20,56 @@ secrets: tasks: - mount-perms: | - cd oil + cd oils soil/host-shim.sh mount-perms $PWD - dummy: | - cd oil + cd oils soil/host-shim.sh run-job-uke podman $PWD dummy # Relies on SSH key, so do it outside the container - publish-dummy: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key dummy - job-reset-1: | - cd oil + cd oils soil/host-shim.sh job-reset - pea: | - cd oil + cd oils soil/host-shim.sh run-job-uke podman $PWD pea - publish-pea: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key pea - job-reset-2: | - cd oil + cd oils soil/host-shim.sh job-reset - cpp-small: | - cd oil + cd oils soil/host-shim.sh run-job-uke podman $PWD cpp-small - publish-cpp-small: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key cpp-small # - cpp-coverage: | - # cd oil + # cd oils # soil/host-shim.sh run-job-uke podman $PWD cpp-coverage # # - publish-cpp-coverage: | - # cd oil + # cd oils # soil/sourcehut.sh publish-html-assuming-ssh-key # # - job-reset-2: | - # cd oil + # cd oils # soil/host-shim.sh job-reset - did-all-succeed: | - cd oil + cd oils soil/host-shim.sh did-all-succeed dummy pea cpp-small #soil/host-shim.sh did-all-succeed pea cpp-coverage dummy diff --git a/.builds/worker2.yml b/.builds/worker2.yml index 13e0777b8d..6e19771550 100644 --- a/.builds/worker2.yml +++ b/.builds/worker2.yml @@ -17,45 +17,45 @@ secrets: tasks: - mount-perms: | - cd oil + cd oils soil/host-shim.sh mount-perms $PWD - cpp-tarball: | - cd oil + cd oils soil/host-shim.sh run-job-uke podman $PWD cpp-tarball - publish-cpp-tarball: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key cpp-tarball - job-reset-1: | - cd oil + cd oils soil/host-shim.sh job-reset - benchmarks: | - cd oil + cd oils soil/host-shim.sh run-job-uke podman $PWD benchmarks - publish-benchmarks: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key benchmarks #- cpp-spec: | - #cd oil + #cd oils #soil/host-shim.sh run-job-uke podman $PWD cpp-spec #- publish-cpp-spec: | - #cd oil + #cd oils #soil/sourcehut.sh publish-html-assuming-ssh-key cpp-spec #- other-tests: | - # cd oil + # cd oils # soil/host-shim.sh run-job-uke podman $PWD other-tests #- publish-other-tests: | - # cd oil + # cd oils # soil/sourcehut.sh publish-html-assuming-ssh-key other-tests - did-all-succeed: | - cd oil + cd oils soil/host-shim.sh did-all-succeed cpp-tarball benchmarks diff --git a/.builds/worker3.yml b/.builds/worker3.yml index 456bd67000..b59ab878bd 100644 --- a/.builds/worker3.yml +++ b/.builds/worker3.yml @@ -23,40 +23,40 @@ secrets: tasks: - dev-setup-debian: | - cd oil + cd oils soil/worker.sh JOB-dev-setup-debian - publish-dev-setup-debian: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key dev-setup-debian #- mount-perms: | - # cd oil + # cd oils # soil/host-shim.sh mount-perms $PWD #- dev-minimal: | - # cd oil + # cd oils # soil/host-shim.sh run-job-uke podman $PWD dev-minimal ## Relies on SSH key, so do it outside the container #- publish-dev-minimal: | - # cd oil + # cd oils # soil/sourcehut.sh publish-html-assuming-ssh-key dev-minimal # Outside container #- job-reset: | - # cd oil + # cd oils # soil/host-shim.sh job-reset #- other-tests: | - # cd oil + # cd oils # soil/host-shim.sh run-job-uke podman $PWD other-tests #- publish-other-tests: | - # cd oil + # cd oils # soil/sourcehut.sh publish-html-assuming-ssh-key # - did-all-succeed: | - # cd oil + # cd oils # soil/host-shim.sh did-all-succeed dev-minimal # #soil/host-shim.sh did-all-succeed dev-minimal other-tests diff --git a/.builds/worker4.yml_disabled b/.builds/worker4.yml_disabled index 5d1f119fd3..9037340d4f 100644 --- a/.builds/worker4.yml_disabled +++ b/.builds/worker4.yml_disabled @@ -20,31 +20,31 @@ secrets: tasks: - mount-perms: | - cd oil + cd oils soil/host-shim.sh mount-perms $PWD #- ovm-tarball: | - # cd oil + # cd oils # soil/host-shim.sh run-job-uke podman $PWD ovm-tarball ## Relies on SSH key, so do it outside the container #- publish-ovm-tarball: | - # cd oil + # cd oils # soil/sourcehut.sh publish-html-assuming-ssh-key #- job-reset-2: | - # cd oil + # cd oils # soil/host-shim.sh job-reset - benchmarks2: | - cd oil + cd oils soil/host-shim.sh run-job-uke podman $PWD benchmarks2 - publish-benchmarks2: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key benchmarks2 - did-all-succeed: | - cd oil + cd oils soil/host-shim.sh did-all-succeed benchmarks2 #soil/host-shim.sh did-all-succeed ovm-tarball benchmarks2 diff --git a/.builds/worker5.yml b/.builds/worker5.yml index 59f06a985e..ae0fcd4179 100644 --- a/.builds/worker5.yml +++ b/.builds/worker5.yml @@ -17,13 +17,13 @@ secrets: tasks: - dev-setup-fedora: | - cd oil + cd oils soil/worker.sh JOB-dev-setup-fedora - publish-dev-setup-fedora: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key dev-setup-fedora - did-all-succeed: | - cd oil + cd oils soil/host-shim.sh did-all-succeed dev-setup-fedora diff --git a/.builds/worker6.yml b/.builds/worker6.yml index 80c353b586..14ed892f61 100644 --- a/.builds/worker6.yml +++ b/.builds/worker6.yml @@ -16,13 +16,13 @@ secrets: tasks: - dev-setup-alpine: | - cd oil + cd oils soil/worker.sh JOB-dev-setup-alpine - publish-dev-setup-alpine: | - cd oil + cd oils soil/sourcehut.sh publish-html-assuming-ssh-key dev-setup-alpine - did-all-succeed: | - cd oil + cd oils soil/host-shim.sh did-all-succeed dev-setup-alpine From 5b1a84f8aadbbb794c48852e440fb8fddaed5c65 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 20 Jul 2024 11:35:22 -0400 Subject: [PATCH 002/506] [README] Renamed repo to oils-for-unix/oils --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index aa59531f0c..0977f8dc56 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Oils Source Code ================ [![Build -Status](https://github.com/oilshell/oil/actions/workflows/all-builds.yml/badge.svg)](https://github.com/oilshell/oil/actions/workflows/all-builds.yml) +Status](https://github.com/oils-for-unix/oils/actions/workflows/all-builds.yml/badge.svg)](https://github.com/oils-for-unix/oils/actions/workflows/all-builds.yml) Contribute with Gitpod @@ -32,7 +32,7 @@ The deployed executable doesn't depend on Python. This README is at the root of the [git repo][git-repo]. -[git-repo]: https://github.com/oilshell/oil +[git-repo]: https://github.com/oils-for-unix/oils
@@ -45,10 +45,10 @@ This README is at the root of the [git repo][git-repo]. * If it doesn't, let us know. You can post on the `#oil-dev` channel of [oilshell.zulipchat.com][], or file an issue on Github. * Feel free to grab an [issue from - Github](https://github.com/oilshell/oil/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). + Github](https://github.com/oils-for-unix/oils/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). Let us know what you're thinking before you get too far. -[Contributing]: https://github.com/oilshell/oil/wiki/Contributing +[Contributing]: https://github.com/oils-for-unix/oils/wiki/Contributing [oilshell.zulipchat.com]: https://oilshell.zulipchat.com/ [blog]: https://www.oilshell.org/blog/ @@ -96,7 +96,7 @@ It's great for prototyping. step, although it often just works. - You can **influence the design** of [YSH][]. If you have an itch to scratch, be ambitious. For example, you might want to show us how to - implement [nonlinear pipelines](https://github.com/oilshell/oil/issues/843). + implement [nonlinear pipelines](https://github.com/oils-for-unix/oils/issues/843). ### I aim for 24 hour response time @@ -113,7 +113,7 @@ Thank you for the contributions! ### Docs -The [Wiki](https://github.com/oilshell/oil/wiki) has many developer docs. Feel +The [Wiki](https://github.com/oils-for-unix/oils/wiki) has many developer docs. Feel free to edit them. If you make a major change, let us know on Zulip! There are also READMEs in some subdirectories, like `opy/` and `mycpp/`. From f5861cb6804bdaca8a83f12a684a31629ff28480 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 20 Jul 2024 13:17:15 -0400 Subject: [PATCH 003/506] [soil] Update maybe-merge with new repo name --- soil/maybe-merge.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soil/maybe-merge.sh b/soil/maybe-merge.sh index 1e8fd89223..cab1bb2c2e 100755 --- a/soil/maybe-merge.sh +++ b/soil/maybe-merge.sh @@ -57,7 +57,7 @@ fast-forward() { -H "Content-Type: application/json" \ -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: token ${github_token}" \ - https://api.github.com/repos/oilshell/oil/git/refs/heads/$to_branch \ + https://api.github.com/repos/oils-for-unix/oils/git/refs/heads/$to_branch \ -d '{"sha": "'$commit_hash'", "force": false }' local error From f73ca194669901e38505169c4acd106aab82ad56 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 20 Jul 2024 18:33:20 -0400 Subject: [PATCH 004/506] [soil] Try to fix missing status-api entries The number of digits in a github job ID changed! --- soil/web-worker.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/soil/web-worker.sh b/soil/web-worker.sh index 1fe06ae7e3..414f22f29f 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -62,7 +62,9 @@ remote-cleanup-jobs-index() { } remote-cleanup-status-api() { - sshq soil-web/soil/web.sh cleanup-status-api false + #sshq soil-web/soil/web.sh cleanup-status-api false + # 2024-07 - work around bug. The logic in soil/web.sh doesn't seem right + sshq soil-web/soil/web.sh cleanup-status-api true } my-scp() { From 5ad7d6e4fd672b84f7ea107461133243fce14669 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 21 Jul 2024 23:08:06 -0400 Subject: [PATCH 005/506] [spec/ysh-json] Failing test cases Including for bug #2026 Also fix blocking bug in test/sh_spec.py! --- spec/ysh-json.test.sh | 63 ++++++++++++++++++++++++++++++++++++++++++- test/sh_spec.py | 6 +---- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index a8a0879809..d66e1248fa 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 4 ## tags: dev-minimal #### usage errors @@ -1078,3 +1078,64 @@ status=1 status=1 ## END +#### Data after internal NUL (issue #2026) + +$SH <<'EOF' +pp line (fromJson(b'123\y00abc')) +EOF +echo status=$? + +$SH <<'EOF' +pp line (fromJson(b'123\y01abc')) +EOF +echo status=$? + +$SH <<'EOF' +shopt --set ysh:upgrade # b'' syntax +json read <<< b'123\y00abc' +EOF +echo status=$? + +$SH <<'EOF' +shopt --set ysh:upgrade # b'' syntax +json read <<< b'123\y01abc' +EOF +echo status=$? + +## STDOUT: +status=4 +status=4 +status=1 +status=1 +## END + +#### Number too big + +$SH <<'EOF' +json read <<< '123456789123456789123456789' +pp line (_reply) +EOF +echo status=$? + +$SH <<'EOF' +json read <<< '-123456789123456789123456789' +pp line (_reply) +EOF +echo status=$? + +## STDOUT: +status=1 +status=1 +## END + +#### Too many opening [[[ - blocking stack + +python2 -c 'print("[" * 10000)' | json read +pp line (_reply) + +python2 -c 'print("{" * 10000)' | json read +pp line (_reply) + +## STDOUT: +## END + diff --git a/test/sh_spec.py b/test/sh_spec.py index 1666f3e053..51a0a1fd9a 100755 --- a/test/sh_spec.py +++ b/test/sh_spec.py @@ -728,13 +728,9 @@ def RunCases(cases, case_predicate, shells, env, out, opts): sys.exit(1) p.stdin.write(code) - p.stdin.close() actual = {} - actual['stdout'] = p.stdout.read() - actual['stderr'] = p.stderr.read() - p.stdout.close() - p.stderr.close() + actual['stdout'], actual['stderr'] = p.communicate() actual['status'] = p.wait() From 4e6f61c890b67c67557993e02245aaff01191e78 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 22 Jul 2024 01:06:07 -0400 Subject: [PATCH 006/506] [data_lang/j8] Handle error from overflowing mops::BigInt Right now the spec is 64 bits. So we mark C++ as passing and Python failing. But we will change this when we have big integers. [doc/ref] Document implementation defined limits for JSON/J8 --- data_lang/j8.py | 6 ++- doc/ref/chap-errors.md | 19 +++++++++ spec/ysh-json.test.sh | 96 +++++++++++++++++++++++++++--------------- 3 files changed, 85 insertions(+), 36 deletions(-) diff --git a/data_lang/j8.py b/data_lang/j8.py index f9cadc4b42..8a6bfd4006 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -884,7 +884,11 @@ def _ParseValue(self): elif self.tok_id == Id.J8_Int: part = self.s[self.start_pos:self.end_pos] self._Next() - return value.Int(mops.FromStr(part)) + try: + big = mops.FromStr(part) + except ValueError: + raise self._ParseError('Integer is too big') + return value.Int(big) elif self.tok_id == Id.J8_Float: part = self.s[self.start_pos:self.end_pos] diff --git a/doc/ref/chap-errors.md b/doc/ref/chap-errors.md index 2457b2913a..8d4f6bed8f 100644 --- a/doc/ref/chap-errors.md +++ b/doc/ref/chap-errors.md @@ -75,6 +75,12 @@ are **no encoding errors**. 1. Byte escapes like `\yff` should not be in `u''` string. - By design, they're only valid in `b''` strings. +Implementation-defined limit: + +4. Max string length (NYI) + - e.g. more than 4 billion bytes could overflow a length field, in some + implementations + ## J8 Lines Roughly speaking, J8 Lines are an encoding for a stream of J8 strings. In @@ -124,6 +130,19 @@ character, not a hard error. 1. Unexpected trailing input - like the message `42]` or `{}]` +Implementation-defined limits, i.e. outside the grammar: + +5. Integer too big + - implementations may decode to a 64-bit integer +1. Floats that are too big + - may decode to `Inf` +1. Max array length (NYI) + - e.g. more than 4 billion objects in an array could overflow a length + field, in some implementations +1. Max object length (NYI) +1. Max depth for arrays and objects (NYI) + - to avoid a recursive parser blowing the stack + ## JSON8 ### err-json8-encode diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index d66e1248fa..9edcd60eb7 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -964,48 +964,42 @@ echo status=$? status=1 ## END -#### decode deeply nested structure (stack overflow) - -shopt -s ysh:upgrade - -proc pairs(n) { - var m = int(n) # TODO: 1 .. n should auto-convert? - - for i in (1 .. m) { - write -n -- '[' - } - for i in (1 .. m) { - write -n -- ']' - } -} - -# This is all Python can handle; C++ can handle more -msg=$(pairs 50) +#### decode integer larger than 2^32 -#echo $msg +json=$(( 1 << 33 )) +echo $json -echo "$msg" | json read +echo $json | json read pp line (_reply) -echo len=$[len(_reply)] ## STDOUT: -(List) [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] -len=1 +8589934592 +(Int) 8589934592 ## END -#### decode integer larger than 2^32 +#### decode integer larger than 2^64 -json=$(( 1 << 33 )) -echo $json +$SH <<'EOF' +json read <<< '123456789123456789123456789' +echo status=$? +pp line (_reply) +EOF -echo $json | json read +$SH <<'EOF' +json read <<< '-123456789123456789123456789' +echo status=$? pp line (_reply) +EOF + +echo ok ## STDOUT: -8589934592 -(Int) 8589934592 +status=1 +status=1 +ok ## END + #### round trip: read/write with ysh var file = "$REPO_ROOT/spec/testdata/bug.json" @@ -1109,25 +1103,57 @@ status=1 status=1 ## END -#### Number too big +#### Float too big $SH <<'EOF' -json read <<< '123456789123456789123456789' +json read <<< '123456789123456789123456789.12345e67890' +echo status=$? pp line (_reply) EOF -echo status=$? $SH <<'EOF' -json read <<< '-123456789123456789123456789' +json read <<< '-123456789123456789123456789.12345e67890' +echo status=$? pp line (_reply) EOF -echo status=$? ## STDOUT: -status=1 -status=1 +status=0 +(Float) inf +status=0 +(Float) -inf +## END + +#### Many [[[ , but not too many + +shopt -s ysh:upgrade + +proc pairs(n) { + var m = int(n) # TODO: 1 .. n should auto-convert? + + for i in (1 .. m) { + write -n -- '[' + } + for i in (1 .. m) { + write -n -- ']' + } +} + +# This is all Python can handle; C++ can handle more +msg=$(pairs 50) + +#echo $msg + +echo "$msg" | json read +pp line (_reply) +echo len=$[len(_reply)] + +## STDOUT: +(List) [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +len=1 ## END + #### Too many opening [[[ - blocking stack python2 -c 'print("[" * 10000)' | json read From 0d66b140ca15ea4135c80ced727cb2301dfd8b9f Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 22 Jul 2024 01:52:24 -0400 Subject: [PATCH 007/506] [data_lang/j8] Fix trailing input check It's now based on the input length, not Id.Eol_Tok. This is issue #2026, found by Ellen Potter, via running JSONTestSuite: https://github.com/nst/JSONTestSuite --- data_lang/j8.py | 9 +++++++-- spec/ysh-json.test.sh | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/data_lang/j8.py b/data_lang/j8.py index 8a6bfd4006..72316fb7d8 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -915,8 +915,13 @@ def ParseValue(self): """ Raises error.Decode. """ self._Next() obj = self._ParseValue() - if self.tok_id != Id.Eol_Tok: - raise self._ParseError('Unexpected trailing input') + + n = len(self.s) + if self.start_pos != n: + extra = n - self.start_pos + #log('n %d pos %d', n, self.start_pos) + raise self._ParseError( + 'Got %d bytes of unexpected trailing input' % extra) return obj diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index 9edcd60eb7..b03c2be111 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 4 +## oils_failures_allowed: 3 ## tags: dev-minimal #### usage errors From 2a83a7cdc8c181e6c6c3af619e3f0cb3b4d81e41 Mon Sep 17 00:00:00 2001 From: Aidan <46799759+PossiblyAShrub@users.noreply.github.com> Date: Tue, 23 Jul 2024 23:22:14 -0600 Subject: [PATCH 008/506] [mycpp/runtime] Fix pointer arith bug in List::Pop(int) (#2031) --- mycpp/gc_list.h | 2 +- mycpp/gc_list_test.cc | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mycpp/gc_list.h b/mycpp/gc_list.h index a3524af424..165346be1f 100644 --- a/mycpp/gc_list.h +++ b/mycpp/gc_list.h @@ -342,7 +342,7 @@ T List::pop(int i) { len_--; // Shift everything by one - memmove(slab_->items_ + i, slab_->items_ + (i + 1), len_ * sizeof(T)); + memmove(slab_->items_ + i, slab_->items_ + (i + 1), (len_ - i) * sizeof(T)); /* for (int j = 0; j < len_; j++) { diff --git a/mycpp/gc_list_test.cc b/mycpp/gc_list_test.cc index f85add6ca1..41eacc4c45 100644 --- a/mycpp/gc_list_test.cc +++ b/mycpp/gc_list_test.cc @@ -463,6 +463,21 @@ TEST test_list_remove() { PASS(); } +TEST test_list_pop_mem_safe() { + auto l = NewList(); + + // List::pop(int) had a memory bug where it would buffer overflow due to a + // mistake when calling memmove. To reproduce, the list had to be at least 16 + // items long, otherwise ASAN will not catch the error. + for (int i = 0; i < 16; ++i) { + l->append(i); + } + + l->pop(15); // This would cause a buffer overflow + + PASS(); +} + GREATEST_MAIN_DEFS(); int main(int argc, char** argv) { @@ -484,6 +499,8 @@ int main(int argc, char** argv) { RUN_TEST(test_list_sort); RUN_TEST(test_list_remove); + RUN_TEST(test_list_pop_mem_safe); + gHeap.CleanProcessExit(); GREATEST_MAIN_END(); From d5a3a90818c714aad39922141931dd70463ee07f Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 24 Jul 2024 11:47:45 -0400 Subject: [PATCH 009/506] [stdlib/ysh] Implement repeat() on Str and List Like Python's 's' * 3 ['foo', 'bar'] * 3 [doc/ref] Re-organize docs. TODO: - Move everything to stdlib/ysh, so we source $LIB_YSH/math.ysh - we are deprecating source --builtin - Start using our test framework on the stdlib - Right now it uses the BYO protocol to list procs - which relies on compgen -A function! - We might want to have a separate compgen -A proc --- builtin/func_misc.py | 11 ---- core/shell.py | 1 - doc/ref/chap-builtin-func.md | 87 ----------------------------- doc/ref/chap-stdlib.md | 103 +++++++++++++++++++++++++++++++++++ doc/ref/toc-ysh.md | 14 +++-- spec/ysh-json.test.sh | 17 +++++- spec/ysh-stdlib.test.sh | 62 +++++++++++++++++++++ stdlib/list.ysh | 41 +++++++++++--- 8 files changed, 223 insertions(+), 113 deletions(-) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index fe71996dab..0191d09be6 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -74,17 +74,6 @@ def Call(self, rd): return value.Str(ui.ValType(val)) -class Repeat(vm._Callable): - - def __init__(self): - # type: () -> None - pass - - def Call(self, rd): - # type: (typed_args.Reader) -> value_t - return value.Null - - class Join(vm._Callable): """Both free function join() and List->join() method.""" diff --git a/core/shell.py b/core/shell.py index 373b37c4a8..5355b2d474 100644 --- a/core/shell.py +++ b/core/shell.py @@ -818,7 +818,6 @@ def Main( _SetGlobalFunc(mem, 'len', func_misc.Len()) _SetGlobalFunc(mem, 'type', func_misc.Type()) - _SetGlobalFunc(mem, 'repeat', func_misc.Repeat()) g = func_eggex.MatchFunc(func_eggex.G, expr_ev, mem) _SetGlobalFunc(mem, '_group', g) diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index b7f21f3bba..f2144946bc 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -233,35 +233,6 @@ It's also often called with the `=>` chaining operator: json write (items => join(' ')) # => "1 2 3" json write (items => join(', ')) # => "1, 2, 3" - -### any() - -Returns true if any value in the list is truthy (`x` is truthy if `Bool(x)` -returns true). - -If the list is empty, return false. - - = any([]) # => false - = any([true, false]) # => true - = any([false, false]) # => false - = any([false, "foo", false]) # => true - -Note, you will need to `source --builtin list.ysh` to use this function. - -### all() - -Returns true if all values in the list are truthy (`x` is truthy if `Bool(x)` -returns true). - -If the list is empty, return true. - - = any([]) # => true - = any([true, true]) # => true - = any([false, true]) # => false - = any(["foo", true, true]) # => true - -Note, you will need to `source --builtin list.ysh` to use this function. - ## Word ### glob() @@ -270,64 +241,6 @@ See `glob-pat` topic for syntax. ### maybe() -## Math - -### abs() - -Compute the absolute (positive) value of a number (float or int). - - = abs(-1) # => 1 - = abs(0) # => 0 - = abs(1) # => 1 - -Note, you will need to `source --builtin math.ysh` to use this function. - -### max() - -Compute the maximum of 2 or more values. - -`max` takes two different signatures: - - 1. `max(a, b)` to return the maximum of `a`, `b` - 2. `max(list)` to return the greatest item in the `list` - -For example: - - = max(1, 2) # => 2 - = max([1, 2, 3]) # => 3 - -Note, you will need to `source --builtin math.ysh` to use this function. - -### min() - -Compute the minimum of 2 or more values. - -`min` takes two different signatures: - - 1. `min(a, b)` to return the minimum of `a`, `b` - 2. `min(list)` to return the least item in the `list` - -For example: - - = min(2, 3) # => 2 - = max([1, 2, 3]) # => 1 - -Note, you will need to `source --builtin math.ysh` to use this function. - -### round() - -### sum() - -Computes the sum of all elements in the list. - -Returns 0 for an empty list. - - = sum([]) # => 0 - = sum([0]) # => 0 - = sum([1, 2, 3]) # => 6 - -Note, you will need to `source --builtin list.ysh` to use this function. - ## Serialize ### toJson() diff --git a/doc/ref/chap-stdlib.md b/doc/ref/chap-stdlib.md index 1677cbdce0..b5e37d7af8 100644 --- a/doc/ref/chap-stdlib.md +++ b/doc/ref/chap-stdlib.md @@ -22,6 +22,109 @@ for OSH and YSH.
+## math + +### abs() + +Compute the absolute (positive) value of a number (float or int). + + = abs(-1) # => 1 + = abs(0) # => 0 + = abs(1) # => 1 + +Note, you will need to `source --builtin math.ysh` to use this function. + +### max() + +Compute the maximum of 2 or more values. + +`max` takes two different signatures: + + 1. `max(a, b)` to return the maximum of `a`, `b` + 2. `max(list)` to return the greatest item in the `list` + +For example: + + = max(1, 2) # => 2 + = max([1, 2, 3]) # => 3 + +Note, you will need to `source --builtin math.ysh` to use this function. + +### min() + +Compute the minimum of 2 or more values. + +`min` takes two different signatures: + + 1. `min(a, b)` to return the minimum of `a`, `b` + 2. `min(list)` to return the least item in the `list` + +For example: + + = min(2, 3) # => 2 + = max([1, 2, 3]) # => 1 + +Note, you will need to `source --builtin math.ysh` to use this function. + +### round() + +TODO + +### sum() + +Computes the sum of all elements in the list. + +Returns 0 for an empty list. + + = sum([]) # => 0 + = sum([0]) # => 0 + = sum([1, 2, 3]) # => 6 + +Note, you will need to `source --builtin list.ysh` to use this function. + + +## list + +### all() + +Returns true if all values in the list are truthy (`x` is truthy if `Bool(x)` +returns true). + +If the list is empty, return true. + + = any([]) # => true + = any([true, true]) # => true + = any([false, true]) # => false + = any(["foo", true, true]) # => true + +Note, you will need to `source --builtin list.ysh` to use this function. + +### any() + +Returns true if any value in the list is truthy (`x` is truthy if `Bool(x)` +returns true). + +If the list is empty, return false. + + = any([]) # => false + = any([true, false]) # => true + = any([false, false]) # => false + = any([false, "foo", false]) # => true + +Note, you will need to `source --builtin list.ysh` to use this function. + +### repeat() + +Repeat a string or a list: + + = repeat('foo', 3) # => 'foofoofoo' + = repeat(['foo', 'bar'], 2) # => ['foo', 'bar', 'foo', 'bar'] + +Negative repetitions are equivalent to zero: + + = repeat('foo', -5) # => '' + = repeat(['foo', 'bar'], -5) # => [] + ## two These functions are in `two.sh` diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index ee198f9e58..c53047958c 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -72,15 +72,15 @@ X [Guts] heapId() ```chapter-links-builtin-func - [Values] len() func/type() X repeat() + [Values] len() func/type() [Conversions] bool() int() float() str() list() dict() X runes() X encodeRunes() X bytes() X encodeBytes() [Str] X strcmp() X split() shSplit() - [List] join() any() all() + [List] join() + [Float] X isinf() X isnan() [Collections] X copy() X deepCopy() [Word] glob() maybe() - [Math] abs() max() min() X round() sum() [Serialize] toJson() fromJson() toJson8() fromJson8() X [J8 Decode] J8.Bool() J8.Int() ... @@ -137,7 +137,13 @@ X [Testing] assert takes an expression ```chapter-links-stdlib_42 - [Args Parser] parser Parse command line arguments + [math] abs() + max() min() + X round() + sum() + [list] all() any() + repeat() + [args] parser Parse command line arguments flag arg rest diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index b03c2be111..e836fd5ba8 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -895,9 +895,21 @@ status=0 #### Inf and NaN can't be encoded or decoded # This works in Python, should probably support it +#var n = float("NaN") +#var i = float("inf") + +# WRONG LOCATION! Gah +#var x = fromJson(repeat('123', 20)) + +shopt --set ysh:upgrade + +source --builtin list.ysh + +var s = repeat('123', 20) +pp line (s) +var x = fromJson(s) +pp line (x) -var n = float("NaN") -var i = float("inf") pp line (n) pp line (i) @@ -907,6 +919,7 @@ json dump (i) ## status: 2 ## STDOUT: +fds ## END #### Invalid UTF-8 in JSON is rejected diff --git a/spec/ysh-stdlib.test.sh b/spec/ysh-stdlib.test.sh index ce15e4e297..068fca75c5 100644 --- a/spec/ysh-stdlib.test.sh +++ b/spec/ysh-stdlib.test.sh @@ -158,3 +158,65 @@ json write (sum([1, 2, 3])) 0 6 ## END + + +#### repeat() string + +source --builtin list.ysh + +echo three=$[repeat('foo', 3)] +echo zero=$[repeat('foo', 0)] +echo negative=$[repeat('foo', -1)] + +## STDOUT: +three=foofoofoo +zero= +negative= +## END + +#### repeat() list + +source --builtin list.ysh + +var L = ['foo', 'bar'] +echo three @[repeat(L, 3)] +echo zero @[repeat(L, 0)] +echo negative @[repeat(L, -1)] + +## STDOUT: +three foo bar foo bar foo bar +zero +negative +## END + +#### repeat() error + +try { + $SH -c ' + source --builtin list.ysh + pp line (repeat(null, 3)) + echo bad' +} +echo code=$[_error.code] + +try { + $SH -c ' + source --builtin list.ysh + pp line (repeat({}, 3)) + echo bad' +} +echo code=$[_error.code] + +try { + $SH -c ' + source --builtin list.ysh + pp line (repeat(42, 3)) + echo bad' +} +echo code=$[_error.code] + +## STDOUT: +code=10 +code=10 +code=10 +## END diff --git a/stdlib/list.ysh b/stdlib/list.ysh index 4aeee0d382..3c1767c2bf 100644 --- a/stdlib/list.ysh +++ b/stdlib/list.ysh @@ -1,7 +1,6 @@ func any(list) { - ## Returns true if any value in the list is truthy. - ## - ## If the list is empty, return false. + ### Returns true if any value in the list is truthy. + # Empty list: returns false for item in (list) { if (item) { @@ -13,8 +12,7 @@ func any(list) { func all(list) { ## Returns true if all values in the list are truthy. - ## - ## If the list is empty, return true. + # Empty list: returns true for item in (list) { if (not item) { @@ -25,9 +23,8 @@ func all(list) { } func sum(list; start=0) { - ## Computes the sum of all elements in the list. - ## - ## Returns 0 for an empty list. + ### Returns the sum of all elements in the list. + # Empty list: returns 0 var sum = start for item in (list) { @@ -35,3 +32,31 @@ func sum(list; start=0) { } return (sum) } + +func repeat(x, n) { + ### Returns a list with the given string or list repeated + + # Like Python's 'foo'*3 or ['foo', 'bar']*3 + # negative numbers are like 0 in Python + + var t = type(x) + case (t) { + Str { + var parts = [] + for i in (0 .. n) { + call parts->append(x) + } + return (join(parts)) + } + List { + var result = [] + for i in (0 .. n) { + call result->extend(x) + } + return (result) + } + (else) { + error "Expected Str or List, got $t" + } + } +} From 40147cd9e64f9a1ed4bfb5370746ad723728dfcf Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 24 Jul 2024 12:36:01 -0400 Subject: [PATCH 010/506] [ysh] Fix json serialization of NaN, Infinity We output null, like JavaScript does. Python erroneously prints a string, or it fails if allow_nan=False is passed. Add NAN and INFINITY constants -- using the C spelling for these values. --- core/pyutil.py | 14 ++++++++ core/state.py | 16 +++++++++ cpp/core.cc | 11 ++++++- cpp/core.h | 3 ++ cpp/stdlib.cc | 13 ++++++++ cpp/stdlib.h | 7 ++++ data_lang/j8.py | 27 ++++++++++----- data_lang/json-survey.sh | 2 ++ doc/ref/chap-special-var.md | 14 ++++++++ doc/ref/toc-ysh.md | 1 + spec/ysh-int-float.test.sh | 9 +++++ spec/ysh-json.test.sh | 66 +++++++++++++++++++++++++++---------- 12 files changed, 156 insertions(+), 27 deletions(-) diff --git a/core/pyutil.py b/core/pyutil.py index 005c52d81b..a587851dd8 100644 --- a/core/pyutil.py +++ b/core/pyutil.py @@ -19,6 +19,20 @@ _PUNCT = """!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" +def nan(): + # type: () -> float + + # note: Python 3 has math.nan + return float('nan') + + +def infinity(): + # type: () -> float + + # note: Python 3 has math.inf + return float('inf') + + def IsValidCharEscape(ch): # type: (str) -> bool """Is this a valid character escape when unquoted?""" diff --git a/core/state.py b/core/state.py index e534ed2c6d..1d598c5d3f 100644 --- a/core/state.py +++ b/core/state.py @@ -932,6 +932,16 @@ def InitMem(mem, environ, version_str): SetGlobalString(mem, 'LIB_OSH', '///osh') SetGlobalString(mem, 'LIB_YSH', '///ysh') + # - C spells it NAN + # - JavaScript spells it NaN + # - Python 2 has float('nan'), while Python 3 has math.nan. + # + # - libc prints the strings 'nan' and 'inf' + # - Python 3 prints the strings 'nan' and 'inf' + # - JavaScript prints 'NaN' and 'Infinity', which is more stylized + _SetGlobalValue(mem, 'NAN', value.Float(pyutil.nan())) + _SetGlobalValue(mem, 'INFINITY', value.Float(pyutil.infinity())) + _InitDefaults(mem) _InitVarsFromEnv(mem, environ) @@ -2371,6 +2381,12 @@ def SetGlobalArray(mem, name, a): mem.SetNamed(location.LName(name), value.BashArray(a), scope_e.GlobalOnly) +def _SetGlobalValue(mem, name, val): + # type: (Mem, str, value_t) -> None + """Helper for completion, etc.""" + mem.SetNamed(location.LName(name), val, scope_e.GlobalOnly) + + def ExportGlobalString(mem, name, s): # type: (Mem, str, str) -> None """Helper for completion, $PWD, $OLDPWD, etc.""" diff --git a/cpp/core.cc b/cpp/core.cc index e734610c28..53ec67e3f3 100644 --- a/cpp/core.cc +++ b/cpp/core.cc @@ -4,6 +4,7 @@ #include // ispunct() #include +#include #include // fmod() #include // passwd #include @@ -329,7 +330,13 @@ void PopTermAttrs(int fd, int orig_local_modes, void* term_attrs) { namespace pyutil { -static grammar::Grammar* gOilGrammar = nullptr; +float infinity() { + return INFINITY; // float.h +} + +float nan() { + return NAN; // float.h +} // TODO: SHARE with pyext bool IsValidCharEscape(BigStr* c) { @@ -405,6 +412,8 @@ BigStr* strerror(IOError_OSError* e) { return s; } +static grammar::Grammar* gOilGrammar = nullptr; + grammar::Grammar* LoadYshGrammar(_ResourceLoader*) { if (gOilGrammar != nullptr) { return gOilGrammar; diff --git a/cpp/core.h b/cpp/core.h index 8e843cdf30..c2b73abf48 100644 --- a/cpp/core.h +++ b/cpp/core.h @@ -242,6 +242,9 @@ Tuple2* MakeDirCacheKey(BigStr* path); namespace pyutil { +float infinity(); +float nan(); + bool IsValidCharEscape(BigStr* c); BigStr* ChArrayToString(List* ch_array); diff --git a/cpp/stdlib.cc b/cpp/stdlib.cc index f37ba3da97..b2631e139b 100644 --- a/cpp/stdlib.cc +++ b/cpp/stdlib.cc @@ -6,6 +6,7 @@ #include // closedir(), opendir(), readdir() #include #include // open +#include // isinf, isnan #include // kill #include // umask #include // umask @@ -19,6 +20,18 @@ using error::e_die; +namespace math { + +bool isinf(float f) { + return ::isinf(f); +} + +bool isnan(float f) { + return ::isnan(f); +} + +} // namespace math + namespace fcntl_ { int fcntl(int fd, int cmd) { diff --git a/cpp/stdlib.h b/cpp/stdlib.h index a88c991040..4b9208c726 100644 --- a/cpp/stdlib.h +++ b/cpp/stdlib.h @@ -9,6 +9,13 @@ #include "mycpp/runtime.h" +namespace math { + +bool isinf(float f); +bool isnan(float f); + +} // namespace math + namespace fcntl_ { // for F_GETFD diff --git a/data_lang/j8.py b/data_lang/j8.py index 72316fb7d8..11223d8c3f 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -2,11 +2,6 @@ """ j8.py: J8 Notation, a superset of JSON -TODO: - -- Many more tests - - Run JSONTestSuite - Later: - PrettyPrinter uses hnode.asdl? @@ -33,6 +28,8 @@ - NIL8 at least has no commas for [1 2 "hi"] """ +import math + from _devbuild.gen.id_kind_asdl import Id, Id_t, Id_str from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str) from _devbuild.gen.nil8_asdl import (nvalue, nvalue_t) @@ -145,6 +142,7 @@ def Utf8Encode(code): SHOW_CYCLES = 1 << 1 # show as [...] or {...} I think, with object ID SHOW_NON_DATA = 1 << 2 # non-data objects like Eggex can be LOSSY_JSON = 1 << 3 # JSON is lossy +INF_NAN_ARE_NULL = 1 << 4 # for JSON # Hack until we fully translate assert pyj8.LOSSY_JSON == LOSSY_JSON @@ -176,7 +174,7 @@ def PrintJsonMessage(val, buf, indent): Caller must handle error.Encode() Doesn't decay to b'' strings - will use Unicode replacement char. """ - _Print(val, buf, indent, options=LOSSY_JSON) + _Print(val, buf, indent, options=LOSSY_JSON | INF_NAN_ARE_NULL) def PrintLine(val, f): @@ -352,9 +350,20 @@ def Print(self, val, level=0): elif case(value_e.Float): val = cast(value.Float, UP_val) - # TODO: avoid intrmediate allocation with - # self.buf.WriteFloat(val.f) - self.buf.write(str(val.f)) + + fl = val.f + if ((self.options & INF_NAN_ARE_NULL) and + (math.isnan(fl) or math.isinf(fl))): + # JavaScript JSON lib behavior: Inf and NaN are null + # Python has a bug in the encoder by default, and then + # allow_nan=False raises an error + s = 'null' + else: + # TODO: can we avoid intermediate allocation? + # self.buf.WriteFloat(val.f) + s = str(fl) + + self.buf.write(s) elif case(value_e.Str): val = cast(value.Str, UP_val) diff --git a/data_lang/json-survey.sh b/data_lang/json-survey.sh index 04882ff6a4..eb2faa1e0b 100755 --- a/data_lang/json-survey.sh +++ b/data_lang/json-survey.sh @@ -285,6 +285,7 @@ encode-nan() { python3 -c 'import json; val = float("nan"); s = json.dumps(val); print(s); print(json.loads(s))' || true echo + # raises error python3 -c 'import json; val = float("nan"); s = json.dumps(val, allow_nan=False); print(s); print(json.loads(s))' || true echo @@ -302,6 +303,7 @@ encode-inf() { python3 -c 'import json; val = float("-inf"); print(val); s = json.dumps(val); print(s); print(json.loads(s))' || true echo + # raises error python3 -c 'import json; val = float("-inf"); print(val); s = json.dumps(val, allow_nan=False); print(s); print(json.loads(s))' || true echo diff --git a/doc/ref/chap-special-var.md b/doc/ref/chap-special-var.md index 243b73ce7a..3a2bb21248 100644 --- a/doc/ref/chap-special-var.md +++ b/doc/ref/chap-special-var.md @@ -141,6 +141,20 @@ When the shell process exists, print GC stats to stderr. When the shell process exists, print GC stats to this file descriptor. +## Float + +### NAN + +The float value for "not a number". + +(The name is consistent with the C language.) + +### INFINITY + +The float value for "infinity". You can negate it to get "negative infinity". + +(The name is consistent with the C language.) + ## Shell Vars ### IFS diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index c53047958c..94c9453ef8 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -337,6 +337,7 @@ X [External Lang] BEGIN END when (awk) OILS_GC_THRESHOLD OILS_GC_ON_EXIT OILS_GC_STATS OILS_GC_STATS_FD LIB_YSH + [Float] NAN INFINITY ``` + +``` + pp line (42.0 === x) + ^~~ +[ -c flag ]:3: fatal: Equality isn't defined on Float values (OILS-ERR-202) +``` + +Floating point numbers shouldn't be tested for equality. Alternatives: + + = abs(42.0 - x) < 0.1 + = floatEquals(42.0, x) + ## Appendix ### Kinds of Errors from Oils diff --git a/spec/ysh-expr-compare.test.sh b/spec/ysh-expr-compare.test.sh index 6794b8aee2..ebed1b963f 100644 --- a/spec/ysh-expr-compare.test.sh +++ b/spec/ysh-expr-compare.test.sh @@ -1,7 +1,7 @@ ## oils_failures_allowed: 1 #### Exact equality with === and !== -shopt -s oil:all +shopt -s ysh:all if (3 === 3) { echo 'ok' @@ -39,7 +39,7 @@ ok ## END #### Approximate equality of Str x {Str, Int, Bool} with ~== -shopt -s oil:all +shopt -s ysh:all # Note: for now there's no !~== operator. Use: not (a ~== b) @@ -82,7 +82,7 @@ bool matrix ## END #### Wrong Types with ~== -shopt -s oil:all +shopt -s ysh:all # The LHS side should be a string @@ -101,9 +101,30 @@ if (3 ~== 3) { one ## END +#### === on float not allowed + +$SH -c ' +shopt -s ysh:upgrade +pp line (1.0 === 2.0) +echo ok +' +echo status=$? + +$SH -c ' +shopt -s ysh:upgrade +pp line (42 === 3.0) +echo ok +' +echo status=$? + +## STDOUT: +status=3 +status=3 +## END + #### ~== on Float - TODO floatEquals() -shopt -s oil:all +shopt -s ysh:all if (42 ~== 42.0) { echo int-float diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 560393595b..21372e792f 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -731,6 +731,14 @@ test-equality() { ' } +test-float-equality() { + _ysh-expr-error ' +var x = 1 +pp line (42.0 === x)' + + _ysh-expr-error 'pp line (2.0 === 1.0)' +} + test-place() { _ysh-expr-error ' var a = null diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 18f4dadd99..58832f6ab8 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -685,15 +685,9 @@ def _EvalCompare(self, node): result = self._CompareNumeric(left, right, op) elif op.id == Id.Expr_TEqual: - if left.tag() != right.tag(): - result = False - else: - result = val_ops.ExactlyEqual(left, right, op) + result = val_ops.ExactlyEqual(left, right, op) elif op.id == Id.Expr_NotDEqual: - if left.tag() != right.tag(): - result = True - else: - result = not val_ops.ExactlyEqual(left, right, op) + result = not val_ops.ExactlyEqual(left, right, op) elif op.id == Id.Expr_In: result = val_ops.Contains(left, right) diff --git a/ysh/val_ops.py b/ysh/val_ops.py index 213f2976e4..ea92d0bdcf 100644 --- a/ysh/val_ops.py +++ b/ysh/val_ops.py @@ -373,6 +373,11 @@ def ToBool(val): def ExactlyEqual(left, right, blame_loc): # type: (value_t, value_t, loc_t) -> bool + + if left.tag() == value_e.Float or right.tag() == value_e.Float: + raise error.TypeErrVerbose( + "Equality isn't defined on Float values (OILS-ERR-202)", blame_loc) + if left.tag() != right.tag(): return False @@ -396,10 +401,7 @@ def ExactlyEqual(left, right, blame_loc): return mops.Equal(left.i, right.i) elif case(value_e.Float): - # Note: could provide floatEquals(), and suggest it - # Suggested idiom is abs(f1 - f2) < 0.1 - raise error.TypeErrVerbose("Equality isn't defined on Float", - blame_loc) + raise AssertionError() elif case(value_e.Str): left = cast(value.Str, UP_left) From b63c3442d2bb37f912e18a5c17d5e22f3fff4f08 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 24 Jul 2024 15:42:16 -0400 Subject: [PATCH 015/506] [ysh/builtin] Add floatsEqual() This can be used for unit tests, instead of === --- builtin/func_misc.py | 15 +++++++++++++++ core/shell.py | 13 +++++++++++-- doc/ref/chap-builtin-func.md | 14 ++++++++++++++ doc/ref/toc-ysh.md | 31 ++++++++++++++++--------------- mycpp/mylib.py | 9 +++++++-- spec/ysh-expr-compare.test.sh | 28 +++++++--------------------- 6 files changed, 70 insertions(+), 40 deletions(-) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 0191d09be6..7f983b29b2 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -379,6 +379,21 @@ def Call(self, rd): return value.List(l) +class FloatsEqual(vm._Callable): + + def __init__(self): + # type: () -> None + pass + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + left = rd.PosFloat() + right = rd.PosFloat() + rd.Done() + + return value.Bool(left == right) + + class Glob(vm._Callable): def __init__(self, globber): diff --git a/core/shell.py b/core/shell.py index 5355b2d474..ab3860fb3e 100644 --- a/core/shell.py +++ b/core/shell.py @@ -826,8 +826,6 @@ def Main( mem)) _SetGlobalFunc(mem, '_end', func_eggex.MatchFunc(func_eggex.E, None, mem)) - _SetGlobalFunc(mem, 'join', func_misc.Join()) - _SetGlobalFunc(mem, 'maybe', func_misc.Maybe()) _SetGlobalFunc(mem, 'evalExpr', func_misc.EvalExpr(expr_ev)) # type conversions @@ -843,21 +841,32 @@ def Main( _SetGlobalFunc(mem, 'bytes', func_misc.Bytes()) _SetGlobalFunc(mem, 'encodeBytes', func_misc.EncodeBytes()) + # Str + #_SetGlobalFunc(mem, 'strcmp', None) # TODO: This should be Python style splitting _SetGlobalFunc(mem, 'split', func_misc.Split(splitter)) _SetGlobalFunc(mem, 'shSplit', func_misc.Split(splitter)) + # Float + _SetGlobalFunc(mem, 'floatsEqual', func_misc.FloatsEqual()) + + # List + _SetGlobalFunc(mem, 'join', func_misc.Join()) + _SetGlobalFunc(mem, 'maybe', func_misc.Maybe()) _SetGlobalFunc(mem, 'glob', func_misc.Glob(globber)) + _SetGlobalFunc(mem, 'shvarGet', func_misc.Shvar_get(mem)) _SetGlobalFunc(mem, 'getVar', func_misc.GetVar(mem)) _SetGlobalFunc(mem, 'assert_', func_misc.Assert()) + # Serialize _SetGlobalFunc(mem, 'toJson8', func_misc.ToJson8(True)) _SetGlobalFunc(mem, 'toJson', func_misc.ToJson8(False)) _SetGlobalFunc(mem, 'fromJson8', func_misc.FromJson8(True)) _SetGlobalFunc(mem, 'fromJson', func_misc.FromJson8(False)) + # Demos _SetGlobalFunc(mem, '_a2sp', func_misc.BashArrayToSparse()) _SetGlobalFunc(mem, '_d2sp', func_misc.DictToSparse()) _SetGlobalFunc(mem, '_opsp', func_misc.SparseOp()) diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index f2144946bc..09371fc0c1 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -233,6 +233,20 @@ It's also often called with the `=>` chaining operator: json write (items => join(' ')) # => "1 2 3" json write (items => join(', ')) # => "1, 2, 3" +## Float + +### floatsEqual() + +Check if two floating point numbers are equal. + + = floatsEqual(42.0, 42.0) + (Bool) true + +It's usually better to make an approximate comparison: + + = abs(float1 - float2) < 0.001 + (Bool) false + ## Word ### glob() diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 94c9453ef8..3cac9a3186 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -72,22 +72,23 @@ X [Guts] heapId() ```chapter-links-builtin-func - [Values] len() func/type() - [Conversions] bool() int() float() str() list() dict() - X runes() X encodeRunes() - X bytes() X encodeBytes() - [Str] X strcmp() X split() shSplit() + [Values] len() func/type() + [Conversions] bool() int() float() + str() list() dict() + X runes() X encodeRunes() + X bytes() X encodeBytes() + [Str] X strcmp() X split() shSplit() [List] join() - [Float] X isinf() X isnan() - [Collections] X copy() X deepCopy() - [Word] glob() maybe() - [Serialize] toJson() fromJson() - toJson8() fromJson8() -X [J8 Decode] J8.Bool() J8.Int() ... - [Pattern] _group() _start() _end() - [Introspection] shvarGet() getVar() evalExpr() - [Hay Config] parseHay() evalHay() -X [Hashing] sha1dc() sha256() + [Float] floatsEqual() X isinf() X isnan() + [Collections] X copy() X deepCopy() + [Word] glob() maybe() + [Serialize] toJson() fromJson() + toJson8() fromJson8() +X [J8 Decode] J8.Bool() J8.Int() ... + [Pattern] _group() _start() _end() + [Introspection] shvarGet() getVar() evalExpr() + [Hay Config] parseHay() evalHay() +X [Hashing] sha1dc() sha256() ``` +## END + #### Bash Array declare -a array_0=() declare -a array_1=(hello) From fa5b0e27248f325a25704686f0a2ecb6872c41b0 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 19:40:20 -0400 Subject: [PATCH 035/506] [pretty] Add spec tests, tweak unit tests Prepare to omit redundant types. Also compare against pp line (x). In some cases, we are MISSING types. e.g. with Range and BashArray. --- data_lang/pretty.py | 6 +- data_lang/pretty_test.py | 26 ++++----- spec/ysh-printing.test.sh | 120 +++++++++++++++++++++++++++++++++----- 3 files changed, 122 insertions(+), 30 deletions(-) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 48ebee3e1d..d636342cc5 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -440,7 +440,7 @@ def Value(self, val): _Concat([ _Text("(" + ysh_type + ")"), _Break(" "), - self._Value(val) + self._Value(val, type_shown=True) ])) else: return self._Value(val) @@ -680,8 +680,8 @@ def _BashAssoc(self, vassoc): return self._SurroundedAndPrefixed("(", type_name, " ", self._Join(mdocs, "", " "), ")") - def _Value(self, val): - # type: (value_t) -> MeasuredDoc + def _Value(self, val, type_shown=False): + # type: (value_t, bool) -> MeasuredDoc with tagswitch(val) as case: if case(value_e.Null): diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 0cb7ca4928..3e3e1f5bc8 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -23,14 +23,11 @@ def IntValue(i): class PrettyTest(unittest.TestCase): - @classmethod - def setUpClass(cls): + def setUp(self): # Use settings that make testing easier. - cls.printer = pretty.PrettyPrinter() - cls.printer.SetUseStyles(False) - cls.printer.SetShowTypePrefix(False) - # NOTE: We don't SetYshStyle() here ... we changed the format after - # writing these tests + self.printer = pretty.PrettyPrinter() + self.printer.SetUseStyles(False) + self.printer.SetYshStyle() def assertPretty(self, width, value_str, expected, lineno=None): # type: (int, str, str, Optional[int]) -> None @@ -54,6 +51,10 @@ def assertPretty(self, width, value_str, expected, lineno=None): self.assertEqual(buf.getvalue(), expected) def testsFromFile(self): + # TODO: convert tests to this new style + self.printer.SetShowTypePrefix(False) + self.printer.ysh_style = False + chunks = [(None, -1, [])] for lineno, line in enumerate( open(TEST_DATA_FILENAME).read().splitlines()): @@ -97,16 +98,13 @@ def testsFromFile(self): def testStyles(self): self.printer.SetUseStyles(True) self.assertPretty( - 20, '[null, "ok", 15]', '[' + ansi.BOLD + ansi.RED + 'null' + - ansi.RESET + ", " + ansi.GREEN + '"ok"' + ansi.RESET + ", " + + 20, '[null, "ok", 15]', '(List)\n[' + ansi.BOLD + ansi.RED + 'null' + + ansi.RESET + ", " + ansi.GREEN + "'ok'" + ansi.RESET + ", " + ansi.YELLOW + '15' + ansi.RESET + ']') - self.printer.SetUseStyles(False) def testTypePrefix(self): - self.printer.SetShowTypePrefix(True) - self.assertPretty(25, '[null, "ok", 15]', '(List) [null, "ok", 15]') - self.assertPretty(24, '[null, "ok", 15]', '(List)\n[null, "ok", 15]') - self.printer.SetShowTypePrefix(False) + self.assertPretty(25, '[null, "ok", 15]', "(List) [null, 'ok', 15]") + self.assertPretty(24, '[null, "ok", 15]', "(List)\n[null, 'ok', 15]") if __name__ == '__main__': diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index 13f86ae97c..514fb3b233 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -30,34 +30,99 @@ #### Range var x = 1..100 -= x -## stdout: (Range) 1 .. 100 + +pp (x) + +# TODO: show type here, like (Range 1 .. 100) + +pp ({k: x}) + +echo + +remove-addr() { + sed 's/0x[0-9a-f]\+/0x---/' +} + +pp line (x) | remove-addr +pp line ({k: x}) | remove-addr + +## STDOUT: +(Range) 1 .. 100 +(Dict) {k: 1 .. 100} + +(Range) +(Dict) {"k":} +## END + #### Eggex (reference type) var pat = /d+/ -pp (pat) | sed 's/0x[0-9a-f]\+/0x---/' + +remove-addr() { + sed 's/0x[0-9a-f]\+/0x---/' +} + +pp (pat) | remove-addr + +pp ({k: pat}) | remove-addr # TODO: change this +echo + +pp line (pat) | remove-addr +pp line ({k: pat}) | remove-addr + ## STDOUT: (Eggex) +(Dict) {k: } + +(Eggex) +(Dict) {"k":} ## END -#### Bash Array -declare -a array_0=() +#### BashArray +declare -a empty=() declare -a array_1=(hello) + +pp (empty) +pp (array_1) +echo + +pp ({k: empty}) +pp ({k: array_1}) +echo + +pp line (empty) +pp line (array_1) +echo + +pp line ({k: empty}) +pp line ({k: array_1}) + +## STDOUT: +(BashArray) (BashArray) +(BashArray) (BashArray 'hello') + +(Dict) {k: (BashArray)} +(Dict) {k: (BashArray 'hello')} + +(BashArray) [] +(BashArray) ["hello"] + +(Dict) {"k":[]} +(Dict) {"k":["hello"]} +## END + +#### BashArray Long declare -a array_3 array_3[0]="world" array_3[2]=*.py declare array_long=(Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod.) -= array_0 -= array_1 = array_3 = array_long ## STDOUT: -(BashArray) (BashArray) -(BashArray) (BashArray 'hello') (BashArray) (BashArray 'world' null '*.py') (BashArray) (BashArray @@ -67,12 +132,41 @@ do eiusmod.) ) ## END -#### Bash Assoc: string formatting +#### BashAssoc, short +declare -A empty declare -A assoc=(['k']=$'foo \x01\u03bc') -= assoc -## stdout: (BashAssoc) (BashAssoc ['k']=$'foo \u0001μ') -#### Bash Assoc +pp (empty) +pp (assoc) +echo + +pp ({k:empty}) +pp ({k:assoc}) +echo + +pp line (empty) +pp line (assoc) +echo + +pp line ({k:empty}) +pp line ({k:assoc}) + +## STDOUT: +(BashAssoc) (BashAssoc) +(BashAssoc) (BashAssoc ['k']=$'foo \u0001μ') + +(Dict) {k: (BashAssoc)} +(Dict) {k: (BashAssoc ['k']=$'foo \u0001μ')} + +(BashAssoc) {} +(BashAssoc) {"k":"foo \u0001μ"} + +(Dict) {"k":{}} +(Dict) {"k":{"k":"foo \u0001μ"}} +## END + + +#### BashAssoc, long declare -A assoc_0=() declare -A assoc_1=([1]=one) declare assoc_3=([1]=one [two]=2 [3]=three) From 4316acfea7162a9482fccdf8de49d0548a8b8373 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 19:55:48 -0400 Subject: [PATCH 036/506] [pretty] Only show type prefix for JSON-like types All the rest of the types have it in the "data", e.g. (BashArray ...) Also change Range pretty printing to include type: (Range 1 .. 2) --- data_lang/pretty.py | 29 ++++++++++++++++------------- data_lang/pretty_test.py | 6 +++--- spec/ysh-printing.test.sh | 26 ++++++++++++-------------- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index d636342cc5..05f99202d5 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -435,13 +435,18 @@ def Value(self, val): """Convert an Oils value into a `doc`, which can then be pretty printed.""" self.visiting.clear() if self.show_type_prefix: - ysh_type = ValType(val) + # These JSON-like types have a special notation, so print type + # explicitly + if val.tag() in (value_e.Null, value_e.Bool, value_e.Int, + value_e.Float, value_e.Str, value_e.List, + value_e.Dict): + ysh_type = ValType(val) + maybe_type = [_Text("(" + ysh_type + ")"), _Break(" ")] + else: + maybe_type = [] + return _Group( - _Concat([ - _Text("(" + ysh_type + ")"), - _Break(" "), - self._Value(val, type_shown=True) - ])) + _Concat(maybe_type + [self._Value(val, type_shown=True)])) else: return self._Value(val) @@ -706,13 +711,11 @@ def _Value(self, val, type_shown=False): elif case(value_e.Range): r = cast(value.Range, val) - return self._Styled( - self.number_style, - _Concat([ - _Text(str(r.lower)), - _Text(" .. "), - _Text(str(r.upper)) - ])) + type_name = self._Styled(self.type_style, _Text(ValType(r))) + mdocs = [_Text(str(r.lower)), _Text(".."), _Text(str(r.upper))] + return self._SurroundedAndPrefixed("(", type_name, " ", + self._Join(mdocs, "", " "), + ")") elif case(value_e.List): vlist = cast(value.List, val) diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 3e3e1f5bc8..f4580836e2 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -98,9 +98,9 @@ def testsFromFile(self): def testStyles(self): self.printer.SetUseStyles(True) self.assertPretty( - 20, '[null, "ok", 15]', '(List)\n[' + ansi.BOLD + ansi.RED + 'null' + - ansi.RESET + ", " + ansi.GREEN + "'ok'" + ansi.RESET + ", " + - ansi.YELLOW + '15' + ansi.RESET + ']') + 20, '[null, "ok", 15]', '(List)\n[' + ansi.BOLD + ansi.RED + + 'null' + ansi.RESET + ", " + ansi.GREEN + "'ok'" + ansi.RESET + + ", " + ansi.YELLOW + '15' + ansi.RESET + ']') def testTypePrefix(self): self.assertPretty(25, '[null, "ok", 15]', "(List) [null, 'ok', 15]") diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index 514fb3b233..2e114aa2ff 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -47,8 +47,8 @@ pp line (x) | remove-addr pp line ({k: x}) | remove-addr ## STDOUT: -(Range) 1 .. 100 -(Dict) {k: 1 .. 100} +(Range 1 .. 100) +(Dict) {k: (Range 1 .. 100)} (Range) (Dict) {"k":} @@ -74,14 +74,14 @@ pp line (pat) | remove-addr pp line ({k: pat}) | remove-addr ## STDOUT: -(Eggex) + (Dict) {k: } (Eggex) (Dict) {"k":} ## END -#### BashArray +#### BashArray, short declare -a empty=() declare -a array_1=(hello) @@ -101,8 +101,8 @@ pp line ({k: empty}) pp line ({k: array_1}) ## STDOUT: -(BashArray) (BashArray) -(BashArray) (BashArray 'hello') +(BashArray) +(BashArray 'hello') (Dict) {k: (BashArray)} (Dict) {k: (BashArray 'hello')} @@ -114,7 +114,7 @@ pp line ({k: array_1}) (Dict) {"k":["hello"]} ## END -#### BashArray Long +#### BashArray, long declare -a array_3 array_3[0]="world" array_3[2]=*.py @@ -123,8 +123,7 @@ do eiusmod.) = array_3 = array_long ## STDOUT: -(BashArray) (BashArray 'world' null '*.py') -(BashArray) +(BashArray 'world' null '*.py') (BashArray 'Lorem' 'ipsum' 'dolor' 'sit' 'amet,' 'consectetur' 'adipiscing' 'elit,' 'sed' 'do' @@ -152,8 +151,8 @@ pp line ({k:empty}) pp line ({k:assoc}) ## STDOUT: -(BashAssoc) (BashAssoc) -(BashAssoc) (BashAssoc ['k']=$'foo \u0001μ') +(BashAssoc) +(BashAssoc ['k']=$'foo \u0001μ') (Dict) {k: (BashAssoc)} (Dict) {k: (BashAssoc ['k']=$'foo \u0001μ')} @@ -176,10 +175,9 @@ declare assoc_long=([Lorem]=ipsum [dolor]="sit amet," ['consectetur adipiscing'] = assoc_3 = assoc_long ## STDOUT: -(BashAssoc) (BashAssoc) -(BashAssoc) (BashAssoc ['1']='one') -(BashAssoc) (BashAssoc ['1']='one' ['two']='2' ['3']='three') (BashAssoc) +(BashAssoc ['1']='one') +(BashAssoc ['1']='one' ['two']='2' ['3']='three') (BashAssoc ['Lorem']='ipsum' ['dolor']='sit amet,' From 51b74bfdc9e12dce6f1c983e4847215e9cf0a621 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 20:18:03 -0400 Subject: [PATCH 037/506] [builtin/pp] Make pp line (x) consistent with pp (x) With repsect to type name I also want to remove JSON serialization of BashArray and BashAssoc. BashArray will change to Dict[int, str] anyway. --- builtin/io_ysh.py | 8 ++++++-- data_lang/pretty.py | 11 ++++++++--- spec/ysh-printing.test.sh | 4 ++-- spec/ysh-slice-range.test.sh | 4 ++-- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index ebc3b806d0..03239581a7 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -5,6 +5,7 @@ from __future__ import print_function from _devbuild.gen import arg_types +from _devbuild.gen.value_asdl import value_e from _devbuild.gen.runtime_asdl import cmd_value from _devbuild.gen.syntax_asdl import command_e, BraceGroup, loc from asdl import format as fmt @@ -14,6 +15,7 @@ from core import ui from core import vm from data_lang import j8 +from data_lang import pretty from frontend import flag_util from frontend import match from frontend import typed_args @@ -126,8 +128,10 @@ def Run(self, cmd_val): val = rd.PosValue() rd.Done() - ysh_type = ui.ValType(val) - self.stdout_.write('(%s) ' % ysh_type) + if pretty.TypeNotPrinted(val) or val.tag() in (value_e.BashArray, + value_e.BashAssoc): + ysh_type = ui.ValType(val) + self.stdout_.write('(%s) ' % ysh_type) j8.PrintLine(val, self.stdout_) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 05f99202d5..13f19a059c 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -120,6 +120,13 @@ def ValType(val): return value_str(val.tag(), dot=False) +def TypeNotPrinted(val): + # type: (value_t) -> bool + return val.tag() in (value_e.Null, value_e.Bool, value_e.Int, + value_e.Float, value_e.Str, value_e.List, + value_e.Dict) + + def _FloatString(fl): # type: (float) -> str @@ -437,9 +444,7 @@ def Value(self, val): if self.show_type_prefix: # These JSON-like types have a special notation, so print type # explicitly - if val.tag() in (value_e.Null, value_e.Bool, value_e.Int, - value_e.Float, value_e.Str, value_e.List, - value_e.Dict): + if TypeNotPrinted(val): ysh_type = ValType(val) maybe_type = [_Text("(" + ysh_type + ")"), _Break(" ")] else: diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index 2e114aa2ff..8686cec161 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -50,7 +50,7 @@ pp line ({k: x}) | remove-addr (Range 1 .. 100) (Dict) {k: (Range 1 .. 100)} -(Range) + (Dict) {"k":} ## END @@ -77,7 +77,7 @@ pp line ({k: pat}) | remove-addr (Dict) {k: } -(Eggex) + (Dict) {"k":} ## END diff --git a/spec/ysh-slice-range.test.sh b/spec/ysh-slice-range.test.sh index f42db54812..156c80d2e6 100644 --- a/spec/ysh-slice-range.test.sh +++ b/spec/ysh-slice-range.test.sh @@ -17,13 +17,13 @@ = 1..3 ## STDOUT: -(Range) 1 .. 3 +(Range 1 .. 3) ## END #### precedence of 1:3 vs bitwise operator = 3..3|4 ## STDOUT: -(Range) 3 .. 7 +(Range 3 .. 7) ## END #### subscript and slice :| 1 2 3 4 | From 5ea3ae616c8349fe80a8a3914584a0312692820f Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 20:28:03 -0400 Subject: [PATCH 038/506] [pretty] Fix translation Can't use + operator on lists --- data_lang/pretty.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 13f19a059c..ca4223bbbb 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -445,13 +445,12 @@ def Value(self, val): # These JSON-like types have a special notation, so print type # explicitly if TypeNotPrinted(val): - ysh_type = ValType(val) - maybe_type = [_Text("(" + ysh_type + ")"), _Break(" ")] + mdocs = [_Text("(" + ValType(val) + ")"), _Break(" ")] else: - maybe_type = [] + mdocs = [] - return _Group( - _Concat(maybe_type + [self._Value(val, type_shown=True)])) + mdocs.append(self._Value(val)) + return _Group(_Concat(mdocs)) else: return self._Value(val) @@ -690,8 +689,8 @@ def _BashAssoc(self, vassoc): return self._SurroundedAndPrefixed("(", type_name, " ", self._Join(mdocs, "", " "), ")") - def _Value(self, val, type_shown=False): - # type: (value_t, bool) -> MeasuredDoc + def _Value(self, val): + # type: (value_t) -> MeasuredDoc with tagswitch(val) as case: if case(value_e.Null): From 154e6b0d3b8387059bd0fca750b06c29bb583551 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 20:55:03 -0400 Subject: [PATCH 039/506] [pretty] Always print type name in color Adjust colors a bit - roughly like node.js, with a few differences - Remove usage of BashArray - I think we don't want to serialize these as JSON --- core/state.py | 5 +++-- data_lang/pretty.py | 16 ++++++++-------- spec/ysh-expr-bool.test.sh | 16 ++++++++++------ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/core/state.py b/core/state.py index b35f061e63..32457a53b9 100644 --- a/core/state.py +++ b/core/state.py @@ -731,9 +731,10 @@ def __repr__(self): def Dump(self): # type: () -> Dict[str, value_t] + items = [value.Str(s) for s in self.argv] # type: List[value_t] + argv = value.List(items) return { - # Easier to serialize value.BashArray than value.List - 'argv': value.BashArray(self.argv), + 'argv': argv, 'num_shifted': num.ToBig(self.num_shifted), } diff --git a/data_lang/pretty.py b/data_lang/pretty.py index ca4223bbbb..404f14e993 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -431,11 +431,11 @@ def __init__(self, indent, use_styles, show_type_prefix, max_tabular_width, # These can be configurable later self.number_style = ansi.YELLOW - self.null_style = ansi.BOLD + ansi.RED - self.bool_style = ansi.BOLD + ansi.BLUE + self.null_style = ansi.BOLD + self.bool_style = ansi.YELLOW self.string_style = ansi.GREEN - self.cycle_style = ansi.BOLD + ansi.MAGENTA - self.type_style = ansi.CYAN + self.cycle_style = ansi.BOLD + ansi.BLUE + self.type_style = ansi.MAGENTA def Value(self, val): # type: (value_t) -> MeasuredDoc @@ -445,7 +445,8 @@ def Value(self, val): # These JSON-like types have a special notation, so print type # explicitly if TypeNotPrinted(val): - mdocs = [_Text("(" + ValType(val) + ")"), _Break(" ")] + type_name = self._Styled(self.type_style, _Text(ValType(val))) + mdocs = [_Text("("), type_name, _Text(")"), _Break(" ")] else: mdocs = [] @@ -760,10 +761,9 @@ def _Value(self, val): return self._BashAssoc(vassoc) else: - ysh_type = ValType(val) + type_name = self._Styled(self.type_style, _Text(ValType(val))) id_str = j8.ValueIdString(val) - return self._Styled(self.type_style, - _Text("<" + ysh_type + id_str + ">")) + return _Concat([_Text("<"), type_name, _Text(id_str + ">")]) # vim: sw=4 diff --git a/spec/ysh-expr-bool.test.sh b/spec/ysh-expr-bool.test.sh index 251c1ae27f..0a1988fac3 100644 --- a/spec/ysh-expr-bool.test.sh +++ b/spec/ysh-expr-bool.test.sh @@ -119,12 +119,6 @@ pp line ({"d": 1} and {}) echo $[0 or 0.0 or false or [] or {} or "OR"] echo $[1 and 1.0 and true and [5] and {"d":1} and "AND"] -declare -a array=(1 2 3) -pp line (array or 'yy') - -declare -A assoc=([k]=v) -pp line (assoc or 'zz') - ## STDOUT: s None @@ -163,6 +157,16 @@ y (Dict) {} OR AND +## END + +#### or BashArray, or BashAssoc +declare -a array=(1 2 3) +pp line (array or 'yy') + +declare -A assoc=([k]=v) +pp line (assoc or 'zz') + +## STDOUT: (BashArray) ["1","2","3"] (BashAssoc) {"k":"v"} ## END From 58d7057de95bf044d100ac888a6b258018087542 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 21:32:29 -0400 Subject: [PATCH 040/506] [pretty] Fix test, and adjust colors Make int and float look different. --- core/ansi.py | 1 + data_lang/pretty.py | 13 +++++++------ data_lang/pretty_test.py | 6 +++--- devtools/release-note.sh | 8 ++++---- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/core/ansi.py b/core/ansi.py index 47a0d3995d..b8a69021ce 100644 --- a/core/ansi.py +++ b/core/ansi.py @@ -12,3 +12,4 @@ BLUE = '\x1b[34m' MAGENTA = '\x1b[35m' CYAN = '\x1b[36m' +WHITE = '\x1b[37m' diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 404f14e993..0e2051a6b0 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -416,7 +416,7 @@ def _PrintDoc(self, document, buf): class _DocConstructor: - """Converts Oil values into `doc`s, which can then be pretty printed.""" + """Converts Oils values into `doc`s, which can then be pretty printed.""" def __init__(self, indent, use_styles, show_type_prefix, max_tabular_width, ysh_style): @@ -430,9 +430,10 @@ def __init__(self, indent, use_styles, show_type_prefix, max_tabular_width, self.visiting = {} # type: Dict[int, bool] # These can be configurable later - self.number_style = ansi.YELLOW - self.null_style = ansi.BOLD - self.bool_style = ansi.YELLOW + self.int_style = ansi.YELLOW + self.float_style = ansi.BLUE + self.null_style = ansi.RED + self.bool_style = ansi.CYAN self.string_style = ansi.GREEN self.cycle_style = ansi.BOLD + ansi.BLUE self.type_style = ansi.MAGENTA @@ -704,11 +705,11 @@ def _Value(self, val): elif case(value_e.Int): i = cast(value.Int, val).i - return self._Styled(self.number_style, _Text(mops.ToStr(i))) + return self._Styled(self.int_style, _Text(mops.ToStr(i))) elif case(value_e.Float): f = cast(value.Float, val).f - return self._Styled(self.number_style, _Text(_FloatString(f))) + return self._Styled(self.float_style, _Text(_FloatString(f))) elif case(value_e.Str): s = cast(value.Str, val).s diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index f4580836e2..6a106369d9 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -98,9 +98,9 @@ def testsFromFile(self): def testStyles(self): self.printer.SetUseStyles(True) self.assertPretty( - 20, '[null, "ok", 15]', '(List)\n[' + ansi.BOLD + ansi.RED + - 'null' + ansi.RESET + ", " + ansi.GREEN + "'ok'" + ansi.RESET + - ", " + ansi.YELLOW + '15' + ansi.RESET + ']') + 20, '[null, "ok", 15]', '(' + ansi.MAGENTA + 'List' + ansi.RESET + + ')\n[' + ansi.RED + 'null' + ansi.RESET + ", " + ansi.GREEN + + "'ok'" + ansi.RESET + ", " + ansi.YELLOW + '15' + ansi.RESET + ']') def testTypePrefix(self): self.assertPretty(25, '[null, "ok", 15]', "(List) [null, 'ok', 15]") diff --git a/devtools/release-note.sh b/devtools/release-note.sh index 2aff857ec3..0cf0541ef1 100755 --- a/devtools/release-note.sh +++ b/devtools/release-note.sh @@ -28,7 +28,7 @@ _git-changelog-body() { # %x00 generates the byte \x00 local format=' %h + href="https://github.com/oils-for-unix/oils/commit/%H">%h %x00%an%x01 %x00%s%x01 @@ -57,7 +57,7 @@ contrib-commit-table() { } fetch-issues() { - local url='https://api.github.com/repos/oilshell/oil/issues?labels=pending-release' + local url='https://api.github.com/repos/oils-for-unix/oils/issues?labels=pending-release' curl "$url" > _tmp/issues.json } @@ -132,7 +132,7 @@ If you're new to the project, see [Why Create a New Shell?][why-oil] and posts tagged #[FAQ](\$blog-tag). [INSTALL.txt]: /release/$OILS_VERSION/doc/INSTALL.html -[github-bugs]: https://github.com/oilshell/oil/issues +[github-bugs]: https://github.com/oils-for-unix/oils/issues [why-oil]: ../../2021/01/why-a-new-shell.html [release-index]: /release/$OILS_VERSION/ @@ -178,7 +178,7 @@ EOF ### Wiki Pages -- [How Interactive Shells Work](https://github.com/oilshell/oil/wiki/How-Interactive-Shells-Work) +- [How Interactive Shells Work](https://github.com/oils-for-unix/oils/wiki/How-Interactive-Shells-Work) ## What's Next? From 5d869c2a35a6b66311ec39f98738541e0243e7be Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 22:16:06 -0400 Subject: [PATCH 041/506] [json, pretty] Change encoding of value.BashArray, more like SparseArray It now looks like {type: "BashArray", value: {...}} where the value is a Dict, to prepare for the value.SparseArray change --- builtin/io_ysh.py | 4 ++-- core/state.py | 2 ++ data_lang/j8.py | 41 +++++++++++++++++++++++++++++------ demo/osh-crash.sh | 5 ++++- spec/ysh-builtin-meta.test.sh | 4 ++-- spec/ysh-dev.test.sh | 1 - spec/ysh-expr-bool.test.sh | 2 +- spec/ysh-printing.test.sh | 8 +++---- 8 files changed, 49 insertions(+), 18 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 03239581a7..ef946ca545 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -128,8 +128,8 @@ def Run(self, cmd_val): val = rd.PosValue() rd.Done() - if pretty.TypeNotPrinted(val) or val.tag() in (value_e.BashArray, - value_e.BashAssoc): + if pretty.TypeNotPrinted(val) or val.tag() in ( + value_e.BashAssoc, ): ysh_type = ui.ValType(val) self.stdout_.write('(%s) ' % ysh_type) diff --git a/core/state.py b/core/state.py index 32457a53b9..fc8925e6d1 100644 --- a/core/state.py +++ b/core/state.py @@ -796,10 +796,12 @@ def _DumpVarFrame(frame): elif case(value_e.BashArray): cell_json['type'] = value.Str('BashArray') + # TODO: this results in a nested {type: ..., value: ...} dict cell_json['value'] = cell.val elif case(value_e.BashAssoc): cell_json['type'] = value.Str('BashAssoc') + # TODO: this results in a nested {type: ..., value: ...} dict cell_json['value'] = cell.val else: diff --git a/data_lang/j8.py b/data_lang/j8.py index 0c54ec2dd2..fde9f9609b 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -448,23 +448,50 @@ def Print(self, val, level=0): elif case(value_e.BashArray): val = cast(value.BashArray, UP_val) - self.buf.write('[') + self.buf.write('{') self._MaybeNewline() + self._ItemIndent(level) + self.buf.write('"type":') + self._MaybeSpace() + self.buf.write('"BashArray",') + + self._MaybeNewline() + + self._ItemIndent(level) + self.buf.write('"value":') + self._MaybeSpace() + self.buf.write('{') + self._MaybeNewline() + + level += 1 + first = True for i, s in enumerate(val.strs): - if i != 0: + if s is None: + continue + + if not first: self.buf.write(',') self._MaybeNewline() self._ItemIndent(level) - if s is None: - self.buf.write('null') - else: - pyj8.WriteString(s, self.options, self.buf) + + pyj8.WriteString(str(i), self.options, self.buf) + self.buf.write(':') + self._MaybeSpace() + + pyj8.WriteString(s, self.options, self.buf) + + first = False self._MaybeNewline() self._BracketIndent(level) - self.buf.write(']') + self.buf.write('}') + + level -= 1 + self._MaybeNewline() + self._BracketIndent(level) + self.buf.write('}') elif case(value_e.BashAssoc): val = cast(value.BashAssoc, UP_val) diff --git a/demo/osh-crash.sh b/demo/osh-crash.sh index 5b8342b6b7..b53ee869e9 100755 --- a/demo/osh-crash.sh +++ b/demo/osh-crash.sh @@ -8,7 +8,10 @@ set -o pipefail set -o errexit g() { - local g=1 + readonly g=1 + readonly -a bash_array=(a b) + bash_array[5]=z + echo foo > $bar } diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index e4174ac565..c045b428f7 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -152,8 +152,8 @@ assoc['k3']= pp line (assoc) ## STDOUT: -(BashArray) ["a","b","c"] -(BashArray) ["a","b","c",null,null,"z"] +{"type":"BashArray","value":{"0":"a","1":"b","2":"c"}} +{"type":"BashArray","value":{"0":"a","1":"b","2":"c","5":"z"}} (BashAssoc) {"k":"v","k2":"v2"} (BashAssoc) {"k":"v","k2":"v2","k3":""} ## END diff --git a/spec/ysh-dev.test.sh b/spec/ysh-dev.test.sh index 466fa5b913..cd94b37540 100644 --- a/spec/ysh-dev.test.sh +++ b/spec/ysh-dev.test.sh @@ -161,4 +161,3 @@ status=0 status=1 ## END - diff --git a/spec/ysh-expr-bool.test.sh b/spec/ysh-expr-bool.test.sh index 0a1988fac3..d0565d3196 100644 --- a/spec/ysh-expr-bool.test.sh +++ b/spec/ysh-expr-bool.test.sh @@ -167,7 +167,7 @@ declare -A assoc=([k]=v) pp line (assoc or 'zz') ## STDOUT: -(BashArray) ["1","2","3"] +{"type":"BashArray","value":{"0":"1","1":"2","2":"3"}} (BashAssoc) {"k":"v"} ## END diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index 8686cec161..a0c9c740fe 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -107,11 +107,11 @@ pp line ({k: array_1}) (Dict) {k: (BashArray)} (Dict) {k: (BashArray 'hello')} -(BashArray) [] -(BashArray) ["hello"] +{"type":"BashArray","value":{}} +{"type":"BashArray","value":{"0":"hello"}} -(Dict) {"k":[]} -(Dict) {"k":["hello"]} +(Dict) {"k":{"type":"BashArray","value":{}}} +(Dict) {"k":{"type":"BashArray","value":{"0":"hello"}}} ## END #### BashArray, long From 6958fd15e9531cf1fc1dc0350193d322915cfa7a Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 22:37:49 -0400 Subject: [PATCH 042/506] [json, pretty] Change encoding of value.BashAssoc It's like value.BashArray now Still TODO: change pretty printing of value.BashArray --- builtin/io_ysh.py | 4 +- data_lang/j8.py | 167 ++++++++++++++++++++-------------- demo/osh-crash.sh | 5 +- spec/ysh-builtin-meta.test.sh | 4 +- spec/ysh-expr-bool.test.sh | 2 +- spec/ysh-json.test.sh | 47 ++++++++++ spec/ysh-printing.test.sh | 8 +- 7 files changed, 158 insertions(+), 79 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index ef946ca545..e769df2760 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -5,7 +5,6 @@ from __future__ import print_function from _devbuild.gen import arg_types -from _devbuild.gen.value_asdl import value_e from _devbuild.gen.runtime_asdl import cmd_value from _devbuild.gen.syntax_asdl import command_e, BraceGroup, loc from asdl import format as fmt @@ -128,8 +127,7 @@ def Run(self, cmd_val): val = rd.PosValue() rd.Done() - if pretty.TypeNotPrinted(val) or val.tag() in ( - value_e.BashAssoc, ): + if pretty.TypeNotPrinted(val): ysh_type = ui.ValType(val) self.stdout_.write('(%s) ' % ysh_type) diff --git a/data_lang/j8.py b/data_lang/j8.py index fde9f9609b..e938c3fe64 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -322,6 +322,103 @@ def _PrintDict(self, val, level): self._BracketIndent(level) self.buf.write('}') + def _PrintBashPrefix(self, type_str, level): + # type: (str, int) -> None + + self.buf.write('{') + self._MaybeNewline() + self._ItemIndent(level) + self.buf.write('"type":') + self._MaybeSpace() + self.buf.write(type_str) # "BashArray", or "BashAssoc", + + self._MaybeNewline() + + self._ItemIndent(level) + self.buf.write('"value":') + self._MaybeSpace() + + def _PrintBashSuffix(self, level): + # type: (int) -> None + level -= 1 + self._MaybeNewline() + self._BracketIndent(level) + self.buf.write('}') + + def _PrintBashArray(self, val, level): + # type: (value.BashArray, int) -> None + + self._PrintBashPrefix('"BashArray",', level) + + if len(val.strs) == 0: # Special case like Python/JS + self.buf.write('{}') + else: + self.buf.write('{') + self._MaybeNewline() + + level += 1 + first = True + for i, s in enumerate(val.strs): + if s is None: + continue + + if not first: + self.buf.write(',') + self._MaybeNewline() + + self._ItemIndent(level) + + pyj8.WriteString(str(i), self.options, self.buf) + self.buf.write(':') + self._MaybeSpace() + + pyj8.WriteString(s, self.options, self.buf) + + first = False + + self._MaybeNewline() + + self._BracketIndent(level) + self.buf.write('}') + + self._PrintBashSuffix(level) + + def _PrintBashAssoc(self, val, level): + # type: (value.BashAssoc, int) -> None + + self._PrintBashPrefix('"BashAssoc",', level) + + if len(val.d) == 0: # Special case like Python/JS + self.buf.write('{}') + else: + self.buf.write('{') + self._MaybeNewline() + + level += 1 + i = 0 + for k2, v2 in iteritems(val.d): + if i != 0: + self.buf.write(',') + self._MaybeNewline() + + self._ItemIndent(level) + + pyj8.WriteString(k2, self.options, self.buf) + + self.buf.write(':') + self._MaybeSpace() + + pyj8.WriteString(v2, self.options, self.buf) + + i += 1 + + self._MaybeNewline() + + self._BracketIndent(level) + self.buf.write('}') + + self._PrintBashSuffix(level) + def Print(self, val, level=0): # type: (value_t, int) -> None @@ -447,77 +544,11 @@ def Print(self, val, level=0): elif case(value_e.BashArray): val = cast(value.BashArray, UP_val) - - self.buf.write('{') - self._MaybeNewline() - self._ItemIndent(level) - self.buf.write('"type":') - self._MaybeSpace() - self.buf.write('"BashArray",') - - self._MaybeNewline() - - self._ItemIndent(level) - self.buf.write('"value":') - self._MaybeSpace() - self.buf.write('{') - self._MaybeNewline() - - level += 1 - first = True - for i, s in enumerate(val.strs): - if s is None: - continue - - if not first: - self.buf.write(',') - self._MaybeNewline() - - self._ItemIndent(level) - - pyj8.WriteString(str(i), self.options, self.buf) - self.buf.write(':') - self._MaybeSpace() - - pyj8.WriteString(s, self.options, self.buf) - - first = False - - self._MaybeNewline() - - self._BracketIndent(level) - self.buf.write('}') - - level -= 1 - self._MaybeNewline() - self._BracketIndent(level) - self.buf.write('}') + self._PrintBashArray(val, level) elif case(value_e.BashAssoc): val = cast(value.BashAssoc, UP_val) - - self.buf.write('{') - self._MaybeNewline() - i = 0 - for k2, v2 in iteritems(val.d): - if i != 0: - self.buf.write(',') - self._MaybeNewline() - - self._ItemIndent(level) - - pyj8.WriteString(k2, self.options, self.buf) - - self.buf.write(':') - self._MaybeSpace() - - pyj8.WriteString(v2, self.options, self.buf) - - i += 1 - - self._MaybeNewline() - self._BracketIndent(level) - self.buf.write('}') + self._PrintBashAssoc(val, level) else: pass # mycpp workaround diff --git a/demo/osh-crash.sh b/demo/osh-crash.sh index b53ee869e9..312794ef77 100755 --- a/demo/osh-crash.sh +++ b/demo/osh-crash.sh @@ -9,8 +9,11 @@ set -o errexit g() { readonly g=1 - readonly -a bash_array=(a b) + local -a bash_array=(a b) bash_array[5]=z + readonly bash_array + + readonly -A bash_assoc=([x]=y [foo]=bar) echo foo > $bar } diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index c045b428f7..b2507f647c 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -154,8 +154,8 @@ pp line (assoc) ## STDOUT: {"type":"BashArray","value":{"0":"a","1":"b","2":"c"}} {"type":"BashArray","value":{"0":"a","1":"b","2":"c","5":"z"}} -(BashAssoc) {"k":"v","k2":"v2"} -(BashAssoc) {"k":"v","k2":"v2","k3":""} +{"type":"BashAssoc","value":{"k":"v","k2":"v2"}} +{"type":"BashAssoc","value":{"k":"v","k2":"v2","k3":""}} ## END diff --git a/spec/ysh-expr-bool.test.sh b/spec/ysh-expr-bool.test.sh index d0565d3196..e3511c1845 100644 --- a/spec/ysh-expr-bool.test.sh +++ b/spec/ysh-expr-bool.test.sh @@ -168,7 +168,7 @@ pp line (assoc or 'zz') ## STDOUT: {"type":"BashArray","value":{"0":"1","1":"2","2":"3"}} -(BashAssoc) {"k":"v"} +{"type":"BashAssoc","value":{"k":"v"}} ## END #### x if b else y diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index 41a7b3f2a3..7082331bed 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -1210,3 +1210,50 @@ pp line (_reply) ## STDOUT: ## END +#### BashArray can be serialized + +declare -a empty_array + +declare -a array=(x y) +array[5]=z + +json write (empty_array) +json write (array) + +## STDOUT: +{ + "type": "BashArray", + "value": {} +} +{ + "type": "BashArray", + "value": { + "0": "x", + "1": "y", + "5": "z" + } +} +## END + +#### BashAssoc can be serialized + +declare -A empty_assoc + +declare -A assoc=([foo]=bar [42]=43) + +json write (empty_assoc) +json write (assoc) + +## STDOUT: +{ + "type": "BashAssoc", + "value": {} +} +{ + "type": "BashAssoc", + "value": { + "foo": "bar", + "42": "43" + } +} +## END diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index a0c9c740fe..dd19aa5184 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -157,11 +157,11 @@ pp line ({k:assoc}) (Dict) {k: (BashAssoc)} (Dict) {k: (BashAssoc ['k']=$'foo \u0001μ')} -(BashAssoc) {} -(BashAssoc) {"k":"foo \u0001μ"} +{"type":"BashAssoc","value":{}} +{"type":"BashAssoc","value":{"k":"foo \u0001μ"}} -(Dict) {"k":{}} -(Dict) {"k":{"k":"foo \u0001μ"}} +(Dict) {"k":{"type":"BashAssoc","value":{}}} +(Dict) {"k":{"type":"BashAssoc","value":{"k":"foo \u0001μ"}}} ## END From 0f8f5346a6f4c0e5d43f8301390e31d0af4476a9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 25 Jul 2024 23:30:17 -0400 Subject: [PATCH 043/506] [core] Simplify crash dump, and encoding of BashArray, BashAssoc --- core/state.py | 17 +++-------------- data_lang/j8.py | 2 +- spec/ysh-builtin-meta.test.sh | 8 ++++---- spec/ysh-expr-bool.test.sh | 4 ++-- spec/ysh-json.test.sh | 8 ++++---- spec/ysh-printing.test.sh | 16 ++++++++-------- 6 files changed, 22 insertions(+), 33 deletions(-) diff --git a/core/state.py b/core/state.py index fc8925e6d1..a7bc3e3907 100644 --- a/core/state.py +++ b/core/state.py @@ -788,21 +788,10 @@ def _DumpVarFrame(frame): with tagswitch(cell.val) as case: if case(value_e.Undef): - cell_json['type'] = value.Str('Undef') + cell_json['val'] = value.Null - elif case(value_e.Str): - cell_json['type'] = value.Str('Str') - cell_json['value'] = cell.val - - elif case(value_e.BashArray): - cell_json['type'] = value.Str('BashArray') - # TODO: this results in a nested {type: ..., value: ...} dict - cell_json['value'] = cell.val - - elif case(value_e.BashAssoc): - cell_json['type'] = value.Str('BashAssoc') - # TODO: this results in a nested {type: ..., value: ...} dict - cell_json['value'] = cell.val + elif case(value_e.Str, value_e.BashArray, value_e.BashAssoc): + cell_json['val'] = cell.val else: # TODO: should we show the object ID here? diff --git a/data_lang/j8.py b/data_lang/j8.py index e938c3fe64..a0247d4d30 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -335,7 +335,7 @@ def _PrintBashPrefix(self, type_str, level): self._MaybeNewline() self._ItemIndent(level) - self.buf.write('"value":') + self.buf.write('"data":') self._MaybeSpace() def _PrintBashSuffix(self, level): diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index b2507f647c..3b5f66728a 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -152,10 +152,10 @@ assoc['k3']= pp line (assoc) ## STDOUT: -{"type":"BashArray","value":{"0":"a","1":"b","2":"c"}} -{"type":"BashArray","value":{"0":"a","1":"b","2":"c","5":"z"}} -{"type":"BashAssoc","value":{"k":"v","k2":"v2"}} -{"type":"BashAssoc","value":{"k":"v","k2":"v2","k3":""}} +{"type":"BashArray","data":{"0":"a","1":"b","2":"c"}} +{"type":"BashArray","data":{"0":"a","1":"b","2":"c","5":"z"}} +{"type":"BashAssoc","data":{"k":"v","k2":"v2"}} +{"type":"BashAssoc","data":{"k":"v","k2":"v2","k3":""}} ## END diff --git a/spec/ysh-expr-bool.test.sh b/spec/ysh-expr-bool.test.sh index e3511c1845..4cfc524165 100644 --- a/spec/ysh-expr-bool.test.sh +++ b/spec/ysh-expr-bool.test.sh @@ -167,8 +167,8 @@ declare -A assoc=([k]=v) pp line (assoc or 'zz') ## STDOUT: -{"type":"BashArray","value":{"0":"1","1":"2","2":"3"}} -{"type":"BashAssoc","value":{"k":"v"}} +{"type":"BashArray","data":{"0":"1","1":"2","2":"3"}} +{"type":"BashAssoc","data":{"k":"v"}} ## END #### x if b else y diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index 7082331bed..2bfee0b0d0 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -1223,11 +1223,11 @@ json write (array) ## STDOUT: { "type": "BashArray", - "value": {} + "data": {} } { "type": "BashArray", - "value": { + "data": { "0": "x", "1": "y", "5": "z" @@ -1247,11 +1247,11 @@ json write (assoc) ## STDOUT: { "type": "BashAssoc", - "value": {} + "data": {} } { "type": "BashAssoc", - "value": { + "data": { "foo": "bar", "42": "43" } diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index dd19aa5184..4b14fd4751 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -107,11 +107,11 @@ pp line ({k: array_1}) (Dict) {k: (BashArray)} (Dict) {k: (BashArray 'hello')} -{"type":"BashArray","value":{}} -{"type":"BashArray","value":{"0":"hello"}} +{"type":"BashArray","data":{}} +{"type":"BashArray","data":{"0":"hello"}} -(Dict) {"k":{"type":"BashArray","value":{}}} -(Dict) {"k":{"type":"BashArray","value":{"0":"hello"}}} +(Dict) {"k":{"type":"BashArray","data":{}}} +(Dict) {"k":{"type":"BashArray","data":{"0":"hello"}}} ## END #### BashArray, long @@ -157,11 +157,11 @@ pp line ({k:assoc}) (Dict) {k: (BashAssoc)} (Dict) {k: (BashAssoc ['k']=$'foo \u0001μ')} -{"type":"BashAssoc","value":{}} -{"type":"BashAssoc","value":{"k":"foo \u0001μ"}} +{"type":"BashAssoc","data":{}} +{"type":"BashAssoc","data":{"k":"foo \u0001μ"}} -(Dict) {"k":{"type":"BashAssoc","value":{}}} -(Dict) {"k":{"type":"BashAssoc","value":{"k":"foo \u0001μ"}}} +(Dict) {"k":{"type":"BashAssoc","data":{}}} +(Dict) {"k":{"type":"BashAssoc","data":{"k":"foo \u0001μ"}}} ## END From 12f9721f76098230f4271f23cfcab31fe29cf8b5 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 26 Jul 2024 00:13:54 -0400 Subject: [PATCH 044/506] [j8] Fix bug in printing of BashArray, BashAssoc The 'level' var got messed up, and somehow this only manifested in C++! Add a DCHECK() that caught the bug. --- data_lang/j8.py | 14 +++++--------- mycpp/gc_mylib.cc | 1 + 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/data_lang/j8.py b/data_lang/j8.py index a0247d4d30..f6eb999169 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -340,7 +340,6 @@ def _PrintBashPrefix(self, type_str, level): def _PrintBashSuffix(self, level): # type: (int) -> None - level -= 1 self._MaybeNewline() self._BracketIndent(level) self.buf.write('}') @@ -356,7 +355,6 @@ def _PrintBashArray(self, val, level): self.buf.write('{') self._MaybeNewline() - level += 1 first = True for i, s in enumerate(val.strs): if s is None: @@ -366,9 +364,9 @@ def _PrintBashArray(self, val, level): self.buf.write(',') self._MaybeNewline() - self._ItemIndent(level) - + self._ItemIndent(level + 1) pyj8.WriteString(str(i), self.options, self.buf) + self.buf.write(':') self._MaybeSpace() @@ -378,7 +376,7 @@ def _PrintBashArray(self, val, level): self._MaybeNewline() - self._BracketIndent(level) + self._BracketIndent(level + 1) self.buf.write('}') self._PrintBashSuffix(level) @@ -394,15 +392,13 @@ def _PrintBashAssoc(self, val, level): self.buf.write('{') self._MaybeNewline() - level += 1 i = 0 for k2, v2 in iteritems(val.d): if i != 0: self.buf.write(',') self._MaybeNewline() - self._ItemIndent(level) - + self._ItemIndent(level + 1) pyj8.WriteString(k2, self.options, self.buf) self.buf.write(':') @@ -414,7 +410,7 @@ def _PrintBashAssoc(self, val, level): self._MaybeNewline() - self._BracketIndent(level) + self._BracketIndent(level + 1) self.buf.write('}') self._PrintBashSuffix(level) diff --git a/mycpp/gc_mylib.cc b/mycpp/gc_mylib.cc index cdd6e77658..dfa90d2f09 100644 --- a/mycpp/gc_mylib.cc +++ b/mycpp/gc_mylib.cc @@ -270,6 +270,7 @@ void BufWriter::write(BigStr* s) { } void BufWriter::write_spaces(int n) { + DCHECK(n >= 0); if (n == 0) { return; } From 92b3fd349a06c44c7528d376df20efce3ad96b37 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 26 Jul 2024 01:19:30 -0400 Subject: [PATCH 045/506] [ysh] Remove dict to SparseArray function that is, _d2sp _a2sp seems to be enough. --- builtin/func_misc.py | 28 ---------------------------- core/shell.py | 1 - demo/sparse-array.sh | 1 - spec/ble-idioms.test.sh | 23 +---------------------- 4 files changed, 1 insertion(+), 52 deletions(-) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 7f983b29b2..1cb98adf95 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -569,34 +569,6 @@ def Call(self, rd): return value.SparseArray(d, max_index) -class DictToSparse(vm._Callable): - """ - value.Dict -> value.SparseArray, for testing - """ - - def __init__(self): - # type: () -> None - pass - - def Call(self, rd): - # type: (typed_args.Reader) -> value_t - - d = rd.PosDict() - rd.Done() - - blame_tok = rd.LeftParenToken() - - mydict = {} # type: Dict[mops.BigInt, str] - for k, v in iteritems(d): - i = mops.FromStr(k) - s = val_ops.ToStr(v, 'expected str', blame_tok) - - mydict[i] = s - - max_index = mops.MINUS_ONE # TODO: - return value.SparseArray(mydict, max_index) - - class SparseOp(vm._Callable): """ All ops on value.SparseArray, for testing performance diff --git a/core/shell.py b/core/shell.py index e6dcd85fa7..01290722f4 100644 --- a/core/shell.py +++ b/core/shell.py @@ -868,7 +868,6 @@ def Main( # Demos _SetGlobalFunc(mem, '_a2sp', func_misc.BashArrayToSparse()) - _SetGlobalFunc(mem, '_d2sp', func_misc.DictToSparse()) _SetGlobalFunc(mem, '_opsp', func_misc.SparseOp()) mem.SetNamed(location.LName('_io'), global_io, scope_e.GlobalOnly) diff --git a/demo/sparse-array.sh b/demo/sparse-array.sh index 51465bf34c..bc0c163bf4 100755 --- a/demo/sparse-array.sh +++ b/demo/sparse-array.sh @@ -8,7 +8,6 @@ # # core/shell.py defines these functions: # _a2sp -# _d2sp # _opsp # builtin/func_misc.py is where they are implemented # diff --git a/spec/ble-idioms.test.sh b/spec/ble-idioms.test.sh index 6e0ab94f11..5615044d9d 100644 --- a/spec/ble-idioms.test.sh +++ b/spec/ble-idioms.test.sh @@ -270,7 +270,7 @@ echo "${a[@]}" ## END -#### Performance demo +#### SparseArray Performance demo case $SH in bash|zsh|mksh|ash) exit ;; esac @@ -312,24 +312,6 @@ call _opsp(sp, 'unset', 11) echo subst: @[_opsp(sp, 'subst')] echo keys: @[_opsp(sp, 'keys')] -echo --- - -# Sparse -var d = { - '1': 'a', - '10': 'b', - '100': 'c', - '1000': 'd', - '10000': 'e', - '100000': 'f', -} - -var sp2 = _d2sp(d) - -echo len: $[_opsp(sp2, 'len')] -echo subst: @[_opsp(sp2, 'subst')] - - ## STDOUT: SparseArray len: 6 @@ -346,9 +328,6 @@ keys: 0 1 2 3 4 10 11 12 unset subst: set0 25 26 27 bar sparse y keys: 0 1 2 3 4 10 12 ---- -len: 6 -subst: a b c d e f ## END ## N-I bash/zsh/mksh/ash STDOUT: From 61fceb81444581c106afbebd3a88b58e9d56c00b Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 26 Jul 2024 02:21:56 -0400 Subject: [PATCH 046/506] [pretty, json] Support for SparseArray This will eventually become value.BashArray. Demo: $ declare -a a=(foo bar) $ a[5]=5 # old format $ = a (BashArray 'foo' 'bar' null null null '5') # new format - array to sparse $ = _a2sp(a) (SparseArray [0]='foo' [1]='bar' [5]='5') --- data_lang/j8.py | 46 ++++++++++++++++++++++++++++++++------- data_lang/pretty.py | 21 ++++++++++++++++++ spec/ysh-printing.test.sh | 37 +++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 8 deletions(-) diff --git a/data_lang/j8.py b/data_lang/j8.py index f6eb999169..79f1817baa 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -344,6 +344,41 @@ def _PrintBashSuffix(self, level): self._BracketIndent(level) self.buf.write('}') + def _PrintSparseArray(self, val, level): + # type: (value.SparseArray, int) -> None + + self._PrintBashPrefix('"SparseArray",', level) + + if len(val.d) == 0: # Special case like Python/JS + self.buf.write('{}') + else: + self.buf.write('{') + self._MaybeNewline() + + first = True + i = 0 + for k, v in iteritems(val.d): + if i != 0: + self.buf.write(',') + self._MaybeNewline() + + self._ItemIndent(level + 1) + pyj8.WriteString(mops.ToStr(k), self.options, self.buf) + + self.buf.write(':') + self._MaybeSpace() + + pyj8.WriteString(v, self.options, self.buf) + + i += 1 + + self._MaybeNewline() + + self._BracketIndent(level + 1) + self.buf.write('}') + + self._PrintBashSuffix(level) + def _PrintBashArray(self, val, level): # type: (value.BashArray, int) -> None @@ -529,14 +564,9 @@ def Print(self, val, level=0): self._PrintDict(val, level) self.visited[heap_id] = FINISHED - # TODO: New format, which should consistent with pretty printing - # pp line (x) supports BashArray and BashAssoc, e.g. for spec - # tests. - - # - BashAssoc is Dict[str, str] - # (BashAssoc ['1']='foo' ['3']='bar') - # - BashArray will be Dict[int, str] - SparseArray. We should write it like - # (BashArray [1]='foo' [3]='bar') + elif case(value_e.SparseArray): + val = cast(value.SparseArray, UP_val) + self._PrintSparseArray(val, level) elif case(value_e.BashArray): val = cast(value.BashArray, UP_val) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 0e2051a6b0..358d0a973f 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -691,6 +691,23 @@ def _BashAssoc(self, vassoc): return self._SurroundedAndPrefixed("(", type_name, " ", self._Join(mdocs, "", " "), ")") + def _SparseArray(self, val): + # type: (value.SparseArray) -> MeasuredDoc + type_name = self._Styled(self.type_style, _Text("SparseArray")) + if len(val.d) == 0: + return _Concat([_Text("("), type_name, _Text(")")]) + mdocs = [] # type: List[MeasuredDoc] + for k2, v2 in iteritems(val.d): + mdocs.append( + _Concat([ + _Text("["), + self._Styled(self.int_style, _Text(mops.ToStr(k2))), + _Text("]="), + self._BashStringLiteral(v2) + ])) + return self._SurroundedAndPrefixed("(", type_name, " ", + self._Join(mdocs, "", " "), ")") + def _Value(self, val): # type: (value_t) -> MeasuredDoc @@ -753,6 +770,10 @@ def _Value(self, val): self.visiting[heap_id] = False return result + elif case(value_e.SparseArray): + sparse = cast(value.SparseArray, val) + return self._SparseArray(sparse) + elif case(value_e.BashArray): varray = cast(value.BashArray, val) return self._BashArray(varray) diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index 4b14fd4751..ede578fa47 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -81,6 +81,43 @@ pp line ({k: pat}) | remove-addr (Dict) {"k":} ## END +#### SparseArray, new representation for bash array +declare -a empty=() +declare -a array_1=(hello) +array_1[5]=5 + +var empty = _a2sp(empty) +var array_1 = _a2sp(array_1) + +pp (empty) +pp (array_1) +echo + +pp ({k: empty}) +pp ({k: array_1}) +echo + +pp line (empty) +pp line (array_1) +echo + +pp line ({k: empty}) +pp line ({k: array_1}) + +## STDOUT: +(SparseArray) +(SparseArray [0]='hello' [5]='5') + +(Dict) {k: (SparseArray)} +(Dict) {k: (SparseArray [0]='hello' [5]='5')} + +{"type":"SparseArray","data":{}} +{"type":"SparseArray","data":{"0":"hello","5":"5"}} + +(Dict) {"k":{"type":"SparseArray","data":{}}} +(Dict) {"k":{"type":"SparseArray","data":{"0":"hello","5":"5"}}} +## END + #### BashArray, short declare -a empty=() declare -a array_1=(hello) From 276634344b6e9adacd3d9746bbe3633432c95db8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 26 Jul 2024 11:33:13 -0400 Subject: [PATCH 047/506] [ysh/builtin] First pass of assert builtin It accepts all of these forms: assert (false) assert [false] # evaluates it for you assert (42 === f()) # eagerly evaluated, not special assert [42 === f()] # evaluates it for you, prints error message We do NOT yet quote the code. But the error is already pretty good without that. We probably want to use the pretty printer to show the values wrapped, and in color. A diff may also be useful, but the user can also invoke 'diff' themselves. --- builtin/error_ysh.py | 79 ++++++++++++++++- builtin/func_misc.py | 21 ----- core/shell.py | 2 +- data_lang/j8.py | 13 ++- demo/url-search-params.ysh | 48 ++++++----- doc/ref/chap-builtin-cmd.md | 21 +++++ doc/ref/toc-ysh.md | 2 +- frontend/builtin_def.py | 1 + frontend/flag_def.py | 1 + spec/ysh-builtin-error.test.sh | 149 +++++++++++++++++++++++++++++++++ spec/ysh-list.test.sh | 16 ++-- 11 files changed, 299 insertions(+), 54 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index 86c4fc3233..e9ca9af643 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -1,8 +1,9 @@ from __future__ import print_function from _devbuild.gen.option_asdl import option_i +from _devbuild.gen.id_kind_asdl import Id from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus -from _devbuild.gen.syntax_asdl import loc +from _devbuild.gen.syntax_asdl import loc, loc_t, expr, expr_e from _devbuild.gen.value_asdl import value, value_e from core import error from core.error import e_die_status, e_usage @@ -10,10 +11,12 @@ from core import num from core import state from core import vm +from data_lang import j8 from frontend import flag_util from frontend import typed_args from mycpp import mops from mycpp.mylib import tagswitch, log +from ysh import val_ops _ = log @@ -21,6 +24,7 @@ if TYPE_CHECKING: from core import ui from osh import cmd_eval + from ysh import expr_eval class ctx_Try(object): @@ -222,3 +226,76 @@ def Run(self, cmd_val): locs[0]) return status + + +class Assert(vm._Builtin): + + def __init__(self, expr_ev, errfmt): + # type: (expr_eval.ExprEvaluator, ui.ErrorFormatter) -> None + self.expr_ev = expr_ev + self.errfmt = errfmt + + def _AssertComparison(self, exp, blame_loc): + # type: (expr.Compare, loc_t) -> None + + # We checked exp.ops + assert len(exp.comparators) == 1, exp.comparators + + expected = self.expr_ev.EvalExpr(exp.left, loc.Missing) + actual = self.expr_ev.EvalExpr(exp.comparators[0], loc.Missing) + + if not val_ops.ExactlyEqual(expected, actual, blame_loc): + self.errfmt.StderrLine('') + self.errfmt.StderrLine(' Expected: %s' % j8.Repr(expected)) + self.errfmt.StderrLine(' Got: %s' % j8.Repr(actual)) + + raise error.Expr("Not equal", exp.ops[0]) + + def _AssertExpression(self, val, blame_loc): + # type: (value.Expr, loc_t) -> None + + # Special case for assert [true === f()] + exp = val.e + UP_exp = exp + with tagswitch(exp) as case: + if case(expr_e.Compare): + exp = cast(expr.Compare, UP_exp) + + # Only assert [x === y] is treated as special + # Not assert [x === y === z] + if len(exp.ops) == 1: + id_ = exp.ops[0].id + if id_ == Id.Expr_TEqual: + self._AssertComparison(exp, blame_loc) + return + + # Any other expression + result = self.expr_ev.EvalExpr(val.e, blame_loc) + b = val_ops.ToBool(result) + if not b: + s = j8.Repr(result) + raise error.Expr('Assertion (of expr) %s' % s, blame_loc) + + def Run(self, cmd_val): + # type: (cmd_value.Argv) -> int + + _, arg_r = flag_util.ParseCmdVal('assert', + cmd_val, + accept_typed_args=True) + + rd = typed_args.ReaderForProc(cmd_val) + val = rd.PosValue() + rd.Done() + + UP_val = val + with tagswitch(val) as case: + if case(value_e.Expr): # Destructured assert [true === f()] + val = cast(value.Expr, UP_val) + self._AssertExpression(val, rd.LeftParenToken()) + else: + b = val_ops.ToBool(val) + if not b: + raise error.Expr('Assert: %s' % j8.Repr(val), + rd.LeftParenToken()) + + return 0 diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 1cb98adf95..547f8b5069 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -443,27 +443,6 @@ def Call(self, rd): return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal) -class Assert(vm._Callable): - - def __init__(self): - # type: () -> None - pass - - def Call(self, rd): - # type: (typed_args.Reader) -> value_t - - val = rd.PosValue() - - msg = rd.OptionalStr(default_='') - - rd.Done() - - if not val_ops.ToBool(val): - raise error.AssertionErr(msg, rd.LeftParenToken()) - - return value.Null - - class EvalExpr(vm._Callable): def __init__(self, expr_ev): diff --git a/core/shell.py b/core/shell.py index 01290722f4..a9cb87da1c 100644 --- a/core/shell.py +++ b/core/shell.py @@ -616,6 +616,7 @@ def Main( b[builtin_i.boolstatus] = error_ysh.BoolStatus(shell_ex, errfmt) b[builtin_i.try_] = error_ysh.Try(mutable_opts, mem, cmd_ev, shell_ex, errfmt) + b[builtin_i.assert_] = error_ysh.Assert(expr_ev, errfmt) # Pure builtins true_ = pure_osh.Boolean(0) @@ -857,7 +858,6 @@ def Main( _SetGlobalFunc(mem, 'shvarGet', func_misc.Shvar_get(mem)) _SetGlobalFunc(mem, 'getVar', func_misc.GetVar(mem)) - _SetGlobalFunc(mem, 'assert_', func_misc.Assert()) # Serialize _SetGlobalFunc(mem, 'toJson8', func_misc.ToJson8(True)) diff --git a/data_lang/j8.py b/data_lang/j8.py index 79f1817baa..fdd6991b80 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -184,13 +184,24 @@ def PrintLine(val, f): # error.Encode should be impossible - we show cycles and non-data buf = mylib.BufWriter() - # TODO: Omit type at top level _Print(val, buf, -1, options=SHOW_CYCLES | SHOW_NON_DATA) f.write(buf.getvalue()) f.write('\n') +def Repr(val): + # type: (value_t) -> str + """ For assert [x] + + This is like Python's repr + """ + # error.Encode should be impossible - we show cycles and non-data + buf = mylib.BufWriter() + _Print(val, buf, -1, options=SHOW_CYCLES | SHOW_NON_DATA) + return buf.getvalue() + + def EncodeString(s, buf, unquoted_ok=False): # type: (str, mylib.BufWriter, bool) -> None """ For pp proc, etc.""" diff --git a/demo/url-search-params.ysh b/demo/url-search-params.ysh index ad501d282e..7477cb746c 100755 --- a/demo/url-search-params.ysh +++ b/demo/url-search-params.ysh @@ -35,7 +35,7 @@ # - need assert [x] for testing # - task files need completion # -# - Eggex can use multiline /// syntax +# - Eggex can use multiline /// syntax, though you can use \ for line continuation # - Eggex could use "which" match # - m=>group('lit') sorta bothers me, it should be # - m.group('lit') @@ -52,13 +52,6 @@ source $LIB_OSH/task-five.sh #source $LIB_YSH/yblocks.ysh -proc _check (; val) { # TODO: assert - if (not val) { - pp line (val) - error "Failed: $val" - } -} - func strFromTwoHex(two_hex) { var result # TODO: provide alternative to old OSH style! @@ -72,7 +65,11 @@ func strFromTwoHex(two_hex) { const Hex = / [0-9 a-f A-F] / -const Quoted = / | | '%' / +const Quoted = / \ + \ + | \ + | '%' \ + / func unquote (s) { ### Turn strings with %20 into space, etc. @@ -144,12 +141,16 @@ proc test-part() { #_check ('foo bar' === unquote('foo+bar')) for s in (PART_CASES) { - js-decode-part $s | json read + js-decode-part $s | json read (&js) echo 'JS' - pp line (_reply) + pp line (js) echo 'YSH' - = unquote(s) + var y = unquote(s) + pp line (y) + + assert [y === js] + echo #break } @@ -228,9 +229,6 @@ const QUERY_CASES = [ 'k=v&foo%23=bar+baz+%24%25&k=v', 'foo+bar=z', - # JavaScript converts %ff to the Unicode replacement char - its strings can't represent bytes - 'foo%ffbar=z', - 'missing_val=&k=', '=missing_key&=m2', @@ -239,28 +237,36 @@ const QUERY_CASES = [ '=&=', '=&=&', +] + +const OTHER_CASES = [ + + # JavaScript converts %ff to the Unicode replacement char - its strings can't represent bytes + 'foo%ffbar=z', + # JavaScript treats = as literal - that seems wrong # YSH treating this as an error seems right - #'==', + '==', ] -proc test-query() { - #_check ('foo bar' === unquote('foo+bar')) +proc test-query() { for s in (QUERY_CASES) { + #for s in (OTHER_CASES) { echo 'INPUT' echo " $s" - js-decode-query $s | json read + js-decode-query $s | json read (&js) echo 'JS' - pp line (_reply) + pp line (js) echo 'YSH' var pairs = URLSearchParams(s) pp line (pairs) + assert [pairs === js] + echo - #break } } diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index 3e5ce94aad..f7651f38e1 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -155,6 +155,27 @@ Runs a command, and requires the exit code to be 0 or 1. It's meant for external commands that "return" more than 2 values, like true / false / fail, rather than pass / fail. +### assert + +Evaluates and expression, and fails if it is not truthy. + + assert (false) # fails + assert [false] # also fails (the expression is evaluated) + +It's common to pass an unevaluated expression with `===`: + + func f() { return (42) } + + assert [43 === f()] + +In this special case, you get a nicer error message: + +> Expected: 43 +> Got: 42 + +That is, the left-hand side should be the expected value, and the right-hand +side should be the actual value. + ## Shell State ### ysh-cd diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 2ef0917649..0048ec7405 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -110,6 +110,7 @@ X [Wok] _field() try Run with errexit, set _error failed Test if _error.code !== 0 boolstatus Enforce 0 or 1 exit status + assert assert [42 === f(x)] [Shell State] ysh-cd ysh-shopt compatible, and takes a block shvar Temporary modify global settings ctx Share and update a temporary "context" @@ -128,7 +129,6 @@ X [Wok] _field() [Completion] compadjust compexport [Data Formats] json read write json8 read write -X [Testing] assert takes an expression ```

diff --git a/frontend/builtin_def.py b/frontend/builtin_def.py index 35f438790a..80a0c43153 100644 --- a/frontend/builtin_def.py +++ b/frontend/builtin_def.py @@ -127,6 +127,7 @@ def _Init(b): b.Add('true', enum_name='true_') # C++ Keywords b.Add('false', enum_name='false_') b.Add('try', enum_name='try_') + b.Add('assert', enum_name='assert_') # avoid Python keyword for name in _NORMAL_BUILTINS: b.Add(name) diff --git a/frontend/flag_def.py b/frontend/flag_def.py index 735b2474c9..9998fa418a 100644 --- a/frontend/flag_def.py +++ b/frontend/flag_def.py @@ -437,6 +437,7 @@ def _DefineCompletionActions(spec): FAILED_SPEC = FlagSpec('failed') BOOLSTATUS_SPEC = FlagSpec('boolstatus') +ASSERT_SPEC = FlagSpec('assert') # Future directions: # run --builtin, run --command, run --proc: diff --git a/spec/ysh-builtin-error.test.sh b/spec/ysh-builtin-error.test.sh index e3790084ce..69f368f25f 100644 --- a/spec/ysh-builtin-error.test.sh +++ b/spec/ysh-builtin-error.test.sh @@ -261,4 +261,153 @@ ok 2 ## END +#### assert on values +try { + $SH -c ' + assert (true) + echo passed + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + func f() { return (false) } + + assert (f()) + echo "unreachable" + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + assert (null) + echo "unreachable" + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + func f() { return (false) } + + assert (true === f()) + echo "unreachable" + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + assert (42 === 42) + echo passed + ' +} +echo code $[_error.code] +echo + +## STDOUT: +passed +code 0 + +code 3 + +code 3 + +code 3 + +passed +code 0 + +## END + + +#### assert on expressions + +try { + $SH -c ' + assert [true] + echo passed + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + func f() { return (false) } + + assert [f()] + echo "unreachable" + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + assert [null] + echo "unreachable" + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + func f() { return (false) } + + assert [true === f()] + echo "unreachable" + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + assert [42 === 42] + echo passed + ' +} +echo code $[_error.code] +echo + +## STDOUT: +passed +code 0 + +code 3 + +code 3 + +code 3 + +passed +code 0 + +## END + + +#### assert on chained comparison expression is not special + +try { + $SH -c ' + #pp line (42 === 42 === 43) + assert [42 === 42 === 43] + echo unreachable + ' +} +echo code $[_error.code] +echo + +## STDOUT: +code 3 + +## END diff --git a/spec/ysh-list.test.sh b/spec/ysh-list.test.sh index 05d9a8553c..50482e6115 100644 --- a/spec/ysh-list.test.sh +++ b/spec/ysh-list.test.sh @@ -70,14 +70,14 @@ echo $[len(l)] ## END #### List append()/extend() should return null -shopt -s oil:all +shopt -s ysh:all var l = list(1..3) var result = l->extend(list(3..6)) -call assert_(result === null) +assert [null === result] setvar result = l->append(6) -call assert_(result === null) +assert [null === result] echo pass ## STDOUT: @@ -85,12 +85,12 @@ pass ## END #### List pop() -shopt -s oil:all +shopt -s ysh:all var l = list(1..5) -call assert_(l->pop() === 4) -call assert_(l->pop() === 3) -call assert_(l->pop() === 2) -call assert_(l->pop() === 1) +assert [4 === l->pop()] +assert [3 === l->pop()] +assert [2 === l->pop()] +assert [1 === l->pop()] echo pass ## STDOUT: pass From 33cccc4effe3bafee6daf5828f7d084f392d4538 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 26 Jul 2024 21:56:25 -0400 Subject: [PATCH 048/506] [builtin/assert] Tweak error messages I also wonder if we can use the pretty printer in all 3 cases ... hm --- builtin/error_ysh.py | 12 +++++------- core/ui.py | 4 +--- spec/ysh-builtin-error.test.sh | 28 ++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index e9ca9af643..4734b65273 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -263,18 +263,16 @@ def _AssertExpression(self, val, blame_loc): # Only assert [x === y] is treated as special # Not assert [x === y === z] - if len(exp.ops) == 1: - id_ = exp.ops[0].id - if id_ == Id.Expr_TEqual: - self._AssertComparison(exp, blame_loc) - return + if len(exp.ops) == 1 and exp.ops[0].id == Id.Expr_TEqual: + self._AssertComparison(exp, blame_loc) + return # Any other expression result = self.expr_ev.EvalExpr(val.e, blame_loc) b = val_ops.ToBool(result) if not b: s = j8.Repr(result) - raise error.Expr('Assertion (of expr) %s' % s, blame_loc) + raise error.Expr("Expression isn't true: %s" % s, blame_loc) def Run(self, cmd_val): # type: (cmd_value.Argv) -> int @@ -295,7 +293,7 @@ def Run(self, cmd_val): else: b = val_ops.ToBool(val) if not b: - raise error.Expr('Assert: %s' % j8.Repr(val), + raise error.Expr("Value isn't true: %s" % j8.Repr(val), rd.LeftParenToken()) return 0 diff --git a/core/ui.py b/core/ui.py index c8d094da2d..297ceab546 100644 --- a/core/ui.py +++ b/core/ui.py @@ -533,7 +533,5 @@ def PrettyPrintValue(val, f): except (IOError, OSError): pass - buf = mylib.BufWriter() - printer.PrintValue(val, buf) - f.write(buf.getvalue()) + printer.PrintValue(val, f) f.write('\n') diff --git a/spec/ysh-builtin-error.test.sh b/spec/ysh-builtin-error.test.sh index 69f368f25f..24f7eac515 100644 --- a/spec/ysh-builtin-error.test.sh +++ b/spec/ysh-builtin-error.test.sh @@ -395,6 +395,34 @@ code 0 ## END +#### assert on expression that fails + +try { + $SH -c ' + assert [NAN === 1/0] # not true + echo unreachable + ' +} +echo code $[_error.code] +echo + +try { + $SH -c ' + assert ["oof" === $(false)] + echo unreachable + ' +} +echo code $[_error.code] +echo + + +## STDOUT: +code 3 + +code 1 + +## END + #### assert on chained comparison expression is not special try { From 8c732abfd61aad7e47d2f728b21866f3480277a4 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 26 Jul 2024 22:16:38 -0400 Subject: [PATCH 049/506] [fix] Restore BufWriter We use write_spaces() internally --- core/ui.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/ui.py b/core/ui.py index 297ceab546..c8d094da2d 100644 --- a/core/ui.py +++ b/core/ui.py @@ -533,5 +533,7 @@ def PrettyPrintValue(val, f): except (IOError, OSError): pass - printer.PrintValue(val, f) + buf = mylib.BufWriter() + printer.PrintValue(val, buf) + f.write(buf.getvalue()) f.write('\n') From 51ef077ed80d92a264fb6a38bb26e06abd51b7c2 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 26 Jul 2024 23:34:57 -0400 Subject: [PATCH 050/506] [pretty refactor] Decouple printer and value "encoder" I want to add another caller -- the assert builtin. It will have extra prefixes for values. The type prefix should also be separated. --- builtin/error_ysh.py | 10 ++++ core/ui.py | 10 ++-- data_lang/pretty.py | 104 +++++++++++++++++---------------------- data_lang/pretty_test.py | 16 +++--- 4 files changed, 72 insertions(+), 68 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index 4734b65273..8f85ea8834 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -228,6 +228,10 @@ def Run(self, cmd_val): return status +#from core import ui +#from mycpp import mylib + + class Assert(vm._Builtin): def __init__(self, expr_ev, errfmt): @@ -249,6 +253,12 @@ def _AssertComparison(self, exp, blame_loc): self.errfmt.StderrLine(' Expected: %s' % j8.Repr(expected)) self.errfmt.StderrLine(' Got: %s' % j8.Repr(actual)) + # Long values could also show DIFF, rather than wrapping + # We could have assert --diff or something + # TODO: Prefix + #ui.PrettyPrintValue(expected, mylib.Stdout()) + #ui.PrettyPrintValue(actual, mylib.Stdout()) + raise error.Expr("Not equal", exp.ops[0]) def _AssertExpression(self, val, blame_loc): diff --git a/core/ui.py b/core/ui.py index c8d094da2d..279ee23db1 100644 --- a/core/ui.py +++ b/core/ui.py @@ -523,9 +523,13 @@ def PrettyPrintValue(val, f): # type: (value_t, mylib.Writer) -> None """For the = keyword""" + encoder = pretty.ValueEncoder() + encoder.SetUseStyles(f.isatty()) + encoder.SetYshStyle() + + doc = encoder.Value(val) + printer = pretty.PrettyPrinter() - printer.SetUseStyles(f.isatty()) - printer.SetYshStyle() try: width = libc.get_terminal_width() if width > 0: @@ -534,6 +538,6 @@ def PrettyPrintValue(val, f): pass buf = mylib.BufWriter() - printer.PrintValue(val, buf) + printer.PrintDoc(doc, buf) f.write(buf.getvalue()) f.write('\n') diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 358d0a973f..a24e470a44 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -90,7 +90,7 @@ # newline, or -1 if it doesn't contain a Break. # # Measures are used in two steps. First, they're computed bottom-up on the -# `doc`, measuring the size of each node. Later, _PrintDoc() stores a measure in +# `doc`, measuring the size of each node. Later, PrintDoc() stores a measure in # each DocFragment. These Measures measure something different: the width from # the doc _to the end of the entire doc tree_. This second set of Measures (the # ones in the DocFragments) are computed top-down, and they're used to decide @@ -254,13 +254,6 @@ def _Flat(mdoc): ################### _DEFAULT_MAX_WIDTH = 80 -_DEFAULT_INDENTATION = 4 -_DEFAULT_USE_STYLES = True -_DEFAULT_SHOW_TYPE_PREFIX = True - -# Tuned for 'data_lang/pretty-benchmark.sh float-demo' -# TODO: might want options for float width -_DEFAULT_MAX_TABULAR_WIDTH = 22 class PrettyPrinter(object): @@ -272,11 +265,6 @@ def __init__(self): Use the Set*() methods for configuration before printing.""" self.max_width = _DEFAULT_MAX_WIDTH - self.indent = _DEFAULT_INDENTATION - self.use_styles = _DEFAULT_USE_STYLES - self.show_type_prefix = _DEFAULT_SHOW_TYPE_PREFIX - self.max_tabular_width = _DEFAULT_MAX_TABULAR_WIDTH - self.ysh_style = False def SetMaxWidth(self, max_width): # type: (int) -> None @@ -287,42 +275,6 @@ def SetMaxWidth(self, max_width): """ self.max_width = max_width - def SetIndent(self, indent): - # type: (int) -> None - """Set the number of spaces per indent.""" - self.indent = indent - - def SetUseStyles(self, use_styles): - # type: (bool) -> None - """Print with ansi colors and styles, rather than plain text.""" - self.use_styles = use_styles - - def SetShowTypePrefix(self, show_type_prefix): - # type: (bool) -> None - """Set whether or not to print a type before the top-level value. - - E.g. `(Bool) true`""" - self.show_type_prefix = show_type_prefix - - def SetMaxTabularWidth(self, max_tabular_width): - # type: (int) -> None - """Set the maximum width that list elements can be, for them to be - vertically aligned.""" - self.max_tabular_width = max_tabular_width - - def SetYshStyle(self): - # type: () -> None - self.ysh_style = True - - def PrintValue(self, val, buf): - # type: (value_t, BufWriter) -> None - """Pretty print an Oils value to a BufWriter.""" - constructor = _DocConstructor(self.indent, self.use_styles, - self.show_type_prefix, - self.max_tabular_width, self.ysh_style) - document = constructor.Value(val) - self._PrintDoc(document, buf) - def _Fits(self, prefix_len, group, suffix_measure): # type: (int, doc.Group, Measure) -> bool """Will `group` fit flat on the current line?""" @@ -330,7 +282,7 @@ def _Fits(self, prefix_len, group, suffix_measure): suffix_measure) return prefix_len + _SuffixLen(measure) <= self.max_width - def _PrintDoc(self, document, buf): + def PrintDoc(self, document, buf): # type: (MeasuredDoc, BufWriter) -> None """Pretty print a `pretty.doc` to a BufWriter.""" @@ -414,18 +366,25 @@ def _PrintDoc(self, document, buf): # Value -> Doc # ################ +_DEFAULT_INDENTATION = 4 +_DEFAULT_USE_STYLES = True +_DEFAULT_SHOW_TYPE_PREFIX = True + +# Tuned for 'data_lang/pretty-benchmark.sh float-demo' +# TODO: might want options for float width +_DEFAULT_MAX_TABULAR_WIDTH = 22 + -class _DocConstructor: +class ValueEncoder: """Converts Oils values into `doc`s, which can then be pretty printed.""" - def __init__(self, indent, use_styles, show_type_prefix, max_tabular_width, - ysh_style): - # type: (int, bool, bool, int, bool) -> None - self.indent = indent - self.use_styles = use_styles - self.show_type_prefix = show_type_prefix - self.max_tabular_width = max_tabular_width - self.ysh_style = ysh_style + def __init__(self): + # type: () -> None + self.indent = _DEFAULT_INDENTATION + self.use_styles = _DEFAULT_USE_STYLES + self.show_type_prefix = _DEFAULT_SHOW_TYPE_PREFIX + self.max_tabular_width = _DEFAULT_MAX_TABULAR_WIDTH + self.ysh_style = False self.visiting = {} # type: Dict[int, bool] @@ -438,6 +397,33 @@ def __init__(self, indent, use_styles, show_type_prefix, max_tabular_width, self.cycle_style = ansi.BOLD + ansi.BLUE self.type_style = ansi.MAGENTA + def SetIndent(self, indent): + # type: (int) -> None + """Set the number of spaces per indent.""" + self.indent = indent + + def SetUseStyles(self, use_styles): + # type: (bool) -> None + """Print with ansi colors and styles, rather than plain text.""" + self.use_styles = use_styles + + def SetShowTypePrefix(self, show_type_prefix): + # type: (bool) -> None + """Set whether or not to print a type before the top-level value. + + E.g. `(Bool) true`""" + self.show_type_prefix = show_type_prefix + + def SetMaxTabularWidth(self, max_tabular_width): + # type: (int) -> None + """Set the maximum width that list elements can be, for them to be + vertically aligned.""" + self.max_tabular_width = max_tabular_width + + def SetYshStyle(self): + # type: () -> None + self.ysh_style = True + def Value(self, val): # type: (value_t) -> MeasuredDoc """Convert an Oils value into a `doc`, which can then be pretty printed.""" diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 6a106369d9..71fd3294dc 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -26,8 +26,9 @@ class PrettyTest(unittest.TestCase): def setUp(self): # Use settings that make testing easier. self.printer = pretty.PrettyPrinter() - self.printer.SetUseStyles(False) - self.printer.SetYshStyle() + self.encoder = pretty.ValueEncoder() + self.encoder.SetUseStyles(False) + self.encoder.SetYshStyle() def assertPretty(self, width, value_str, expected, lineno=None): # type: (int, str, str, Optional[int]) -> None @@ -36,7 +37,10 @@ def assertPretty(self, width, value_str, expected, lineno=None): buf = mylib.BufWriter() self.printer.SetMaxWidth(width) - self.printer.PrintValue(val, buf) + + doc = self.encoder.Value(val) + self.printer.PrintDoc(doc, buf) + actual = buf.getvalue() if actual != expected: @@ -52,8 +56,8 @@ def assertPretty(self, width, value_str, expected, lineno=None): def testsFromFile(self): # TODO: convert tests to this new style - self.printer.SetShowTypePrefix(False) - self.printer.ysh_style = False + self.encoder.SetShowTypePrefix(False) + self.encoder.ysh_style = False chunks = [(None, -1, [])] for lineno, line in enumerate( @@ -96,7 +100,7 @@ def testsFromFile(self): self.assertPretty(width, value, expected, lineno) def testStyles(self): - self.printer.SetUseStyles(True) + self.encoder.SetUseStyles(True) self.assertPretty( 20, '[null, "ok", 15]', '(' + ansi.MAGENTA + 'List' + ansi.RESET + ')\n[' + ansi.RED + 'null' + ansi.RESET + ", " + ansi.GREEN + From eaa4a5b34071b5c8bae1026364bda3d247232c47 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 00:21:14 -0400 Subject: [PATCH 051/506] [pretty refactor] Move type prefix out of core value_t logic. This is so we can add another prefix for the 'assert' builtin. --- builtin/io_ysh.py | 3 +- core/ui.py | 36 ++++++++++++++++------- data_lang/pretty.py | 36 +++++------------------ data_lang/pretty_test.py | 62 ++++++++++++++++++++++++---------------- mycpp/mylib.py | 5 +++- 5 files changed, 76 insertions(+), 66 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index e769df2760..c3ae08513c 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -14,7 +14,6 @@ from core import ui from core import vm from data_lang import j8 -from data_lang import pretty from frontend import flag_util from frontend import match from frontend import typed_args @@ -127,7 +126,7 @@ def Run(self, cmd_val): val = rd.PosValue() rd.Done() - if pretty.TypeNotPrinted(val): + if ui.TypeNotPrinted(val): ysh_type = ui.ValType(val) self.stdout_.write('(%s) ' % ysh_type) diff --git a/core/ui.py b/core/ui.py index 279ee23db1..32e6cfb7cd 100644 --- a/core/ui.py +++ b/core/ui.py @@ -21,7 +21,7 @@ source, source_e, ) -from _devbuild.gen.value_asdl import value_t +from _devbuild.gen.value_asdl import value_e, value_t from asdl import format as fmt from data_lang import pretty from frontend import lexer @@ -519,23 +519,39 @@ def PrintAst(node, flag): ast_f.write('\n') -def PrettyPrintValue(val, f): - # type: (value_t, mylib.Writer) -> None +def TypeNotPrinted(val): + # type: (value_t) -> bool + return val.tag() in (value_e.Null, value_e.Bool, value_e.Int, + value_e.Float, value_e.Str, value_e.List, + value_e.Dict) + + +def PrettyPrintValue(val, f, max_width=-1): + # type: (value_t, mylib.Writer, int) -> None """For the = keyword""" encoder = pretty.ValueEncoder() encoder.SetUseStyles(f.isatty()) encoder.SetYshStyle() - doc = encoder.Value(val) + if TypeNotPrinted(val): + mdocs = encoder.TypePrefix(pretty.ValType(val)) + mdocs.append(encoder.Value(val)) + # TOOD: these constructor wrappers shouldn't be private + doc = pretty._Group(pretty._Concat(mdocs)) + else: + doc = encoder.Value(val) printer = pretty.PrettyPrinter() - try: - width = libc.get_terminal_width() - if width > 0: - printer.SetMaxWidth(width) - except (IOError, OSError): - pass + if max_width != -1: # for testing + printer.SetMaxWidth(max_width) + else: + try: + width = libc.get_terminal_width() + if width > 0: + printer.SetMaxWidth(width) + except (IOError, OSError): + pass buf = mylib.BufWriter() printer.PrintDoc(doc, buf) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index a24e470a44..b951b58051 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -120,13 +120,6 @@ def ValType(val): return value_str(val.tag(), dot=False) -def TypeNotPrinted(val): - # type: (value_t) -> bool - return val.tag() in (value_e.Null, value_e.Bool, value_e.Int, - value_e.Float, value_e.Str, value_e.List, - value_e.Dict) - - def _FloatString(fl): # type: (float) -> str @@ -368,7 +361,6 @@ def PrintDoc(self, document, buf): _DEFAULT_INDENTATION = 4 _DEFAULT_USE_STYLES = True -_DEFAULT_SHOW_TYPE_PREFIX = True # Tuned for 'data_lang/pretty-benchmark.sh float-demo' # TODO: might want options for float width @@ -382,7 +374,6 @@ def __init__(self): # type: () -> None self.indent = _DEFAULT_INDENTATION self.use_styles = _DEFAULT_USE_STYLES - self.show_type_prefix = _DEFAULT_SHOW_TYPE_PREFIX self.max_tabular_width = _DEFAULT_MAX_TABULAR_WIDTH self.ysh_style = False @@ -407,13 +398,6 @@ def SetUseStyles(self, use_styles): """Print with ansi colors and styles, rather than plain text.""" self.use_styles = use_styles - def SetShowTypePrefix(self, show_type_prefix): - # type: (bool) -> None - """Set whether or not to print a type before the top-level value. - - E.g. `(Bool) true`""" - self.show_type_prefix = show_type_prefix - def SetMaxTabularWidth(self, max_tabular_width): # type: (int) -> None """Set the maximum width that list elements can be, for them to be @@ -424,23 +408,17 @@ def SetYshStyle(self): # type: () -> None self.ysh_style = True + def TypePrefix(self, type_str): + # type: (str) -> List[MeasuredDoc] + type_name = self._Styled(self.type_style, _Text(type_str)) + mdocs = [_Text("("), type_name, _Text(")"), _Break(" ")] + return mdocs + def Value(self, val): # type: (value_t) -> MeasuredDoc """Convert an Oils value into a `doc`, which can then be pretty printed.""" self.visiting.clear() - if self.show_type_prefix: - # These JSON-like types have a special notation, so print type - # explicitly - if TypeNotPrinted(val): - type_name = self._Styled(self.type_style, _Text(ValType(val))) - mdocs = [_Text("("), type_name, _Text(")"), _Break(" ")] - else: - mdocs = [] - - mdocs.append(self._Value(val)) - return _Group(_Concat(mdocs)) - else: - return self._Value(val) + return self._Value(val) def _Styled(self, style, mdoc): # type: (str, MeasuredDoc) -> MeasuredDoc diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 71fd3294dc..513f3e6ca5 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -4,11 +4,11 @@ import os import unittest -from _devbuild.gen.value_asdl import value, value_t from core import ansi +from core import ui from data_lang import j8 from data_lang import pretty # module under test -from mycpp import mylib, mops +from mycpp import mylib from typing import Optional import libc @@ -16,9 +16,37 @@ TEST_DATA_FILENAME = os.path.join(os.path.dirname(__file__), "pretty_test.txt") -def IntValue(i): - # type: (int) -> value_t - return value.Int(mops.IntWiden(i)) +def _PrintCase(actual, expected, lineno=None): + if actual != expected: + # Print the different with real newlines, for easier reading. + print("ACTUAL:") + print(actual) + print("EXPECTED:") + print(expected) + print("END") + if lineno is not None: + print("ON LINE " + str(lineno + 1)) + + +class UiTest(unittest.TestCase): + """Test higher level ui.PrettyPrintValue().""" + + def assertPretty(self, width, value_str, expected): + # type: (int, str, str, Optional[int]) -> None + parser = j8.Parser(value_str, True) + val = parser.ParseValue() + + buf = mylib.BufWriter() + ui.PrettyPrintValue(val, buf, max_width=width) + + actual = buf.getvalue() + _PrintCase(actual, expected) + self.assertEqual(actual, expected) + + def testTypePrefix(self): + self.assertPretty(25, '[null, "ok", 15]', + "(List) [null, 'ok', 15]\n") + self.assertPretty(24, '[null, "ok", 15]', "(List)\n[null, 'ok', 15]\n") class PrettyTest(unittest.TestCase): @@ -42,21 +70,11 @@ def assertPretty(self, width, value_str, expected, lineno=None): self.printer.PrintDoc(doc, buf) actual = buf.getvalue() - - if actual != expected: - # Print the different with real newlines, for easier reading. - print("ACTUAL:") - print(actual) - print("EXPECTED:") - print(expected) - print("END") - if lineno is not None: - print("ON LINE " + str(lineno + 1)) - self.assertEqual(buf.getvalue(), expected) + _PrintCase(actual, expected, lineno=lineno) + self.assertEqual(actual, expected) def testsFromFile(self): # TODO: convert tests to this new style - self.encoder.SetShowTypePrefix(False) self.encoder.ysh_style = False chunks = [(None, -1, [])] @@ -102,13 +120,9 @@ def testsFromFile(self): def testStyles(self): self.encoder.SetUseStyles(True) self.assertPretty( - 20, '[null, "ok", 15]', '(' + ansi.MAGENTA + 'List' + ansi.RESET + - ')\n[' + ansi.RED + 'null' + ansi.RESET + ", " + ansi.GREEN + - "'ok'" + ansi.RESET + ", " + ansi.YELLOW + '15' + ansi.RESET + ']') - - def testTypePrefix(self): - self.assertPretty(25, '[null, "ok", 15]', "(List) [null, 'ok', 15]") - self.assertPretty(24, '[null, "ok", 15]', "(List)\n[null, 'ok', 15]") + 20, '[null, "ok", 15]', + '[' + ansi.RED + 'null' + ansi.RESET + ", " + ansi.GREEN + "'ok'" + + ansi.RESET + ", " + ansi.YELLOW + '15' + ansi.RESET + ']') if __name__ == '__main__': diff --git a/mycpp/mylib.py b/mycpp/mylib.py index eab6b2bdba..3782ee9ee0 100644 --- a/mycpp/mylib.py +++ b/mycpp/mylib.py @@ -209,6 +209,10 @@ def write(self, s): # type: (str) -> None self.parts.append(s) + def isatty(self): + # type: () -> bool + return False + def write_spaces(self, n): # type: (int) -> None """For JSON indenting. Avoid intermediate allocations in C++.""" @@ -226,7 +230,6 @@ def close(self): # type: () -> None # No-op for now - we could invalidate write()? - pass From 9fa929688b8611a9bf2a958ae3ff77a1d1f83af5 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 00:35:00 -0400 Subject: [PATCH 052/506] [pretty refactor] Simplify max_width API Now that we have separate Printer and ValueEncoder --- core/ui.py | 26 ++++++++++++++++---------- data_lang/pretty.py | 15 ++------------- data_lang/pretty_test.py | 6 +++--- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/core/ui.py b/core/ui.py index 32e6cfb7cd..739b951e30 100644 --- a/core/ui.py +++ b/core/ui.py @@ -525,6 +525,18 @@ def TypeNotPrinted(val): value_e.Float, value_e.Str, value_e.List, value_e.Dict) +def _GetMaxWidth(): + # type: () -> int + max_width = 80 # default value + try: + width = libc.get_terminal_width() + if width > 0: + max_width = width + except (IOError, OSError): + pass # leave at default + + return max_width + def PrettyPrintValue(val, f, max_width=-1): # type: (value_t, mylib.Writer, int) -> None @@ -542,16 +554,10 @@ def PrettyPrintValue(val, f, max_width=-1): else: doc = encoder.Value(val) - printer = pretty.PrettyPrinter() - if max_width != -1: # for testing - printer.SetMaxWidth(max_width) - else: - try: - width = libc.get_terminal_width() - if width > 0: - printer.SetMaxWidth(width) - except (IOError, OSError): - pass + if max_width == -1: + max_width = _GetMaxWidth() + + printer = pretty.PrettyPrinter(max_width) buf = mylib.BufWriter() printer.PrintDoc(doc, buf) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index b951b58051..50a6f8154c 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -246,26 +246,15 @@ def _Flat(mdoc): # Pretty Printing # ################### -_DEFAULT_MAX_WIDTH = 80 - class PrettyPrinter(object): """Pretty print an Oils value.""" - def __init__(self): - # type: () -> None + def __init__(self, max_width): + # type: (int) -> None """Construct a PrettyPrinter with default configuration options. Use the Set*() methods for configuration before printing.""" - self.max_width = _DEFAULT_MAX_WIDTH - - def SetMaxWidth(self, max_width): - # type: (int) -> None - """Set the maximum line width. - - Pretty printing will attempt to (but does not guarantee to) fit the doc - within this width. - """ self.max_width = max_width def _Fits(self, prefix_len, group, suffix_measure): diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 513f3e6ca5..47e76b0ef6 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -53,7 +53,6 @@ class PrettyTest(unittest.TestCase): def setUp(self): # Use settings that make testing easier. - self.printer = pretty.PrettyPrinter() self.encoder = pretty.ValueEncoder() self.encoder.SetUseStyles(False) self.encoder.SetYshStyle() @@ -64,10 +63,11 @@ def assertPretty(self, width, value_str, expected, lineno=None): val = parser.ParseValue() buf = mylib.BufWriter() - self.printer.SetMaxWidth(width) doc = self.encoder.Value(val) - self.printer.PrintDoc(doc, buf) + + printer = pretty.PrettyPrinter(width) + printer.PrintDoc(doc, buf) actual = buf.getvalue() _PrintCase(actual, expected, lineno=lineno) From 2b202ccbe1374af11a702580678079c9d53dcf53 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 00:35:56 -0400 Subject: [PATCH 053/506] [pretty] Remove redundant _Group() when printing type The top level PrintDoc() already wraps the whole thing in a _Group(). Also update some comments. --- core/ui.py | 5 +++-- data_lang/pretty.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/core/ui.py b/core/ui.py index 739b951e30..e5005ed2ef 100644 --- a/core/ui.py +++ b/core/ui.py @@ -525,6 +525,7 @@ def TypeNotPrinted(val): value_e.Float, value_e.Str, value_e.List, value_e.Dict) + def _GetMaxWidth(): # type: () -> int max_width = 80 # default value @@ -549,8 +550,8 @@ def PrettyPrintValue(val, f, max_width=-1): if TypeNotPrinted(val): mdocs = encoder.TypePrefix(pretty.ValType(val)) mdocs.append(encoder.Value(val)) - # TOOD: these constructor wrappers shouldn't be private - doc = pretty._Group(pretty._Concat(mdocs)) + # TODO: these constructor wrappers shouldn't be private + doc = pretty._Concat(mdocs) else: doc = encoder.Value(val) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 50a6f8154c..0f093916b7 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -89,12 +89,14 @@ # - Measure.nonflat is the width of the doc until the _earliest possible_ # newline, or -1 if it doesn't contain a Break. # -# Measures are used in two steps. First, they're computed bottom-up on the -# `doc`, measuring the size of each node. Later, PrintDoc() stores a measure in -# each DocFragment. These Measures measure something different: the width from -# the doc _to the end of the entire doc tree_. This second set of Measures (the -# ones in the DocFragments) are computed top-down, and they're used to decide -# for each Group whether to use flat mode or not, without needing to scan ahead. +# Measures are used in two steps: +# (1) First, they're computed bottom-up on the `doc`, measuring the size of each +# node. +# (2) Later, PrintDoc() stores a measure in each DocFragment. These Measures +# measure something different: the width from the doc _to the end of the +# entire doc tree_. This second set of Measures (the ones in the +# DocFragments) are computed top-down, and they're used to decide for each +# Group whether to use flat mode or not, without needing to scan ahead. from __future__ import print_function @@ -395,10 +397,12 @@ def SetMaxTabularWidth(self, max_tabular_width): def SetYshStyle(self): # type: () -> None + """Set the string literal style.""" self.ysh_style = True def TypePrefix(self, type_str): # type: (str) -> List[MeasuredDoc] + """Return a fragment for (List), which may break afterward.""" type_name = self._Styled(self.type_style, _Text(type_str)) mdocs = [_Text("("), type_name, _Text(")"), _Break(" ")] return mdocs From b54b3276d0668607c3aeb0f5572f2c62ca041fd8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 01:23:53 -0400 Subject: [PATCH 054/506] [pretty] Remove SetYshStyle() We always use YSH style strings, except in unit tests Also inline the default values. It occurs to me that _Value() should return a List[MeasuredDoc], so you can concatenate it to it more easily. --- core/ui.py | 1 - data_lang/pretty.py | 31 ++++++++++--------------------- data_lang/pretty_test.py | 1 - 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/core/ui.py b/core/ui.py index e5005ed2ef..a1ace5479e 100644 --- a/core/ui.py +++ b/core/ui.py @@ -545,7 +545,6 @@ def PrettyPrintValue(val, f, max_width=-1): encoder = pretty.ValueEncoder() encoder.SetUseStyles(f.isatty()) - encoder.SetYshStyle() if TypeNotPrinted(val): mdocs = encoder.TypePrefix(pretty.ValType(val)) diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 0f093916b7..f4598f1254 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -250,13 +250,9 @@ def _Flat(mdoc): class PrettyPrinter(object): - """Pretty print an Oils value.""" def __init__(self, max_width): # type: (int) -> None - """Construct a PrettyPrinter with default configuration options. - - Use the Set*() methods for configuration before printing.""" self.max_width = max_width def _Fits(self, prefix_len, group, suffix_measure): @@ -350,23 +346,21 @@ def PrintDoc(self, document, buf): # Value -> Doc # ################ -_DEFAULT_INDENTATION = 4 -_DEFAULT_USE_STYLES = True - -# Tuned for 'data_lang/pretty-benchmark.sh float-demo' -# TODO: might want options for float width -_DEFAULT_MAX_TABULAR_WIDTH = 22 - class ValueEncoder: """Converts Oils values into `doc`s, which can then be pretty printed.""" def __init__(self): # type: () -> None - self.indent = _DEFAULT_INDENTATION - self.use_styles = _DEFAULT_USE_STYLES - self.max_tabular_width = _DEFAULT_MAX_TABULAR_WIDTH - self.ysh_style = False + + # Default values + self.indent = 4 + self.use_styles = True + # Tuned for 'data_lang/pretty-benchmark.sh float-demo' + # TODO: might want options for float width + self.max_tabular_width = 22 + + self.ysh_style = True self.visiting = {} # type: Dict[int, bool] @@ -395,14 +389,9 @@ def SetMaxTabularWidth(self, max_tabular_width): vertically aligned.""" self.max_tabular_width = max_tabular_width - def SetYshStyle(self): - # type: () -> None - """Set the string literal style.""" - self.ysh_style = True - def TypePrefix(self, type_str): # type: (str) -> List[MeasuredDoc] - """Return a fragment for (List), which may break afterward.""" + """Return docs for type string "(List)", which may break afterward.""" type_name = self._Styled(self.type_style, _Text(type_str)) mdocs = [_Text("("), type_name, _Text(")"), _Break(" ")] return mdocs diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 47e76b0ef6..b9eb340464 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -55,7 +55,6 @@ def setUp(self): # Use settings that make testing easier. self.encoder = pretty.ValueEncoder() self.encoder.SetUseStyles(False) - self.encoder.SetYshStyle() def assertPretty(self, width, value_str, expected, lineno=None): # type: (int, str, str, Optional[int]) -> None From 6307343d048bb47261222f682d5ab645f1879f4e Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 02:06:16 -0400 Subject: [PATCH 055/506] [builtin/assert] Use pretty printer to show values This works pretty well! I added and optional prefix argument to ui.PrettyPrintValue(). It works with the type prefix. TODO: - Use it for the other kinds of assertions - Probably justify (Str) and (Float) etc. --- builtin/error_ysh.py | 18 +++++++----------- builtin/io_ysh.py | 2 +- core/ui.py | 16 +++++++++++++--- data_lang/pretty_test.py | 2 +- osh/cmd_eval.py | 14 +++++++++----- test/ysh-runtime-errors.sh | 12 ++++++++++++ 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index 8f85ea8834..325e87e877 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -10,14 +10,17 @@ from core import executor from core import num from core import state +from core import ui from core import vm from data_lang import j8 from frontend import flag_util from frontend import typed_args from mycpp import mops +from mycpp import mylib from mycpp.mylib import tagswitch, log from ysh import val_ops + _ = log from typing import Any, cast, TYPE_CHECKING @@ -228,16 +231,13 @@ def Run(self, cmd_val): return status -#from core import ui -#from mycpp import mylib - - class Assert(vm._Builtin): def __init__(self, expr_ev, errfmt): # type: (expr_eval.ExprEvaluator, ui.ErrorFormatter) -> None self.expr_ev = expr_ev self.errfmt = errfmt + self.f = mylib.Stdout() def _AssertComparison(self, exp, blame_loc): # type: (expr.Compare, loc_t) -> None @@ -249,15 +249,11 @@ def _AssertComparison(self, exp, blame_loc): actual = self.expr_ev.EvalExpr(exp.comparators[0], loc.Missing) if not val_ops.ExactlyEqual(expected, actual, blame_loc): - self.errfmt.StderrLine('') - self.errfmt.StderrLine(' Expected: %s' % j8.Repr(expected)) - self.errfmt.StderrLine(' Got: %s' % j8.Repr(actual)) - + self.f.write('\n') # Long values could also show DIFF, rather than wrapping # We could have assert --diff or something - # TODO: Prefix - #ui.PrettyPrintValue(expected, mylib.Stdout()) - #ui.PrettyPrintValue(actual, mylib.Stdout()) + ui.PrettyPrintValue('Expected: ', expected, self.f) + ui.PrettyPrintValue('Got: ', actual, self.f) raise error.Expr("Not equal", exp.ops[0]) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index c3ae08513c..a506d221c3 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -66,7 +66,7 @@ def Run(self, cmd_val): rd.Done() # IOError caught by caller - ui.PrettyPrintValue(val, mylib.Stdout()) + ui.PrettyPrintValue('', val, mylib.Stdout()) return 0 arg_r.Next() diff --git a/core/ui.py b/core/ui.py index a1ace5479e..ba4ff66480 100644 --- a/core/ui.py +++ b/core/ui.py @@ -539,21 +539,31 @@ def _GetMaxWidth(): return max_width -def PrettyPrintValue(val, f, max_width=-1): - # type: (value_t, mylib.Writer, int) -> None +def PrettyPrintValue(prefix, val, f, max_width=-1): + # type: (str, value_t, mylib.Writer, int) -> None """For the = keyword""" encoder = pretty.ValueEncoder() encoder.SetUseStyles(f.isatty()) + # TODO: pretty._Concat, etc. shouldn't be private if TypeNotPrinted(val): mdocs = encoder.TypePrefix(pretty.ValType(val)) mdocs.append(encoder.Value(val)) - # TODO: these constructor wrappers shouldn't be private doc = pretty._Concat(mdocs) else: doc = encoder.Value(val) + if len(prefix): + # If you want the type name to be indented, which we don't + # inner = pretty._Concat([pretty._Break(""), doc]) + + doc = pretty._Concat([ + pretty._Text(prefix), + #pretty._Break(""), + pretty._Indent(4, doc) + ]) + if max_width == -1: max_width = _GetMaxWidth() diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index b9eb340464..26cfc06e76 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -37,7 +37,7 @@ def assertPretty(self, width, value_str, expected): val = parser.ParseValue() buf = mylib.BufWriter() - ui.PrettyPrintValue(val, buf, max_width=width) + ui.PrettyPrintValue('', val, buf, max_width=width) actual = buf.getvalue() _PrintCase(actual, expected) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 797dbd218c..eefd1e1439 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -942,7 +942,7 @@ def _DoExpr(self, node): io_errors = [] # type: List[error.IOError_OSError] with vm.ctx_FlushStdout(io_errors): try: - ui.PrettyPrintValue(val, mylib.Stdout()) + ui.PrettyPrintValue('', val, mylib.Stdout()) except (IOError, OSError) as e: self.errfmt.PrintMessage( 'I/O error during = keyword: %s' % pyutil.strerror(e), @@ -1282,17 +1282,20 @@ def _DoForExpr(self, node): def _DoShFunction(self, node): # type: (command.ShFunction) -> None - if self.procs.Get(node.name) and not self.exec_opts.redefine_proc_func(): + if (self.procs.Get(node.name) and + not self.exec_opts.redefine_proc_func()): e_die( "Function %s was already defined (redefine_proc_func)" % node.name, node.name_tok) - sh_func = value.Proc(node.name, node.name_tok, proc_sig.Open, node.body, None, True) + sh_func = value.Proc(node.name, node.name_tok, proc_sig.Open, + node.body, None, True) self.procs.SetShFunc(node.name, sh_func) def _DoProc(self, node): # type: (Proc) -> None proc_name = lexer.TokenVal(node.name) - if self.procs.Get(proc_name) and not self.exec_opts.redefine_proc_func(): + if (self.procs.Get(proc_name) and + not self.exec_opts.redefine_proc_func()): e_die( "Proc %s was already defined (redefine_proc_func)" % proc_name, node.name) @@ -1304,7 +1307,8 @@ def _DoProc(self, node): proc_defaults = None # no dynamic scope - proc = value.Proc(proc_name, node.name, node.sig, node.body, proc_defaults, False) + proc = value.Proc(proc_name, node.name, node.sig, node.body, + proc_defaults, False) self.procs.SetProc(proc_name, proc) def _DoFunc(self, node): diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 21372e792f..44c41dcf60 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -924,6 +924,18 @@ pp line (d) ' } +test-assert() { + _ysh-expr-error 'assert [null === 42]' + + # One is long + _ysh-expr-error 'assert [null === list(1 .. 100)]' + + # Both are long + _ysh-expr-error ' +assert [{k: list(3 .. 50)} === list(1 .. 100)] + ' +} + soil-run-py() { run-test-funcs } From 6c7a179c84b3db5eff6cf1adaed64423a82e6336 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 02:24:33 -0400 Subject: [PATCH 056/506] [builtin/assert] Tweak error messages Also align pretty-printed values with type prefix: (Int) 42 (Float) 42.0 Prior to this change, it was a fixed 3 spaces. Now it's in column 8. --- builtin/error_ysh.py | 15 +++++++++----- data_lang/j8.py | 19 +++++++++--------- data_lang/pretty.py | 10 +++++++++- spec/ysh-builtin-error.test.sh | 12 ++++++++---- spec/ysh-int-float.test.sh | 2 +- spec/ysh-printing.test.sh | 36 ++++++++++++++++++++-------------- test/ysh-runtime-errors.sh | 12 ++++++++---- 7 files changed, 67 insertions(+), 39 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index 325e87e877..bcefd40791 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -20,7 +20,6 @@ from mycpp.mylib import tagswitch, log from ysh import val_ops - _ = log from typing import Any, cast, TYPE_CHECKING @@ -277,8 +276,10 @@ def _AssertExpression(self, val, blame_loc): result = self.expr_ev.EvalExpr(val.e, blame_loc) b = val_ops.ToBool(result) if not b: - s = j8.Repr(result) - raise error.Expr("Expression isn't true: %s" % s, blame_loc) + # Don't print the value for something like assert [x < 4] + #self.f.write('\n') + #ui.PrettyPrintValue("Expression isn't true: ", result, self.f) + raise error.Expr("Expression isn't true", blame_loc) def Run(self, cmd_val): # type: (cmd_value.Argv) -> int @@ -299,7 +300,11 @@ def Run(self, cmd_val): else: b = val_ops.ToBool(val) if not b: - raise error.Expr("Value isn't true: %s" % j8.Repr(val), - rd.LeftParenToken()) + # assert (42 === null) should be written + # assert [42 === null] to get a better error message + # But show the value anyway + self.f.write('\n') + ui.PrettyPrintValue("Value isn't true: ", val, self.f) + raise error.Expr('assertion', rd.LeftParenToken()) return 0 diff --git a/data_lang/j8.py b/data_lang/j8.py index fdd6991b80..d0347546d5 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -190,16 +190,17 @@ def PrintLine(val, f): f.write('\n') -def Repr(val): - # type: (value_t) -> str - """ For assert [x] +if 0: - This is like Python's repr - """ - # error.Encode should be impossible - we show cycles and non-data - buf = mylib.BufWriter() - _Print(val, buf, -1, options=SHOW_CYCLES | SHOW_NON_DATA) - return buf.getvalue() + def Repr(val): + # type: (value_t) -> str + """ Unused + This is like Python's repr + """ + # error.Encode should be impossible - we show cycles and non-data + buf = mylib.BufWriter() + _Print(val, buf, -1, options=SHOW_CYCLES | SHOW_NON_DATA) + return buf.getvalue() def EncodeString(s, buf, unquoted_ok=False): diff --git a/data_lang/pretty.py b/data_lang/pretty.py index f4598f1254..33c81ca17b 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -393,7 +393,15 @@ def TypePrefix(self, type_str): # type: (str) -> List[MeasuredDoc] """Return docs for type string "(List)", which may break afterward.""" type_name = self._Styled(self.type_style, _Text(type_str)) - mdocs = [_Text("("), type_name, _Text(")"), _Break(" ")] + + n = len(type_str) + # Our maximum string is "Float" + assert n <= 5, type_str + + # Start printing in column 8. Adjust to 6 because () takes 2 spaces. + spaces = ' ' * (6 - n) + + mdocs = [_Text("("), type_name, _Text(")"), _Break(spaces)] return mdocs def Value(self, val): diff --git a/spec/ysh-builtin-error.test.sh b/spec/ysh-builtin-error.test.sh index 24f7eac515..e5a5b6ef31 100644 --- a/spec/ysh-builtin-error.test.sh +++ b/spec/ysh-builtin-error.test.sh @@ -278,7 +278,7 @@ try { assert (f()) echo "unreachable" - ' + ' | grep -v Value } echo code $[_error.code] echo @@ -287,7 +287,7 @@ try { $SH -c ' assert (null) echo "unreachable" - ' + ' | grep -v Value } echo code $[_error.code] echo @@ -298,7 +298,7 @@ try { assert (true === f()) echo "unreachable" - ' + ' | grep -v Value } echo code $[_error.code] echo @@ -316,10 +316,13 @@ echo passed code 0 + code 3 + code 3 + code 3 passed @@ -365,7 +368,7 @@ try { assert [true === f()] echo "unreachable" - ' + ' | grep -v '(Bool)' } echo code $[_error.code] echo @@ -387,6 +390,7 @@ code 3 code 3 + code 3 passed diff --git a/spec/ysh-int-float.test.sh b/spec/ysh-int-float.test.sh index 3d08e75222..6c08c62572 100644 --- a/spec/ysh-int-float.test.sh +++ b/spec/ysh-int-float.test.sh @@ -139,7 +139,7 @@ nan is not nan pp line ([INFINITY, -INFINITY, NAN]) ## STDOUT: -(List) [INFINITY, -INFINITY, NAN] +(List) [INFINITY, -INFINITY, NAN] (List) [INFINITY,-INFINITY,NAN] ## END diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index ede578fa47..82a0e1a640 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -2,22 +2,28 @@ #### Int = -123 -## stdout: (Int) -123 +## STDOUT: +(Int) -123 +## END #### Float = -0.00 -## stdout: (Float) -0.0 +## STDOUT: +(Float) -0.0 +## END #### Null = null -## stdout: (Null) null +## STDOUT: +(Null) null +## END #### Bool = true = false ## STDOUT: -(Bool) true -(Bool) false +(Bool) true +(Bool) false ## END #### String @@ -48,7 +54,7 @@ pp line ({k: x}) | remove-addr ## STDOUT: (Range 1 .. 100) -(Dict) {k: (Range 1 .. 100)} +(Dict) {k: (Range 1 .. 100)} (Dict) {"k":} @@ -75,7 +81,7 @@ pp line ({k: pat}) | remove-addr ## STDOUT: -(Dict) {k: } +(Dict) {k: } (Dict) {"k":} @@ -108,8 +114,8 @@ pp line ({k: array_1}) (SparseArray) (SparseArray [0]='hello' [5]='5') -(Dict) {k: (SparseArray)} -(Dict) {k: (SparseArray [0]='hello' [5]='5')} +(Dict) {k: (SparseArray)} +(Dict) {k: (SparseArray [0]='hello' [5]='5')} {"type":"SparseArray","data":{}} {"type":"SparseArray","data":{"0":"hello","5":"5"}} @@ -141,8 +147,8 @@ pp line ({k: array_1}) (BashArray) (BashArray 'hello') -(Dict) {k: (BashArray)} -(Dict) {k: (BashArray 'hello')} +(Dict) {k: (BashArray)} +(Dict) {k: (BashArray 'hello')} {"type":"BashArray","data":{}} {"type":"BashArray","data":{"0":"hello"}} @@ -191,8 +197,8 @@ pp line ({k:assoc}) (BashAssoc) (BashAssoc ['k']=$'foo \u0001μ') -(Dict) {k: (BashAssoc)} -(Dict) {k: (BashAssoc ['k']=$'foo \u0001μ')} +(Dict) {k: (BashAssoc)} +(Dict) {k: (BashAssoc ['k']=$'foo \u0001μ')} {"type":"BashAssoc","data":{}} {"type":"BashAssoc","data":{"k":"foo \u0001μ"}} @@ -231,8 +237,8 @@ setvar cyclic_dict["live_end"] = cyclic_dict = cyclic_array = cyclic_dict ## STDOUT: -(List) ['one', 'two', [...]] -(Dict) {dead_end: null, live_end: {...}} +(List) ['one', 'two', [...]] +(Dict) {dead_end: null, live_end: {...}} ## END #### Complex Cycles diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 44c41dcf60..c2de3392be 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -925,15 +925,19 @@ pp line (d) } test-assert() { + _ysh-expr-error 'assert [0.0]' + _ysh-expr-error 'assert [3 > 4]' + + _ysh-expr-error 'assert (0)' + _ysh-expr-error 'assert (null === 42)' + _ysh-expr-error 'assert [null === 42]' # One is long - _ysh-expr-error 'assert [null === list(1 .. 100)]' + _ysh-expr-error 'assert [null === list(1 .. 50)]' # Both are long - _ysh-expr-error ' -assert [{k: list(3 .. 50)} === list(1 .. 100)] - ' + _ysh-expr-error 'assert [{k: list(3 .. 40)} === list(1 .. 50)]' } soil-run-py() { From 49671dec60a79f911d6c4e0851f044baf5e1679c Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 02:55:06 -0400 Subject: [PATCH 057/506] [fix] Unit test and lint error --- builtin/error_ysh.py | 1 - data_lang/pretty_test.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index bcefd40791..ee75a505b7 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -12,7 +12,6 @@ from core import state from core import ui from core import vm -from data_lang import j8 from frontend import flag_util from frontend import typed_args from mycpp import mops diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 26cfc06e76..06df1b82bb 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -44,9 +44,9 @@ def assertPretty(self, width, value_str, expected): self.assertEqual(actual, expected) def testTypePrefix(self): - self.assertPretty(25, '[null, "ok", 15]', - "(List) [null, 'ok', 15]\n") - self.assertPretty(24, '[null, "ok", 15]', "(List)\n[null, 'ok', 15]\n") + self.assertPretty(24, '[null, "ok", 15]', + "(List) [null, 'ok', 15]\n") + self.assertPretty(23, '[null, "ok", 15]', "(List)\n[null, 'ok', 15]\n") class PrettyTest(unittest.TestCase): From d3f6fa14cc40655c88da9cdcc82db3a0138cdffb Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 10:56:52 -0400 Subject: [PATCH 058/506] [builtin/pp] pp [x + 1] evaluates expression, and quotes code It's supposed to be like the Rust dbg!() macro TODO: update doc/ref --- builtin/io_ysh.py | 51 ++++++++++++++++++++++++++++---------- core/shell.py | 2 +- core/ui.py | 34 ++++++++++++++++++++++--- core/ui_test.py | 1 - data_lang/pretty_test.py | 3 +-- frontend/typed_args.py | 4 +-- test/ysh-runtime-errors.sh | 17 +++++++++++++ 7 files changed, 89 insertions(+), 23 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index a506d221c3..33e1910396 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -7,6 +7,7 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value from _devbuild.gen.syntax_asdl import command_e, BraceGroup, loc +from _devbuild.gen.value_asdl import value, value_e from asdl import format as fmt from core import error from core.error import e_usage @@ -18,13 +19,13 @@ from frontend import match from frontend import typed_args from mycpp import mylib -from mycpp.mylib import log +from mycpp.mylib import tagswitch, log from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from core.alloc import Arena - from core.ui import ErrorFormatter from osh import cmd_eval + from ysh import expr_eval _ = log @@ -32,7 +33,7 @@ class _Builtin(vm._Builtin): def __init__(self, mem, errfmt): - # type: (state.Mem, ErrorFormatter) -> None + # type: (state.Mem, ui.ErrorFormatter) -> None self.mem = mem self.errfmt = errfmt @@ -43,13 +44,43 @@ class Pp(_Builtin): 'pp cell a' is a lot easier to type than 'argv.py "${a[@]}"'. """ - def __init__(self, mem, errfmt, procs, arena): - # type: (state.Mem, ErrorFormatter, state.Procs, Arena) -> None + def __init__( + self, + expr_ev, # type: expr_eval.ExprEvaluator + mem, # type: state.Mem + errfmt, # type: ui.ErrorFormatter + procs, # type: state.Procs + arena, # type: Arena + ): + # type: (...) -> None _Builtin.__init__(self, mem, errfmt) + self.expr_ev = expr_ev self.procs = procs self.arena = arena self.stdout_ = mylib.Stdout() + def _PrettyPrint(self, cmd_val): + # type: (cmd_value.Argv) -> int + rd = typed_args.ReaderForProc(cmd_val) + val = rd.PosValue() + rd.Done() + + UP_val = val + with tagswitch(val) as case: + if case(value_e.Expr): # Destructured assert [true === f()] + val = cast(value.Expr, UP_val) + blame_tok = rd.LeftParenToken() + result = self.expr_ev.EvalExpr(val.e, blame_tok) + + # Show it with location + excerpt, prefix = ui.CodeExcerptAndPrefix(blame_tok) + self.stdout_.write(excerpt) + ui.PrettyPrintValue(prefix, result, self.stdout_) + else: + # IOError caught by caller + ui.PrettyPrintValue('', val, self.stdout_) + return 0 + def Run(self, cmd_val): # type: (cmd_value.Argv) -> int arg, arg_r = flag_util.ParseCmdVal('pp', @@ -61,13 +92,7 @@ def Run(self, cmd_val): # pp (x) prints in the same way that '= x' does # TODO: We also need pp [x], which shows the expression if action is None: - rd = typed_args.ReaderForProc(cmd_val) - val = rd.PosValue() - rd.Done() - - # IOError caught by caller - ui.PrettyPrintValue('', val, mylib.Stdout()) - return 0 + return self._PrettyPrint(cmd_val) arg_r.Next() @@ -192,7 +217,7 @@ class Write(_Builtin): """ def __init__(self, mem, errfmt): - # type: (state.Mem, ErrorFormatter) -> None + # type: (state.Mem, ui.ErrorFormatter) -> None _Builtin.__init__(self, mem, errfmt) self.stdout_ = mylib.Stdout() diff --git a/core/shell.py b/core/shell.py index a9cb87da1c..0108079866 100644 --- a/core/shell.py +++ b/core/shell.py @@ -646,7 +646,7 @@ def Main( b[builtin_i.fopen] = io_ysh.Fopen(mem, cmd_ev) # (pp output format isn't stable) - b[builtin_i.pp] = io_ysh.Pp(mem, errfmt, procs, arena) + b[builtin_i.pp] = io_ysh.Pp(expr_ev, mem, errfmt, procs, arena) # Input b[builtin_i.cat] = io_osh.Cat() # for $( None - """Should we have multiple error formats: + """Print an error message attached to a location. + + We may quote code this: + + echo $foo + ^~~~ + [ -c flag ]:1: Failed + + Should we have multiple locations? - single line and verbose? - and turn on "stack" tracing? For 'source' and more? """ f = mylib.Stderr() if blame_loc.tag() == loc_e.TokenTooLong: + # test/spec.sh parse-errors shows this _PrintTokenTooLong(cast(loc.TokenTooLong, blame_loc), f) return @@ -277,7 +286,7 @@ def _PrintWithLocation(prefix, msg, blame_loc, show_code): orig_col = blame_tok.col src = blame_tok.line.src line = blame_tok.line.content - line_num = blame_tok.line.line_num # overwritten by source__LValue case + line_num = blame_tok.line.line_num # overwritten by source.Reparsed case if show_code: UP_src = src @@ -324,7 +333,7 @@ def _PrintWithLocation(prefix, msg, blame_loc, show_code): f.write('%s:%d\n' % (source_str, line_num)) f.write('\n') - # Now print OUTER location, with error message + # Recursive call: Print OUTER location, with error message _PrintWithLocation(prefix, msg, src.location, show_code) return @@ -338,6 +347,23 @@ def _PrintWithLocation(prefix, msg, blame_loc, show_code): f.write('%s:%d: %s%s\n' % (source_str, line_num, prefix, msg)) +def CodeExcerptAndPrefix(blame_tok): + # type: (Token) -> Tuple[str, str] + """Return a string that quotes code, and a string location prefix. + + Similar logic as _PrintWithLocation, except we know we have a token. + """ + line = blame_tok.line + + buf = mylib.BufWriter() + _PrintCodeExcerpt(line.content, blame_tok.col, blame_tok.length, buf) + + source_str = GetLineSourceString(line, quote_filename=True) + prefix = '%s:%d: ' % (source_str, blame_tok.line.line_num) + + return buf.getvalue(), prefix + + class ctx_Location(object): def __init__(self, errfmt, location): diff --git a/core/ui_test.py b/core/ui_test.py index c60a76e6e7..088bf2b2d0 100755 --- a/core/ui_test.py +++ b/core/ui_test.py @@ -1,6 +1,5 @@ #!/usr/bin/env python2 from __future__ import print_function -"""ui_test.py: Tests for ui.py.""" import unittest diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 06df1b82bb..3437a3f79c 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -44,8 +44,7 @@ def assertPretty(self, width, value_str, expected): self.assertEqual(actual, expected) def testTypePrefix(self): - self.assertPretty(24, '[null, "ok", 15]', - "(List) [null, 'ok', 15]\n") + self.assertPretty(24, '[null, "ok", 15]', "(List) [null, 'ok', 15]\n") self.assertPretty(23, '[null, "ok", 15]', "(List)\n[null, 'ok', 15]\n") diff --git a/frontend/typed_args.py b/frontend/typed_args.py index bb8ab1c2e5..fc06ffd3ea 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -3,7 +3,7 @@ from _devbuild.gen.runtime_asdl import cmd_value from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, LiteralBlock, - command_t, expr_t) + command_t, expr_t, Token) from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch) from core import error from core.error import e_usage @@ -128,7 +128,7 @@ def SetFallbackLocation(self, blame_loc): self.fallback_loc = blame_loc def LeftParenToken(self): - # type: () -> loc_t + # type: () -> Token """ Used by functions in library/func_misc.py """ return self.arg_list.left diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index c2de3392be..77c190362d 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -940,6 +940,23 @@ test-assert() { _ysh-expr-error 'assert [{k: list(3 .. 40)} === list(1 .. 50)]' } +test-pp() { + _ysh-expr-error 'pp (42/0)' + _ysh-expr-error 'pp [42/0]' + + _ysh-expr-error 'pp [5, 6]' + + _ysh-should-run 'pp (42)' + _ysh-should-run 'var x = 42; pp (x)' + _ysh-should-run ' +var x = 42; +pp [x]' + + _ysh-should-run ' +var x = list(1 .. 50); +pp [x]' +} + soil-run-py() { run-test-funcs } From d3b9f3b8fcf93dd767d117de928f5017856cb010 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 12:29:44 -0400 Subject: [PATCH 059/506] [doc/ref] Document pp builtin soil: Try Dreamhost again, because OpalStack SSH is failing a lot. Starting yesterday. --- doc/ref/chap-builtin-cmd.md | 20 ++++++++++++++++---- soil/common.sh | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index f7651f38e1..bcc088b502 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -43,12 +43,21 @@ Similar names: [append][] ### pp -Pretty prints interpreter state. Some of these are implementation details, -subject to change. +The most common use is to pretty print expressions: -Examples: + $ var x = 42 + $ pp [x + 5] # pass unevaluated expression + myfile.ysh:1: (Int) 47 # print value with code location - pp proc # print all procs and their doc comments +You can also print a value, with no code location: + + $ pp (x + 5) + (Int) 47 + +The `pp` builtin can also print low-level interpreter state. Some of of these +are implementation details, subject to change. + +Examples: var x = :| one two | pp cell x # dump the "guts" of a cell, which is a location for a value @@ -57,6 +66,9 @@ Examples: pp line (x) # single-line stable format, for spec tests + pp proc # print all procs and their doc comments + + ## Handle Errors ### error diff --git a/soil/common.sh b/soil/common.sh index 465f7f3dc8..c85d39b4c8 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -20,7 +20,7 @@ dump-env() { env | grep -v '^encrypted_' | sort } -if false; then +if true; then readonly SOIL_USER='travis_admin' readonly SOIL_HOST='travis-ci.oilshell.org' readonly SOIL_HOST_DIR=~/travis-ci.oilshell.org # used on server From a4547502e461a507cf6d7c07dc9be91ccfd7b291 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 12:51:26 -0400 Subject: [PATCH 060/506] [refactor] Start a separate display/ directory Will put pretty printing here too. --- asdl/format.py | 2 +- bin/osh_parse.py | 2 +- builtin/assign_osh.py | 2 +- builtin/bracket_osh.py | 4 ++-- builtin/completion_osh.py | 6 +++--- builtin/dirs_osh.py | 2 +- builtin/error_ysh.py | 4 ++-- builtin/func_hay.py | 2 +- builtin/func_misc.py | 2 +- builtin/hay_ysh.py | 2 +- builtin/io_osh.py | 2 +- builtin/io_ysh.py | 2 +- builtin/json_ysh.py | 4 ++-- builtin/meta_osh.py | 2 +- builtin/meta_ysh.py | 2 +- builtin/misc_osh.py | 2 +- builtin/module_ysh.py | 2 +- builtin/printf_osh.py | 2 +- builtin/process_osh.py | 6 +++--- builtin/pure_osh.py | 2 +- builtin/pure_ysh.py | 2 +- builtin/read_osh.py | 2 +- builtin/readline_osh.py | 6 +++--- builtin/readline_osh_test.py | 2 +- builtin/trap_osh.py | 4 ++-- core/comp_ui.py | 2 +- core/completion.py | 2 +- core/dev.py | 2 +- core/executor.py | 2 +- core/main_loop.py | 7 +++---- core/process.py | 5 ++--- core/process_test.py | 2 +- core/shell.py | 2 +- core/state.py | 2 +- core/test_lib.py | 2 +- core/util.py | 2 +- data_lang/pretty.py | 2 +- data_lang/pretty_test.py | 4 ++-- display/__init__.py | 0 {core => display}/ansi.py | 0 {core => display}/ui.py | 0 {core => display}/ui_test.py | 2 +- metrics/source-code.sh | 5 +++-- osh/arith_parse_test.py | 2 +- osh/bool_parse.py | 2 +- osh/bool_stat.py | 2 +- osh/cmd_eval.py | 2 +- osh/cmd_parse.py | 2 +- osh/cmd_parse_test.py | 2 +- osh/prompt.py | 2 +- osh/sh_expr_eval.py | 7 +++---- osh/string_ops.py | 2 +- osh/tdop.py | 2 +- osh/word_eval.py | 7 +++---- osh/word_parse.py | 2 +- pea/oils-typecheck.txt | 4 ++-- prebuilt/dynamic-deps/filter-translate.txt | 1 - spec/stateful/harness.py | 2 +- test/py2_lint.py | 2 +- ysh/expr_eval.py | 2 +- ysh/expr_parse.py | 2 +- ysh/val_ops.py | 2 +- 62 files changed, 79 insertions(+), 83 deletions(-) create mode 100644 display/__init__.py rename {core => display}/ansi.py (100%) rename {core => display}/ui.py (100%) rename {core => display}/ui_test.py (93%) diff --git a/asdl/format.py b/asdl/format.py index 879d11ba02..15545866f7 100644 --- a/asdl/format.py +++ b/asdl/format.py @@ -15,7 +15,7 @@ from _devbuild.gen.hnode_asdl import (hnode, hnode_e, hnode_t, color_e, color_t) -from core import ansi +from display import ansi from data_lang import j8_lite from pylib import cgi from mycpp import mylib diff --git a/bin/osh_parse.py b/bin/osh_parse.py index 24d63acec5..9ff9f92641 100755 --- a/bin/osh_parse.py +++ b/bin/osh_parse.py @@ -13,7 +13,7 @@ #from core import main_loop from core import pyutil from core import state -from core import ui +from display import ui from frontend import parse_lib from frontend import reader from mycpp import mylib diff --git a/builtin/assign_osh.py b/builtin/assign_osh.py index 9cf47bbce6..1176417b56 100644 --- a/builtin/assign_osh.py +++ b/builtin/assign_osh.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: from core.state import Mem from core import optview - from core import ui + from display import ui from frontend.args import _Attributes _ = log diff --git a/builtin/bracket_osh.py b/builtin/bracket_osh.py index 01873a175d..c95b89a5b8 100644 --- a/builtin/bracket_osh.py +++ b/builtin/bracket_osh.py @@ -24,9 +24,9 @@ from _devbuild.gen.runtime_asdl import cmd_value from _devbuild.gen.syntax_asdl import bool_expr_t from _devbuild.gen.types_asdl import lex_mode_t - from core.ui import ErrorFormatter from core import optview from core import state + from display import ui class _StringWordEmitter(word_parse.WordEmitter): @@ -172,7 +172,7 @@ def _ThreeArgs(w_parser): class Test(vm._Builtin): def __init__(self, need_right_bracket, exec_opts, mem, errfmt): - # type: (bool, optview.Exec, state.Mem, ErrorFormatter) -> None + # type: (bool, optview.Exec, state.Mem, ui.ErrorFormatter) -> None self.need_right_bracket = need_right_bracket self.exec_opts = exec_opts self.mem = mem diff --git a/builtin/completion_osh.py b/builtin/completion_osh.py index a214bea87e..1f911c71c0 100644 --- a/builtin/completion_osh.py +++ b/builtin/completion_osh.py @@ -8,7 +8,7 @@ from core import completion from core import error from core import state -from core import ui +from display import ui from core import vm from mycpp import mylib from mycpp.mylib import log, print_stderr @@ -22,7 +22,7 @@ if TYPE_CHECKING: from _devbuild.gen.runtime_asdl import cmd_value from core.completion import Lookup, OptionState, Api, UserSpec - from core.ui import ErrorFormatter + from display import ui from frontend.args import _Attributes from frontend.parse_lib import ParseContext from osh.cmd_eval import CommandEvaluator @@ -405,7 +405,7 @@ class CompOpt(vm._Builtin): """Adjust options inside user-defined completion functions.""" def __init__(self, comp_state, errfmt): - # type: (OptionState, ErrorFormatter) -> None + # type: (OptionState, ui.ErrorFormatter) -> None self.comp_state = comp_state self.errfmt = errfmt diff --git a/builtin/dirs_osh.py b/builtin/dirs_osh.py index e67cb5dd21..0d7a715455 100644 --- a/builtin/dirs_osh.py +++ b/builtin/dirs_osh.py @@ -6,7 +6,7 @@ from core.error import e_usage from core import pyos from core import state -from core import ui +from display import ui from core import vm from frontend import flag_util from frontend import typed_args diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index ee75a505b7..eac5ea9c1a 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -10,7 +10,7 @@ from core import executor from core import num from core import state -from core import ui +from display import ui from core import vm from frontend import flag_util from frontend import typed_args @@ -23,7 +23,7 @@ from typing import Any, cast, TYPE_CHECKING if TYPE_CHECKING: - from core import ui + from display import ui from osh import cmd_eval from ysh import expr_eval diff --git a/builtin/func_hay.py b/builtin/func_hay.py index f264370252..121aae3ebd 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -9,7 +9,7 @@ from core import error from core import main_loop from core import state -from core import ui +from display import ui from core import vm from frontend import reader from frontend import typed_args diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 547f8b5069..2185e63811 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -10,7 +10,7 @@ from core import error from core import num from core import state -from core import ui +from display import ui from core import vm from data_lang import j8 from frontend import match diff --git a/builtin/hay_ysh.py b/builtin/hay_ysh.py index c1c648ee86..e9bc3ba187 100644 --- a/builtin/hay_ysh.py +++ b/builtin/hay_ysh.py @@ -10,7 +10,7 @@ from core.error import e_usage, e_die from core import num from core import state -from core import ui +from display import ui from core import vm from frontend import args from frontend import consts diff --git a/builtin/io_osh.py b/builtin/io_osh.py index 981e86eb6a..2bd05aca30 100644 --- a/builtin/io_osh.py +++ b/builtin/io_osh.py @@ -23,7 +23,7 @@ from typing import List, Dict, TYPE_CHECKING if TYPE_CHECKING: from _devbuild.gen.runtime_asdl import cmd_value - from core import ui + from display import ui from osh import cmd_eval _ = log diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 33e1910396..482a90ae12 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -12,7 +12,7 @@ from core import error from core.error import e_usage from core import state -from core import ui +from display import ui from core import vm from data_lang import j8 from frontend import flag_util diff --git a/builtin/json_ysh.py b/builtin/json_ysh.py index 719873da9f..62d87d898f 100644 --- a/builtin/json_ysh.py +++ b/builtin/json_ysh.py @@ -22,7 +22,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from core.ui import ErrorFormatter + from display import ui _ = log @@ -37,7 +37,7 @@ class Json(vm._Builtin): """ def __init__(self, mem, errfmt, is_j8): - # type: (state.Mem, ErrorFormatter, bool) -> None + # type: (state.Mem, ui.ErrorFormatter, bool) -> None self.mem = mem self.errfmt = errfmt diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index 4e70fa87ad..8a0dee52fa 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -36,7 +36,7 @@ from frontend import args from frontend.parse_lib import ParseContext from core import optview - from core import ui + from display import ui from osh.cmd_eval import CommandEvaluator from osh.cmd_parse import CommandParser diff --git a/builtin/meta_ysh.py b/builtin/meta_ysh.py index 6be0c6061b..8e603a62ee 100644 --- a/builtin/meta_ysh.py +++ b/builtin/meta_ysh.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from core import state - from core import ui + from display import ui class Shvm(vm._Builtin): diff --git a/builtin/misc_osh.py b/builtin/misc_osh.py index 45012db6f4..50996a9053 100644 --- a/builtin/misc_osh.py +++ b/builtin/misc_osh.py @@ -21,7 +21,7 @@ from typing import Dict, TYPE_CHECKING if TYPE_CHECKING: from core.pyutil import _ResourceLoader - from core import ui + from display import ui _ = log diff --git a/builtin/module_ysh.py b/builtin/module_ysh.py index 81d306bef5..bf5a82fc11 100644 --- a/builtin/module_ysh.py +++ b/builtin/module_ysh.py @@ -6,7 +6,7 @@ from core import error from core import state -from core import ui +from display import ui from core import vm from frontend import args from frontend import flag_util diff --git a/builtin/printf_osh.py b/builtin/printf_osh.py index 02e2368de2..20a9ded37e 100644 --- a/builtin/printf_osh.py +++ b/builtin/printf_osh.py @@ -44,7 +44,7 @@ from typing import Dict, List, Optional, TYPE_CHECKING, cast if TYPE_CHECKING: - from core import ui + from display import ui from frontend import parse_lib _ = log diff --git a/builtin/process_osh.py b/builtin/process_osh.py index 2eba7456e8..bc8976c465 100644 --- a/builtin/process_osh.py +++ b/builtin/process_osh.py @@ -34,7 +34,7 @@ if TYPE_CHECKING: from core.process import Waiter, ExternalProgram, FdState from core.state import Mem, SearchPath - from core.ui import ErrorFormatter + from display import ui class Jobs(vm._Builtin): @@ -188,7 +188,7 @@ def Run(self, cmd_val): class Exec(vm._Builtin): def __init__(self, mem, ext_prog, fd_state, search_path, errfmt): - # type: (Mem, ExternalProgram, FdState, SearchPath, ErrorFormatter) -> None + # type: (Mem, ExternalProgram, FdState, SearchPath, ui.ErrorFormatter) -> None self.mem = mem self.ext_prog = ext_prog self.fd_state = fd_state @@ -240,7 +240,7 @@ class Wait(vm._Builtin): """ def __init__(self, waiter, job_list, mem, tracer, errfmt): - # type: (Waiter, process.JobList, Mem, dev.Tracer, ErrorFormatter) -> None + # type: (Waiter, process.JobList, Mem, dev.Tracer, ui.ErrorFormatter) -> None self.waiter = waiter self.job_list = job_list self.mem = mem diff --git a/builtin/pure_osh.py b/builtin/pure_osh.py index 99ce17bc04..bb4442495c 100644 --- a/builtin/pure_osh.py +++ b/builtin/pure_osh.py @@ -16,7 +16,7 @@ from core import error from core.error import e_usage from core import state -from core import ui +from display import ui from core import vm from data_lang import j8_lite from frontend import args diff --git a/builtin/pure_ysh.py b/builtin/pure_ysh.py index b143e85f6a..4a9c483c63 100644 --- a/builtin/pure_ysh.py +++ b/builtin/pure_ysh.py @@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, cast, Any, Dict, List, Tuple if TYPE_CHECKING: - from core import ui + from display import ui from osh.cmd_eval import CommandEvaluator diff --git a/builtin/read_osh.py b/builtin/read_osh.py index e563c7988d..7946909e63 100644 --- a/builtin/read_osh.py +++ b/builtin/read_osh.py @@ -12,7 +12,7 @@ from core import pyos from core import pyutil from core import state -from core import ui +from display import ui from core import vm from frontend import flag_util from frontend import reader diff --git a/builtin/readline_osh.py b/builtin/readline_osh.py index 7490a6341f..df30cfd300 100644 --- a/builtin/readline_osh.py +++ b/builtin/readline_osh.py @@ -17,15 +17,15 @@ if TYPE_CHECKING: from _devbuild.gen.runtime_asdl import cmd_value from frontend.py_readline import Readline - from core.ui import ErrorFormatter from core import shell + from display import ui class Bind(vm._Builtin): """For :, true, false.""" def __init__(self, readline, errfmt): - # type: (Optional[Readline], ErrorFormatter) -> None + # type: (Optional[Readline], ui.ErrorFormatter) -> None self.readline = readline self.errfmt = errfmt @@ -43,7 +43,7 @@ def __init__( self, readline, # type: Optional[Readline] sh_files, # type: shell.ShellFiles - errfmt, # type: ErrorFormatter + errfmt, # type: ui.ErrorFormatter f, # type: mylib.Writer ): # type: (...) -> None diff --git a/builtin/readline_osh_test.py b/builtin/readline_osh_test.py index 6b30b780b6..7b84a36a24 100755 --- a/builtin/readline_osh_test.py +++ b/builtin/readline_osh_test.py @@ -14,7 +14,7 @@ from core import test_lib from core import state from core import alloc -from core import ui +from display import ui from frontend import flag_def # side effect: flags are defined! _ = flag_def diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index 81242ffb77..10e18ba6a0 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -23,7 +23,7 @@ from typing import Dict, List, Optional, TYPE_CHECKING if TYPE_CHECKING: from _devbuild.gen.syntax_asdl import command_t - from core.ui import ErrorFormatter + from display import ui from frontend.parse_lib import ParseContext _ = log @@ -157,7 +157,7 @@ def _GetSignalNumber(sig_spec): class Trap(vm._Builtin): def __init__(self, trap_state, parse_ctx, tracer, errfmt): - # type: (TrapState, ParseContext, dev.Tracer, ErrorFormatter) -> None + # type: (TrapState, ParseContext, dev.Tracer, ui.ErrorFormatter) -> None self.trap_state = trap_state self.parse_ctx = parse_ctx self.arena = parse_ctx.arena diff --git a/core/comp_ui.py b/core/comp_ui.py index 807a725979..5d71ef387d 100644 --- a/core/comp_ui.py +++ b/core/comp_ui.py @@ -1,7 +1,7 @@ """comp_ui.py.""" from __future__ import print_function -from core import ansi +from display import ansi from core import completion from data_lang import pretty import libc diff --git a/core/completion.py b/core/completion.py index 9d30482ca0..d2957bfdbd 100755 --- a/core/completion.py +++ b/core/completion.py @@ -44,7 +44,7 @@ from core import error from core import pyos from core import state -from core import ui +from display import ui from core import util from frontend import consts from frontend import lexer diff --git a/core/dev.py b/core/dev.py index 9633252a78..d4dbf427cc 100644 --- a/core/dev.py +++ b/core/dev.py @@ -14,7 +14,7 @@ from core import optview from core import num from core import state -from core import ui +from display import ui from data_lang import j8 from frontend import location from osh import word_ diff --git a/core/executor.py b/core/executor.py index ed1c4d1d7a..b4dd04543b 100644 --- a/core/executor.py +++ b/core/executor.py @@ -22,7 +22,7 @@ from core import pyos from core import pyutil from core import state -from core import ui +from display import ui from core import vm from frontend import consts from frontend import lexer diff --git a/core/main_loop.py b/core/main_loop.py index 85776733fe..3fb290c666 100644 --- a/core/main_loop.py +++ b/core/main_loop.py @@ -15,7 +15,7 @@ parse_result_e) from core import error from core import process -from core import ui +from display import ui from core import util from frontend import reader from osh import cmd_eval @@ -28,7 +28,6 @@ from typing import cast, Any, List, TYPE_CHECKING if TYPE_CHECKING: from core.comp_ui import _IDisplay - from core.ui import ErrorFormatter from frontend import parse_lib from osh.cmd_parse import CommandParser from osh.cmd_eval import CommandEvaluator @@ -101,7 +100,7 @@ class Headless(object): """Main loop for headless mode.""" def __init__(self, cmd_ev, parse_ctx, errfmt): - # type: (CommandEvaluator, parse_lib.ParseContext, ErrorFormatter) -> None + # type: (CommandEvaluator, parse_lib.ParseContext, ui.ErrorFormatter) -> None self.cmd_ev = cmd_ev self.parse_ctx = parse_ctx self.errfmt = errfmt @@ -197,7 +196,7 @@ def Interactive( display, # type: _IDisplay prompt_plugin, # type: UserPlugin waiter, # type: process.Waiter - errfmt, # type: ErrorFormatter + errfmt, # type: ui.ErrorFormatter ): # type: (...) -> int status = 0 diff --git a/core/process.py b/core/process.py index affa8cc47e..e967543ca6 100644 --- a/core/process.py +++ b/core/process.py @@ -34,7 +34,7 @@ from core import pyutil from core import pyos from core import state -from core import ui +from display import ui from core import util from data_lang import j8_lite from frontend import location @@ -69,7 +69,6 @@ from _devbuild.gen.syntax_asdl import command_t from builtin import trap_osh from core import optview - from core.ui import ErrorFormatter from core.util import _DebugFile from osh.cmd_eval import CommandEvaluator @@ -658,7 +657,7 @@ def __init__( self, hijack_shebang, # type: str fd_state, # type: FdState - errfmt, # type: ErrorFormatter + errfmt, # type: ui.ErrorFormatter debug_f, # type: _DebugFile ): # type: (...) -> None diff --git a/core/process_test.py b/core/process_test.py index 83cde04c75..7c75d591cf 100755 --- a/core/process_test.py +++ b/core/process_test.py @@ -15,7 +15,7 @@ from core import process # module under test from core import pyos from core import test_lib -from core import ui +from display import ui from core import util from mycpp.mylib import log from core import state diff --git a/core/shell.py b/core/shell.py index 0108079866..2c5f0b578b 100644 --- a/core/shell.py +++ b/core/shell.py @@ -24,7 +24,7 @@ from core import process from core import pyutil from core import state -from core import ui +from display import ui from core import util from core import vm diff --git a/core/state.py b/core/state.py index a7bc3e3907..33fd16ac40 100644 --- a/core/state.py +++ b/core/state.py @@ -26,7 +26,7 @@ from core import pyos from core import pyutil from core import optview -from core import ui +from display import ui from core import util from frontend import consts from frontend import location diff --git a/core/test_lib.py b/core/test_lib.py index 6a399d4037..f7a686ec65 100644 --- a/core/test_lib.py +++ b/core/test_lib.py @@ -34,7 +34,7 @@ from core import pyos from core import pyutil from core import state -from core import ui +from display import ui from core import util from core import vm from frontend import lexer diff --git a/core/util.py b/core/util.py index 49e63273bc..67b9e0960c 100644 --- a/core/util.py +++ b/core/util.py @@ -10,7 +10,7 @@ """ from __future__ import print_function -from core import ansi +from display import ansi from core import pyutil from mycpp import mylib diff --git a/data_lang/pretty.py b/data_lang/pretty.py index 33c81ca17b..e58777c197 100644 --- a/data_lang/pretty.py +++ b/data_lang/pretty.py @@ -106,7 +106,7 @@ from _devbuild.gen.value_asdl import value, value_e, value_t, value_str from data_lang import j8 from data_lang import j8_lite -from core import ansi +from display import ansi from frontend import match from mycpp import mops from mycpp.mylib import log, tagswitch, BufWriter, iteritems diff --git a/data_lang/pretty_test.py b/data_lang/pretty_test.py index 3437a3f79c..0d4ece6294 100755 --- a/data_lang/pretty_test.py +++ b/data_lang/pretty_test.py @@ -4,8 +4,8 @@ import os import unittest -from core import ansi -from core import ui +from display import ansi +from display import ui from data_lang import j8 from data_lang import pretty # module under test from mycpp import mylib diff --git a/display/__init__.py b/display/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/core/ansi.py b/display/ansi.py similarity index 100% rename from core/ansi.py rename to display/ansi.py diff --git a/core/ui.py b/display/ui.py similarity index 100% rename from core/ui.py rename to display/ui.py diff --git a/core/ui_test.py b/display/ui_test.py similarity index 93% rename from core/ui_test.py rename to display/ui_test.py index 088bf2b2d0..bfcd632491 100755 --- a/core/ui_test.py +++ b/display/ui_test.py @@ -4,7 +4,7 @@ import unittest from core import test_lib -from core import ui # module under test +from display import ui # module under test class UiTest(unittest.TestCase): diff --git a/metrics/source-code.sh b/metrics/source-code.sh index de527a54f4..223f037d71 100755 --- a/metrics/source-code.sh +++ b/metrics/source-code.sh @@ -35,8 +35,9 @@ osh-files() { # - line_input.c because I didn't write it. It still should be minimized. # - code generators # - test library - - ls bin/oils_for_unix.py {osh,core,frontend}/*.py builtin/*_osh.py \ + # + # note: could move display/ to a separate part + ls bin/oils_for_unix.py {osh,core,display,frontend}/*.py builtin/*_osh.py \ pyext/*.c */*.pyi \ "${OSH_ASDL[@]}" \ | filter-py | grep -E -v 'posixmodule.c$|line_input.c$|_gen.py$|test_lib.py$|os.pyi$' diff --git a/osh/arith_parse_test.py b/osh/arith_parse_test.py index 7d09721a63..de39080783 100755 --- a/osh/arith_parse_test.py +++ b/osh/arith_parse_test.py @@ -14,7 +14,7 @@ from _devbuild.gen.types_asdl import lex_mode_e from core import error from core import test_lib -from core import ui +from display import ui from osh import sh_expr_eval from osh import split from osh import word_eval diff --git a/osh/bool_parse.py b/osh/bool_parse.py index f8c355cc5d..f71071a9e8 100644 --- a/osh/bool_parse.py +++ b/osh/bool_parse.py @@ -34,7 +34,7 @@ from _devbuild.gen.types_asdl import lex_mode_t, lex_mode_e from _devbuild.gen.syntax_asdl import (loc, word_t, word_e, bool_expr, bool_expr_t, Token) -from core import ui +from display import ui from core.error import p_die from frontend import consts from mycpp.mylib import log diff --git a/osh/bool_stat.py b/osh/bool_stat.py index 4300671c81..9a69cb6b8f 100644 --- a/osh/bool_stat.py +++ b/osh/bool_stat.py @@ -11,7 +11,7 @@ from _devbuild.gen.id_kind_asdl import Id, Id_t from _devbuild.gen.syntax_asdl import word_t, loc from core.error import e_die -from core import ui +from display import ui def isatty(fd_str, blame_word): diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index eefd1e1439..84976f7a20 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -74,7 +74,7 @@ from core import pyos # Time(). TODO: rename from core import pyutil from core import state -from core import ui +from display import ui from core import util from core import vm from frontend import consts diff --git a/osh/cmd_parse.py b/osh/cmd_parse.py index 928487842f..a755caf6f0 100644 --- a/osh/cmd_parse.py +++ b/osh/cmd_parse.py @@ -58,7 +58,7 @@ from core import alloc from core import error from core.error import p_die -from core import ui +from display import ui from frontend import consts from frontend import lexer from frontend import location diff --git a/osh/cmd_parse_test.py b/osh/cmd_parse_test.py index 32ae1cb59f..5c125a85cb 100755 --- a/osh/cmd_parse_test.py +++ b/osh/cmd_parse_test.py @@ -11,7 +11,7 @@ from core import error from core import state from core import test_lib -from core import ui +from display import ui from frontend import lexer from osh import word_ diff --git a/osh/prompt.py b/osh/prompt.py index 1e11256e20..25cce8b126 100644 --- a/osh/prompt.py +++ b/osh/prompt.py @@ -15,7 +15,7 @@ from core import error from core import pyos from core import state -from core import ui +from display import ui from frontend import consts from frontend import match from frontend import reader diff --git a/osh/sh_expr_eval.py b/osh/sh_expr_eval.py index a273792d60..2201a7a127 100644 --- a/osh/sh_expr_eval.py +++ b/osh/sh_expr_eval.py @@ -49,7 +49,7 @@ from core.error import e_die, e_die_status, e_strict, e_usage from core import num from core import state -from core import ui +from display import ui from core import util from frontend import consts from frontend import lexer @@ -69,7 +69,6 @@ from typing import Tuple, Optional, cast, TYPE_CHECKING if TYPE_CHECKING: - from core.ui import ErrorFormatter from core import optview _ = log @@ -310,7 +309,7 @@ def __init__( exec_opts, # type: optview.Exec mutable_opts, # type: state.MutableOpts parse_ctx, # type: Optional[parse_lib.ParseContext] - errfmt, # type: ErrorFormatter + errfmt, # type: ui.ErrorFormatter ): # type: (...) -> None self.word_ev = None # type: word_eval.StringWordEvaluator @@ -954,7 +953,7 @@ def __init__( exec_opts, # type: optview.Exec mutable_opts, # type: Optional[state.MutableOpts] parse_ctx, # type: Optional[parse_lib.ParseContext] - errfmt, # type: ErrorFormatter + errfmt, # type: ui.ErrorFormatter always_strict=False # type: bool ): # type: (...) -> None diff --git a/osh/string_ops.py b/osh/string_ops.py index a7b082ca85..aa98551b8b 100644 --- a/osh/string_ops.py +++ b/osh/string_ops.py @@ -15,7 +15,7 @@ from _devbuild.gen.id_kind_asdl import Id from _devbuild.gen.syntax_asdl import loc, Token, suffix_op from core import pyutil -from core import ui +from display import ui from core import error from core.error import e_die, e_strict from mycpp.mylib import log diff --git a/osh/tdop.py b/osh/tdop.py index dd9aee2687..71ceedf739 100644 --- a/osh/tdop.py +++ b/osh/tdop.py @@ -7,7 +7,7 @@ arith_expr_t, word_e, word_t, CompoundWord, Token) from core.error import p_die -from core import ui +from display import ui from mycpp import mylib from mycpp.mylib import tagswitch from osh import word_ diff --git a/osh/word_eval.py b/osh/word_eval.py index ef31c70de0..76a4de7e30 100644 --- a/osh/word_eval.py +++ b/osh/word_eval.py @@ -52,7 +52,7 @@ from core import pyos from core import pyutil from core import state -from core import ui +from display import ui from core import util from data_lang import j8 from data_lang import j8_lite @@ -76,7 +76,6 @@ from _devbuild.gen.option_asdl import builtin_t from core import optview from core.state import Mem - from core.ui import ErrorFormatter from core.vm import _Executor from osh.split import SplitContext from osh import prompt @@ -2350,7 +2349,7 @@ def __init__( mutable_opts, # type: state.MutableOpts tilde_ev, # type: TildeEvaluator splitter, # type: SplitContext - errfmt, # type: ErrorFormatter + errfmt, # type: ui.ErrorFormatter ): # type: (...) -> None AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts, @@ -2408,7 +2407,7 @@ def __init__( mutable_opts, # type: state.MutableOpts tilde_ev, # type: TildeEvaluator splitter, # type: SplitContext - errfmt, # type: ErrorFormatter + errfmt, # type: ui.ErrorFormatter ): # type: (...) -> None AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts, diff --git a/osh/word_parse.py b/osh/word_parse.py index c90fe854de..3d22efc035 100644 --- a/osh/word_parse.py +++ b/osh/word_parse.py @@ -93,7 +93,7 @@ from core.error import p_die from mycpp.mylib import log from core import pyutil -from core import ui +from display import ui from frontend import consts from frontend import lexer from frontend import reader diff --git a/pea/oils-typecheck.txt b/pea/oils-typecheck.txt index 4432ccaa65..c523d8d057 100644 --- a/pea/oils-typecheck.txt +++ b/pea/oils-typecheck.txt @@ -42,7 +42,6 @@ builtin/read_osh.py builtin/readline_osh.py builtin/trap_osh.py core/alloc.py -core/ansi.py core/comp_ui.py core/completion.py core/dev.py @@ -56,13 +55,14 @@ core/pyos.py core/pyutil.py core/shell.py core/state.py -core/ui.py core/util.py core/vm.py data_lang/j8.py data_lang/j8_lite.py data_lang/pretty.py data_lang/pyj8.py +display/ansi.py +display/ui.py frontend/args.py frontend/builtin_def.py frontend/consts.py diff --git a/prebuilt/dynamic-deps/filter-translate.txt b/prebuilt/dynamic-deps/filter-translate.txt index ca68344572..648dcfff25 100644 --- a/prebuilt/dynamic-deps/filter-translate.txt +++ b/prebuilt/dynamic-deps/filter-translate.txt @@ -15,4 +15,3 @@ pgen2/pnode.py pgen2/token.py pylib/path_stat.py osh/bool_stat.py -tea/ diff --git a/spec/stateful/harness.py b/spec/stateful/harness.py index 4d14edd5d5..9eae6d0de9 100644 --- a/spec/stateful/harness.py +++ b/spec/stateful/harness.py @@ -14,7 +14,7 @@ import signal import sys -from core import ansi +from display import ansi from test import spec_lib # Using this for a common interface log = spec_lib.log diff --git a/test/py2_lint.py b/test/py2_lint.py index d63f27cd0c..7b2611ef1c 100755 --- a/test/py2_lint.py +++ b/test/py2_lint.py @@ -16,7 +16,7 @@ from pyflakes import api from pyflakes import reporter -from core import ansi +from display import ansi # Our config for flake8 # local fatal_errors='E901,E999,F821,F822,F823,F401' diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 58832f6ab8..481251e0ac 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -51,7 +51,7 @@ from core import num from core import pyutil from core import state -from core import ui +from display import ui from core import vm from data_lang import j8 from frontend import lexer diff --git a/ysh/expr_parse.py b/ysh/expr_parse.py index 1b6d232fec..3043caae99 100644 --- a/ysh/expr_parse.py +++ b/ysh/expr_parse.py @@ -7,7 +7,7 @@ from _devbuild.gen.id_kind_asdl import Id, Kind, Id_str from _devbuild.gen.types_asdl import lex_mode_e -from core import ui +from display import ui from core.error import p_die from frontend import consts from frontend import lexer diff --git a/ysh/val_ops.py b/ysh/val_ops.py index ea92d0bdcf..9da110afce 100644 --- a/ysh/val_ops.py +++ b/ysh/val_ops.py @@ -7,7 +7,7 @@ eggex_ops_t, regex_match, RegexMatch) from core import error from core.error import e_die -from core import ui +from display import ui from mycpp import mops from mycpp import mylib from mycpp.mylib import tagswitch, log From 3026525a0338e33776b17dd10fd798795dc0ff1b Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 13:06:22 -0400 Subject: [PATCH 061/506] [prebuilt] Regenerate files --- mycpp/examples/parse.translate.txt | 2 +- mycpp/examples/parse.typecheck.txt | 3 +- prebuilt/asdl/runtime.mycpp.cc | 37 ++-- prebuilt/asdl/runtime.mycpp.h | 14 +- prebuilt/core/error.mycpp.cc | 28 ++- prebuilt/core/error.mycpp.h | 49 +++-- prebuilt/frontend/args.mycpp.cc | 286 +++++++++++++++-------------- prebuilt/frontend/args.mycpp.h | 56 +++--- prebuilt/translate.sh | 2 +- test/ysh-runtime-errors.sh | 4 + 10 files changed, 249 insertions(+), 232 deletions(-) diff --git a/mycpp/examples/parse.translate.txt b/mycpp/examples/parse.translate.txt index 9f010863db..4d762274d3 100644 --- a/mycpp/examples/parse.translate.txt +++ b/mycpp/examples/parse.translate.txt @@ -1,6 +1,6 @@ asdl/format.py asdl/runtime.py -core/ansi.py data_lang/j8_lite.py +display/ansi.py mycpp/examples/parse.py pylib/cgi.py diff --git a/mycpp/examples/parse.typecheck.txt b/mycpp/examples/parse.typecheck.txt index bbcd4086e0..9c81a7aea1 100644 --- a/mycpp/examples/parse.typecheck.txt +++ b/mycpp/examples/parse.typecheck.txt @@ -3,7 +3,8 @@ _devbuild/gen/hnode_asdl.py asdl/format.py asdl/pybase.py asdl/runtime.py -core/ansi.py data_lang/j8_lite.py +display/ansi.py mycpp/examples/parse.py +mycpp/mops.py pylib/cgi.py diff --git a/prebuilt/asdl/runtime.mycpp.cc b/prebuilt/asdl/runtime.mycpp.cc index e12ec57a6b..8f2be2a3d6 100644 --- a/prebuilt/asdl/runtime.mycpp.cc +++ b/prebuilt/asdl/runtime.mycpp.cc @@ -56,12 +56,13 @@ GLOBAL_STR(str47, "\u001b[33m"); GLOBAL_STR(str48, "\u001b[34m"); GLOBAL_STR(str49, "\u001b[35m"); GLOBAL_STR(str50, "\u001b[36m"); -GLOBAL_STR(str51, "&"); -GLOBAL_STR(str52, "&"); -GLOBAL_STR(str53, "<"); -GLOBAL_STR(str54, "<"); -GLOBAL_STR(str55, ">"); -GLOBAL_STR(str56, ">"); +GLOBAL_STR(str51, "\u001b[37m"); +GLOBAL_STR(str52, "&"); +GLOBAL_STR(str53, "&"); +GLOBAL_STR(str54, "<"); +GLOBAL_STR(str55, "<"); +GLOBAL_STR(str56, ">"); +GLOBAL_STR(str57, ">"); namespace ansi { // forward declare @@ -90,7 +91,7 @@ extern BigStr* YELLOW; extern BigStr* BLUE; extern BigStr* MAGENTA; extern BigStr* CYAN; - +extern BigStr* WHITE; } // declare namespace ansi @@ -98,17 +99,16 @@ namespace cgi { // declare BigStr* escape(BigStr* s); - } // declare namespace cgi namespace j8_lite { // declare BigStr* EncodeString(BigStr* s, bool unquoted_ok = false); +BigStr* YshEncodeString(BigStr* s); BigStr* MaybeShellEncode(BigStr* s); BigStr* ShellEncode(BigStr* s); BigStr* YshEncode(BigStr* s, bool unquoted_ok = false); - } // declare namespace j8_lite namespace runtime { // define @@ -220,7 +220,7 @@ Tuple2 ColorOutput::GetRaw() { return Tuple2(f->getvalue(), this->num_chars); } -TextOutput::TextOutput(mylib::Writer* f) : ColorOutput(f) { +TextOutput::TextOutput(mylib::Writer* f) : ::format::ColorOutput(f) { } format::TextOutput* TextOutput::NewTempBuffer() { @@ -235,7 +235,7 @@ void TextOutput::PopColor() { ; // pass } -HtmlOutput::HtmlOutput(mylib::Writer* f) : ColorOutput(f) { +HtmlOutput::HtmlOutput(mylib::Writer* f) : ::format::ColorOutput(f) { } format::HtmlOutput* HtmlOutput::NewTempBuffer() { @@ -294,7 +294,7 @@ void HtmlOutput::write(BigStr* s) { this->num_chars += len(s); } -AnsiOutput::AnsiOutput(mylib::Writer* f) : ColorOutput(f) { +AnsiOutput::AnsiOutput(mylib::Writer* f) : ::format::ColorOutput(f) { } format::AnsiOutput* AnsiOutput::NewTempBuffer() { @@ -718,6 +718,7 @@ BigStr* YELLOW = str47; BigStr* BLUE = str48; BigStr* MAGENTA = str49; BigStr* CYAN = str50; +BigStr* WHITE = str51; } // define namespace ansi @@ -727,9 +728,9 @@ namespace cgi { // define BigStr* escape(BigStr* s) { StackRoot _root0(&s); - s = s->replace(str51, str52); - s = s->replace(str53, str54); - s = s->replace(str55, str56); + s = s->replace(str52, str53); + s = s->replace(str54, str55); + s = s->replace(str56, str57); return s; } @@ -747,6 +748,12 @@ BigStr* EncodeString(BigStr* s, bool unquoted_ok) { return fastfunc::J8EncodeString(s, 1); } +BigStr* YshEncodeString(BigStr* s) { + StackRoot _root0(&s); + + return fastfunc::ShellEncodeString(s, 1); +} + BigStr* MaybeShellEncode(BigStr* s) { StackRoot _root0(&s); diff --git a/prebuilt/asdl/runtime.mycpp.h b/prebuilt/asdl/runtime.mycpp.h index df28f11197..a417a8e5f1 100644 --- a/prebuilt/asdl/runtime.mycpp.h +++ b/prebuilt/asdl/runtime.mycpp.h @@ -45,7 +45,6 @@ class TraversalState { extern BigStr* TRUE_STR; extern BigStr* FALSE_STR; - } // declare namespace runtime namespace format { // declare @@ -78,7 +77,7 @@ class ColorOutput { DISALLOW_COPY_AND_ASSIGN(ColorOutput) }; -class TextOutput : public ColorOutput { +class TextOutput : public ::format::ColorOutput { public: TextOutput(mylib::Writer* f); virtual format::TextOutput* NewTempBuffer(); @@ -86,7 +85,7 @@ class TextOutput : public ColorOutput { virtual void PopColor(); static constexpr uint32_t field_mask() { - return ColorOutput::field_mask(); + return ::format::ColorOutput::field_mask(); } static constexpr ObjHeader obj_header() { @@ -96,7 +95,7 @@ class TextOutput : public ColorOutput { DISALLOW_COPY_AND_ASSIGN(TextOutput) }; -class HtmlOutput : public ColorOutput { +class HtmlOutput : public ::format::ColorOutput { public: HtmlOutput(mylib::Writer* f); virtual format::HtmlOutput* NewTempBuffer(); @@ -107,7 +106,7 @@ class HtmlOutput : public ColorOutput { virtual void write(BigStr* s); static constexpr uint32_t field_mask() { - return ColorOutput::field_mask(); + return ::format::ColorOutput::field_mask(); } static constexpr ObjHeader obj_header() { @@ -117,7 +116,7 @@ class HtmlOutput : public ColorOutput { DISALLOW_COPY_AND_ASSIGN(HtmlOutput) }; -class AnsiOutput : public ColorOutput { +class AnsiOutput : public ::format::ColorOutput { public: AnsiOutput(mylib::Writer* f); virtual format::AnsiOutput* NewTempBuffer(); @@ -125,7 +124,7 @@ class AnsiOutput : public ColorOutput { virtual void PopColor(); static constexpr uint32_t field_mask() { - return ColorOutput::field_mask(); + return ::format::ColorOutput::field_mask(); } static constexpr ObjHeader obj_header() { @@ -156,7 +155,6 @@ bool _TrySingleLineObj(hnode::Record* node, format::ColorOutput* f, int max_char bool _TrySingleLine(hnode_asdl::hnode_t* node, format::ColorOutput* f, int max_chars); void PrintTree(hnode_asdl::hnode_t* node, format::ColorOutput* f); - } // declare namespace format #endif // ASDL_RUNTIME_MYCPP_H diff --git a/prebuilt/core/error.mycpp.cc b/prebuilt/core/error.mycpp.cc index b02a8dee5d..c08fe210cf 100644 --- a/prebuilt/core/error.mycpp.cc +++ b/prebuilt/core/error.mycpp.cc @@ -11,7 +11,7 @@ GLOBAL_STR(str2, "_"); GLOBAL_STR(str3, "T"); GLOBAL_STR(str4, "F"); GLOBAL_STR(str5, "<%s %r>"); -GLOBAL_STR(str6, "status"); +GLOBAL_STR(str6, "code"); GLOBAL_STR(str7, "message"); GLOBAL_STR(str8, "%s, got %s"); GLOBAL_STR(str9, " (line %d, offset %d-%d: %r)"); @@ -49,7 +49,6 @@ class TraversalState { extern BigStr* TRUE_STR; extern BigStr* FALSE_STR; - } // declare namespace runtime namespace num { // declare @@ -62,7 +61,6 @@ int IntDivide2(int x, int y); mops::BigInt IntRemainder(mops::BigInt x, mops::BigInt y); int IntRemainder2(int x, int y); - } // declare namespace num namespace runtime { // define @@ -131,19 +129,19 @@ BigStr* _ErrorWithLocation::UserErrorString() { return this->msg; } -Usage::Usage(BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +Usage::Usage(BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { } -Parse::Parse(BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +Parse::Parse(BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { } -FailGlob::FailGlob(BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +FailGlob::FailGlob(BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { } -RedirectEval::RedirectEval(BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +RedirectEval::RedirectEval(BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { } -FatalRuntime::FatalRuntime(int exit_status, BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +FatalRuntime::FatalRuntime(int exit_status, BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { this->exit_status = exit_status; } @@ -151,17 +149,17 @@ int FatalRuntime::ExitStatus() { return this->exit_status; } -Strict::Strict(BigStr* msg, syntax_asdl::loc_t* location) : FatalRuntime(1, msg, location) { +Strict::Strict(BigStr* msg, syntax_asdl::loc_t* location) : ::error::FatalRuntime(1, msg, location) { } -ErrExit::ErrExit(int exit_status, BigStr* msg, syntax_asdl::loc_t* location, bool show_code) : FatalRuntime(exit_status, msg, location) { +ErrExit::ErrExit(int exit_status, BigStr* msg, syntax_asdl::loc_t* location, bool show_code) : ::error::FatalRuntime(exit_status, msg, location) { this->show_code = show_code; } -Expr::Expr(BigStr* msg, syntax_asdl::loc_t* location) : FatalRuntime(3, msg, location) { +Expr::Expr(BigStr* msg, syntax_asdl::loc_t* location) : ::error::FatalRuntime(3, msg, location) { } -Structured::Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties) : FatalRuntime(status, msg, location) { +Structured::Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties) : ::error::FatalRuntime(status, msg, location) { this->properties = properties; } @@ -174,13 +172,13 @@ value::Dict* Structured::ToDict() { return Alloc(this->properties); } -AssertionErr::AssertionErr(BigStr* msg, syntax_asdl::loc_t* location) : Expr(msg, location) { +AssertionErr::AssertionErr(BigStr* msg, syntax_asdl::loc_t* location) : ::error::Expr(msg, location) { } -TypeErrVerbose::TypeErrVerbose(BigStr* msg, syntax_asdl::loc_t* location) : Expr(msg, location) { +TypeErrVerbose::TypeErrVerbose(BigStr* msg, syntax_asdl::loc_t* location) : ::error::Expr(msg, location) { } -TypeErr::TypeErr(value_asdl::value_t* actual_val, BigStr* msg, syntax_asdl::loc_t* location) : TypeErrVerbose(StrFormat("%s, got %s", msg, _ValType(actual_val)), location) { +TypeErr::TypeErr(value_asdl::value_t* actual_val, BigStr* msg, syntax_asdl::loc_t* location) : ::error::TypeErrVerbose(StrFormat("%s, got %s", msg, _ValType(actual_val)), location) { } Runtime::Runtime(BigStr* msg) { diff --git a/prebuilt/core/error.mycpp.h b/prebuilt/core/error.mycpp.h index 017f8d38f1..fb627067a6 100644 --- a/prebuilt/core/error.mycpp.h +++ b/prebuilt/core/error.mycpp.h @@ -58,12 +58,12 @@ class _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(_ErrorWithLocation) }; -class Usage : public _ErrorWithLocation { +class Usage : public ::error::_ErrorWithLocation { public: Usage(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -73,12 +73,12 @@ class Usage : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(Usage) }; -class Parse : public _ErrorWithLocation { +class Parse : public ::error::_ErrorWithLocation { public: Parse(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -88,12 +88,12 @@ class Parse : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(Parse) }; -class FailGlob : public _ErrorWithLocation { +class FailGlob : public ::error::_ErrorWithLocation { public: FailGlob(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -103,12 +103,12 @@ class FailGlob : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(FailGlob) }; -class RedirectEval : public _ErrorWithLocation { +class RedirectEval : public ::error::_ErrorWithLocation { public: RedirectEval(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -118,7 +118,7 @@ class RedirectEval : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(RedirectEval) }; -class FatalRuntime : public _ErrorWithLocation { +class FatalRuntime : public ::error::_ErrorWithLocation { public: FatalRuntime(int exit_status, BigStr* msg, syntax_asdl::loc_t* location); int ExitStatus(); @@ -126,7 +126,7 @@ class FatalRuntime : public _ErrorWithLocation { int exit_status; static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -136,12 +136,12 @@ class FatalRuntime : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(FatalRuntime) }; -class Strict : public FatalRuntime { +class Strict : public ::error::FatalRuntime { public: Strict(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return FatalRuntime::field_mask(); + return ::error::FatalRuntime::field_mask(); } static constexpr ObjHeader obj_header() { @@ -151,14 +151,14 @@ class Strict : public FatalRuntime { DISALLOW_COPY_AND_ASSIGN(Strict) }; -class ErrExit : public FatalRuntime { +class ErrExit : public ::error::FatalRuntime { public: ErrExit(int exit_status, BigStr* msg, syntax_asdl::loc_t* location, bool show_code = false); bool show_code; static constexpr uint32_t field_mask() { - return FatalRuntime::field_mask(); + return ::error::FatalRuntime::field_mask(); } static constexpr ObjHeader obj_header() { @@ -168,12 +168,12 @@ class ErrExit : public FatalRuntime { DISALLOW_COPY_AND_ASSIGN(ErrExit) }; -class Expr : public FatalRuntime { +class Expr : public ::error::FatalRuntime { public: Expr(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return FatalRuntime::field_mask(); + return ::error::FatalRuntime::field_mask(); } static constexpr ObjHeader obj_header() { @@ -183,7 +183,7 @@ class Expr : public FatalRuntime { DISALLOW_COPY_AND_ASSIGN(Expr) }; -class Structured : public FatalRuntime { +class Structured : public ::error::FatalRuntime { public: Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties = nullptr); value::Dict* ToDict(); @@ -191,7 +191,7 @@ class Structured : public FatalRuntime { Dict* properties; static constexpr uint32_t field_mask() { - return FatalRuntime::field_mask() + return ::error::FatalRuntime::field_mask() | maskbit(offsetof(Structured, properties)); } @@ -202,12 +202,12 @@ class Structured : public FatalRuntime { DISALLOW_COPY_AND_ASSIGN(Structured) }; -class AssertionErr : public Expr { +class AssertionErr : public ::error::Expr { public: AssertionErr(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return Expr::field_mask(); + return ::error::Expr::field_mask(); } static constexpr ObjHeader obj_header() { @@ -217,12 +217,12 @@ class AssertionErr : public Expr { DISALLOW_COPY_AND_ASSIGN(AssertionErr) }; -class TypeErrVerbose : public Expr { +class TypeErrVerbose : public ::error::Expr { public: TypeErrVerbose(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return Expr::field_mask(); + return ::error::Expr::field_mask(); } static constexpr ObjHeader obj_header() { @@ -232,12 +232,12 @@ class TypeErrVerbose : public Expr { DISALLOW_COPY_AND_ASSIGN(TypeErrVerbose) }; -class TypeErr : public TypeErrVerbose { +class TypeErr : public ::error::TypeErrVerbose { public: TypeErr(value_asdl::value_t* actual_val, BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return TypeErrVerbose::field_mask(); + return ::error::TypeErrVerbose::field_mask(); } static constexpr ObjHeader obj_header() { @@ -297,7 +297,6 @@ class Encode { [[noreturn]] void e_die(BigStr* msg, syntax_asdl::loc_t* location = nullptr); [[noreturn]] void e_die_status(int status, BigStr* msg, syntax_asdl::loc_t* location = nullptr); - } // declare namespace error #endif // CORE_ERROR_MYCPP_H diff --git a/prebuilt/frontend/args.mycpp.cc b/prebuilt/frontend/args.mycpp.cc index a5937037c4..01dda3d948 100644 --- a/prebuilt/frontend/args.mycpp.cc +++ b/prebuilt/frontend/args.mycpp.cc @@ -56,70 +56,72 @@ GLOBAL_STR(str47, "\u001b[33m"); GLOBAL_STR(str48, "\u001b[34m"); GLOBAL_STR(str49, "\u001b[35m"); GLOBAL_STR(str50, "\u001b[36m"); -GLOBAL_STR(str51, "&"); -GLOBAL_STR(str52, "&"); -GLOBAL_STR(str53, "<"); -GLOBAL_STR(str54, "<"); -GLOBAL_STR(str55, ">"); -GLOBAL_STR(str56, ">"); -GLOBAL_STR(str57, "<%s %r>"); -GLOBAL_STR(str58, "status"); -GLOBAL_STR(str59, "message"); -GLOBAL_STR(str60, "%s, got %s"); -GLOBAL_STR(str61, " (line %d, offset %d-%d: %r)"); -GLOBAL_STR(str62, "-"); -GLOBAL_STR(str63, "_"); -GLOBAL_STR(str64, "<_Attributes %s>"); -GLOBAL_STR(str65, ""); -GLOBAL_STR(str66, "expected argument to %r"); -GLOBAL_STR(str67, "-"); -GLOBAL_STR(str68, "expected integer after %s, got %r"); +GLOBAL_STR(str51, "\u001b[37m"); +GLOBAL_STR(str52, "&"); +GLOBAL_STR(str53, "&"); +GLOBAL_STR(str54, "<"); +GLOBAL_STR(str55, "<"); +GLOBAL_STR(str56, ">"); +GLOBAL_STR(str57, ">"); +GLOBAL_STR(str58, "<%s %r>"); +GLOBAL_STR(str59, "code"); +GLOBAL_STR(str60, "message"); +GLOBAL_STR(str61, "%s, got %s"); +GLOBAL_STR(str62, " (line %d, offset %d-%d: %r)"); +GLOBAL_STR(str63, "-"); +GLOBAL_STR(str64, "_"); +GLOBAL_STR(str65, "<_Attributes %s>"); +GLOBAL_STR(str66, ""); +GLOBAL_STR(str67, "got too many arguments"); +GLOBAL_STR(str68, "expected argument to %r"); GLOBAL_STR(str69, "-"); -GLOBAL_STR(str70, "got invalid integer for %s: %s"); +GLOBAL_STR(str70, "expected integer after %s, got %r"); GLOBAL_STR(str71, "-"); -GLOBAL_STR(str72, "expected number after %r, got %r"); +GLOBAL_STR(str72, "got invalid integer for %s: %s"); GLOBAL_STR(str73, "-"); -GLOBAL_STR(str74, "got invalid float for %s: %s"); +GLOBAL_STR(str74, "expected number after %r, got %r"); GLOBAL_STR(str75, "-"); -GLOBAL_STR(str76, "got invalid argument %r to %r, expected one of: %s"); +GLOBAL_STR(str76, "got invalid float for %s: %s"); GLOBAL_STR(str77, "-"); -GLOBAL_STR(str78, "|"); -GLOBAL_STR(str79, "0"); -GLOBAL_STR(str80, "F"); -GLOBAL_STR(str81, "false"); -GLOBAL_STR(str82, "False"); -GLOBAL_STR(str83, "1"); -GLOBAL_STR(str84, "T"); -GLOBAL_STR(str85, "true"); -GLOBAL_STR(str86, "Talse"); -GLOBAL_STR(str87, "got invalid argument to boolean flag: %r"); -GLOBAL_STR(str88, "-"); -GLOBAL_STR(str89, "-"); -GLOBAL_STR(str90, "Invalid option %r"); -GLOBAL_STR(str91, "Expected argument for action"); -GLOBAL_STR(str92, "Invalid action name %r"); -GLOBAL_STR(str93, "--"); -GLOBAL_STR(str94, "--"); -GLOBAL_STR(str95, "="); -GLOBAL_STR(str96, "got invalid flag %r"); -GLOBAL_STR(str97, "-"); -GLOBAL_STR(str98, "0"); -GLOBAL_STR(str99, "Z"); -GLOBAL_STR(str100, "-"); -GLOBAL_STR(str101, "doesn't accept flag %s"); +GLOBAL_STR(str78, "got invalid argument %r to %r, expected one of: %s"); +GLOBAL_STR(str79, "-"); +GLOBAL_STR(str80, "|"); +GLOBAL_STR(str81, "0"); +GLOBAL_STR(str82, "F"); +GLOBAL_STR(str83, "false"); +GLOBAL_STR(str84, "False"); +GLOBAL_STR(str85, "1"); +GLOBAL_STR(str86, "T"); +GLOBAL_STR(str87, "true"); +GLOBAL_STR(str88, "Talse"); +GLOBAL_STR(str89, "got invalid argument to boolean flag: %r"); +GLOBAL_STR(str90, "-"); +GLOBAL_STR(str91, "-"); +GLOBAL_STR(str92, "Invalid option %r"); +GLOBAL_STR(str93, "Expected argument for action"); +GLOBAL_STR(str94, "Invalid action name %r"); +GLOBAL_STR(str95, "--"); +GLOBAL_STR(str96, "--"); +GLOBAL_STR(str97, "="); +GLOBAL_STR(str98, "got invalid flag %r"); +GLOBAL_STR(str99, "-"); +GLOBAL_STR(str100, "0"); +GLOBAL_STR(str101, "Z"); GLOBAL_STR(str102, "-"); -GLOBAL_STR(str103, "+"); -GLOBAL_STR(str104, "+"); -GLOBAL_STR(str105, "doesn't accept option %s"); +GLOBAL_STR(str103, "doesn't accept flag %s"); +GLOBAL_STR(str104, "-"); +GLOBAL_STR(str105, "+"); GLOBAL_STR(str106, "+"); -GLOBAL_STR(str107, "-"); -GLOBAL_STR(str108, "--"); -GLOBAL_STR(str109, "--"); -GLOBAL_STR(str110, "got invalid flag %r"); -GLOBAL_STR(str111, "-"); -GLOBAL_STR(str112, "+"); -GLOBAL_STR(str113, "got invalid flag %r"); -GLOBAL_STR(str114, "-"); +GLOBAL_STR(str107, "doesn't accept option %s"); +GLOBAL_STR(str108, "+"); +GLOBAL_STR(str109, "-"); +GLOBAL_STR(str110, "--"); +GLOBAL_STR(str111, "--"); +GLOBAL_STR(str112, "got invalid flag %r"); +GLOBAL_STR(str113, "-"); +GLOBAL_STR(str114, "+"); +GLOBAL_STR(str115, "got invalid flag %r"); +GLOBAL_STR(str116, "-"); namespace ansi { // forward declare @@ -174,7 +176,7 @@ extern BigStr* YELLOW; extern BigStr* BLUE; extern BigStr* MAGENTA; extern BigStr* CYAN; - +extern BigStr* WHITE; } // declare namespace ansi @@ -182,17 +184,16 @@ namespace cgi { // declare BigStr* escape(BigStr* s); - } // declare namespace cgi namespace j8_lite { // declare BigStr* EncodeString(BigStr* s, bool unquoted_ok = false); +BigStr* YshEncodeString(BigStr* s); BigStr* MaybeShellEncode(BigStr* s); BigStr* ShellEncode(BigStr* s); BigStr* YshEncode(BigStr* s, bool unquoted_ok = false); - } // declare namespace j8_lite namespace error { // declare @@ -219,12 +220,12 @@ class _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(_ErrorWithLocation) }; -class Usage : public _ErrorWithLocation { +class Usage : public ::error::_ErrorWithLocation { public: Usage(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -234,12 +235,12 @@ class Usage : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(Usage) }; -class Parse : public _ErrorWithLocation { +class Parse : public ::error::_ErrorWithLocation { public: Parse(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -249,12 +250,12 @@ class Parse : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(Parse) }; -class FailGlob : public _ErrorWithLocation { +class FailGlob : public ::error::_ErrorWithLocation { public: FailGlob(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -264,12 +265,12 @@ class FailGlob : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(FailGlob) }; -class RedirectEval : public _ErrorWithLocation { +class RedirectEval : public ::error::_ErrorWithLocation { public: RedirectEval(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -279,7 +280,7 @@ class RedirectEval : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(RedirectEval) }; -class FatalRuntime : public _ErrorWithLocation { +class FatalRuntime : public ::error::_ErrorWithLocation { public: FatalRuntime(int exit_status, BigStr* msg, syntax_asdl::loc_t* location); int ExitStatus(); @@ -287,7 +288,7 @@ class FatalRuntime : public _ErrorWithLocation { int exit_status; static constexpr uint32_t field_mask() { - return _ErrorWithLocation::field_mask(); + return ::error::_ErrorWithLocation::field_mask(); } static constexpr ObjHeader obj_header() { @@ -297,12 +298,12 @@ class FatalRuntime : public _ErrorWithLocation { DISALLOW_COPY_AND_ASSIGN(FatalRuntime) }; -class Strict : public FatalRuntime { +class Strict : public ::error::FatalRuntime { public: Strict(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return FatalRuntime::field_mask(); + return ::error::FatalRuntime::field_mask(); } static constexpr ObjHeader obj_header() { @@ -312,14 +313,14 @@ class Strict : public FatalRuntime { DISALLOW_COPY_AND_ASSIGN(Strict) }; -class ErrExit : public FatalRuntime { +class ErrExit : public ::error::FatalRuntime { public: ErrExit(int exit_status, BigStr* msg, syntax_asdl::loc_t* location, bool show_code = false); bool show_code; static constexpr uint32_t field_mask() { - return FatalRuntime::field_mask(); + return ::error::FatalRuntime::field_mask(); } static constexpr ObjHeader obj_header() { @@ -329,12 +330,12 @@ class ErrExit : public FatalRuntime { DISALLOW_COPY_AND_ASSIGN(ErrExit) }; -class Expr : public FatalRuntime { +class Expr : public ::error::FatalRuntime { public: Expr(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return FatalRuntime::field_mask(); + return ::error::FatalRuntime::field_mask(); } static constexpr ObjHeader obj_header() { @@ -344,7 +345,7 @@ class Expr : public FatalRuntime { DISALLOW_COPY_AND_ASSIGN(Expr) }; -class Structured : public FatalRuntime { +class Structured : public ::error::FatalRuntime { public: Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties = nullptr); value::Dict* ToDict(); @@ -352,7 +353,7 @@ class Structured : public FatalRuntime { Dict* properties; static constexpr uint32_t field_mask() { - return FatalRuntime::field_mask() + return ::error::FatalRuntime::field_mask() | maskbit(offsetof(Structured, properties)); } @@ -363,12 +364,12 @@ class Structured : public FatalRuntime { DISALLOW_COPY_AND_ASSIGN(Structured) }; -class AssertionErr : public Expr { +class AssertionErr : public ::error::Expr { public: AssertionErr(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return Expr::field_mask(); + return ::error::Expr::field_mask(); } static constexpr ObjHeader obj_header() { @@ -378,12 +379,12 @@ class AssertionErr : public Expr { DISALLOW_COPY_AND_ASSIGN(AssertionErr) }; -class TypeErrVerbose : public Expr { +class TypeErrVerbose : public ::error::Expr { public: TypeErrVerbose(BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return Expr::field_mask(); + return ::error::Expr::field_mask(); } static constexpr ObjHeader obj_header() { @@ -393,12 +394,12 @@ class TypeErrVerbose : public Expr { DISALLOW_COPY_AND_ASSIGN(TypeErrVerbose) }; -class TypeErr : public TypeErrVerbose { +class TypeErr : public ::error::TypeErrVerbose { public: TypeErr(value_asdl::value_t* actual_val, BigStr* msg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return TypeErrVerbose::field_mask(); + return ::error::TypeErrVerbose::field_mask(); } static constexpr ObjHeader obj_header() { @@ -458,7 +459,6 @@ class Encode { [[noreturn]] void e_die(BigStr* msg, syntax_asdl::loc_t* location = nullptr); [[noreturn]] void e_die_status(int status, BigStr* msg, syntax_asdl::loc_t* location = nullptr); - } // declare namespace error namespace num { // declare @@ -471,7 +471,6 @@ int IntDivide2(int x, int y); mops::BigInt IntRemainder(mops::BigInt x, mops::BigInt y); int IntRemainder2(int x, int y); - } // declare namespace num namespace runtime { // define @@ -583,7 +582,7 @@ Tuple2 ColorOutput::GetRaw() { return Tuple2(f->getvalue(), this->num_chars); } -TextOutput::TextOutput(mylib::Writer* f) : ColorOutput(f) { +TextOutput::TextOutput(mylib::Writer* f) : ::format::ColorOutput(f) { } format::TextOutput* TextOutput::NewTempBuffer() { @@ -598,7 +597,7 @@ void TextOutput::PopColor() { ; // pass } -HtmlOutput::HtmlOutput(mylib::Writer* f) : ColorOutput(f) { +HtmlOutput::HtmlOutput(mylib::Writer* f) : ::format::ColorOutput(f) { } format::HtmlOutput* HtmlOutput::NewTempBuffer() { @@ -657,7 +656,7 @@ void HtmlOutput::write(BigStr* s) { this->num_chars += len(s); } -AnsiOutput::AnsiOutput(mylib::Writer* f) : ColorOutput(f) { +AnsiOutput::AnsiOutput(mylib::Writer* f) : ::format::ColorOutput(f) { } format::AnsiOutput* AnsiOutput::NewTempBuffer() { @@ -1081,6 +1080,7 @@ BigStr* YELLOW = str47; BigStr* BLUE = str48; BigStr* MAGENTA = str49; BigStr* CYAN = str50; +BigStr* WHITE = str51; } // define namespace ansi @@ -1090,9 +1090,9 @@ namespace cgi { // define BigStr* escape(BigStr* s) { StackRoot _root0(&s); - s = s->replace(str51, str52); - s = s->replace(str53, str54); - s = s->replace(str55, str56); + s = s->replace(str52, str53); + s = s->replace(str54, str55); + s = s->replace(str56, str57); return s; } @@ -1110,6 +1110,12 @@ BigStr* EncodeString(BigStr* s, bool unquoted_ok) { return fastfunc::J8EncodeString(s, 1); } +BigStr* YshEncodeString(BigStr* s) { + StackRoot _root0(&s); + + return fastfunc::ShellEncodeString(s, 1); +} + BigStr* MaybeShellEncode(BigStr* s) { StackRoot _root0(&s); @@ -1169,19 +1175,19 @@ BigStr* _ErrorWithLocation::UserErrorString() { return this->msg; } -Usage::Usage(BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +Usage::Usage(BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { } -Parse::Parse(BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +Parse::Parse(BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { } -FailGlob::FailGlob(BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +FailGlob::FailGlob(BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { } -RedirectEval::RedirectEval(BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +RedirectEval::RedirectEval(BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { } -FatalRuntime::FatalRuntime(int exit_status, BigStr* msg, syntax_asdl::loc_t* location) : _ErrorWithLocation(msg, location) { +FatalRuntime::FatalRuntime(int exit_status, BigStr* msg, syntax_asdl::loc_t* location) : ::error::_ErrorWithLocation(msg, location) { this->exit_status = exit_status; } @@ -1189,17 +1195,17 @@ int FatalRuntime::ExitStatus() { return this->exit_status; } -Strict::Strict(BigStr* msg, syntax_asdl::loc_t* location) : FatalRuntime(1, msg, location) { +Strict::Strict(BigStr* msg, syntax_asdl::loc_t* location) : ::error::FatalRuntime(1, msg, location) { } -ErrExit::ErrExit(int exit_status, BigStr* msg, syntax_asdl::loc_t* location, bool show_code) : FatalRuntime(exit_status, msg, location) { +ErrExit::ErrExit(int exit_status, BigStr* msg, syntax_asdl::loc_t* location, bool show_code) : ::error::FatalRuntime(exit_status, msg, location) { this->show_code = show_code; } -Expr::Expr(BigStr* msg, syntax_asdl::loc_t* location) : FatalRuntime(3, msg, location) { +Expr::Expr(BigStr* msg, syntax_asdl::loc_t* location) : ::error::FatalRuntime(3, msg, location) { } -Structured::Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties) : FatalRuntime(status, msg, location) { +Structured::Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties) : ::error::FatalRuntime(status, msg, location) { this->properties = properties; } @@ -1207,18 +1213,18 @@ value::Dict* Structured::ToDict() { if (this->properties == nullptr) { this->properties = Alloc>(); } - this->properties->set(str58, num::ToBig(this->ExitStatus())); - this->properties->set(str59, Alloc(this->msg)); + this->properties->set(str59, num::ToBig(this->ExitStatus())); + this->properties->set(str60, Alloc(this->msg)); return Alloc(this->properties); } -AssertionErr::AssertionErr(BigStr* msg, syntax_asdl::loc_t* location) : Expr(msg, location) { +AssertionErr::AssertionErr(BigStr* msg, syntax_asdl::loc_t* location) : ::error::Expr(msg, location) { } -TypeErrVerbose::TypeErrVerbose(BigStr* msg, syntax_asdl::loc_t* location) : Expr(msg, location) { +TypeErrVerbose::TypeErrVerbose(BigStr* msg, syntax_asdl::loc_t* location) : ::error::Expr(msg, location) { } -TypeErr::TypeErr(value_asdl::value_t* actual_val, BigStr* msg, syntax_asdl::loc_t* location) : TypeErrVerbose(StrFormat("%s, got %s", msg, _ValType(actual_val)), location) { +TypeErr::TypeErr(value_asdl::value_t* actual_val, BigStr* msg, syntax_asdl::loc_t* location) : ::error::TypeErrVerbose(StrFormat("%s, got %s", msg, _ValType(actual_val)), location) { } Runtime::Runtime(BigStr* msg) { @@ -1416,7 +1422,7 @@ void _Attributes::Set(BigStr* name, value_asdl::value_t* val) { StackRoot _root0(&name); StackRoot _root1(&val); - name = name->replace(str62, str63); + name = name->replace(str63, str64); this->attrs->set(name, val); } @@ -1490,6 +1496,12 @@ bool Reader::AtEnd() { return this->i >= this->n; } +void Reader::Done() { + if (!this->AtEnd()) { + e_usage(str67, this->Location()); + } +} + syntax_asdl::loc_t* Reader::_FirstLocation() { if ((this->locs != nullptr and this->locs->at(0) != nullptr)) { return this->locs->at(0); @@ -1561,7 +1573,7 @@ bool _ArgAction::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attri arg_r->Next(); arg = arg_r->Peek(); if (arg == nullptr) { - e_usage(StrFormat("expected argument to %r", str_concat(str67, this->name)), arg_r->Location()); + e_usage(StrFormat("expected argument to %r", str_concat(str69, this->name)), arg_r->Location()); } } val = this->_Value(arg, arg_r->Location()); @@ -1569,7 +1581,7 @@ bool _ArgAction::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attri return this->quit_parsing_flags; } -SetToInt::SetToInt(BigStr* name) : _ArgAction(name, false, nullptr) { +SetToInt::SetToInt(BigStr* name) : ::args::_ArgAction(name, false, nullptr) { } value_asdl::value_t* SetToInt::_Value(BigStr* arg, syntax_asdl::loc_t* location) { @@ -1581,15 +1593,15 @@ value_asdl::value_t* SetToInt::_Value(BigStr* arg, syntax_asdl::loc_t* location) i = mops::FromStr(arg); } catch (ValueError*) { - e_usage(StrFormat("expected integer after %s, got %r", str_concat(str69, this->name), arg), location); + e_usage(StrFormat("expected integer after %s, got %r", str_concat(str71, this->name), arg), location); } if (mops::Greater(mops::BigInt(0), i)) { - e_usage(StrFormat("got invalid integer for %s: %s", str_concat(str71, this->name), arg), location); + e_usage(StrFormat("got invalid integer for %s: %s", str_concat(str73, this->name), arg), location); } return Alloc(i); } -SetToFloat::SetToFloat(BigStr* name) : _ArgAction(name, false, nullptr) { +SetToFloat::SetToFloat(BigStr* name) : ::args::_ArgAction(name, false, nullptr) { } value_asdl::value_t* SetToFloat::_Value(BigStr* arg, syntax_asdl::loc_t* location) { @@ -1601,15 +1613,15 @@ value_asdl::value_t* SetToFloat::_Value(BigStr* arg, syntax_asdl::loc_t* locatio f = to_float(arg); } catch (ValueError*) { - e_usage(StrFormat("expected number after %r, got %r", str_concat(str73, this->name), arg), location); + e_usage(StrFormat("expected number after %r, got %r", str_concat(str75, this->name), arg), location); } if (f < 0) { - e_usage(StrFormat("got invalid float for %s: %s", str_concat(str75, this->name), arg), location); + e_usage(StrFormat("got invalid float for %s: %s", str_concat(str77, this->name), arg), location); } return Alloc(f); } -SetToString::SetToString(BigStr* name, bool quit_parsing_flags, List* valid) : _ArgAction(name, quit_parsing_flags, valid) { +SetToString::SetToString(BigStr* name, bool quit_parsing_flags, List* valid) : ::args::_ArgAction(name, quit_parsing_flags, valid) { } value_asdl::value_t* SetToString::_Value(BigStr* arg, syntax_asdl::loc_t* location) { @@ -1617,7 +1629,7 @@ value_asdl::value_t* SetToString::_Value(BigStr* arg, syntax_asdl::loc_t* locati StackRoot _root1(&location); if ((this->valid != nullptr and !list_contains(this->valid, arg))) { - e_usage(StrFormat("got invalid argument %r to %r, expected one of: %s", arg, str_concat(str77, this->name), str78->join(this->valid)), location); + e_usage(StrFormat("got invalid argument %r to %r, expected one of: %s", arg, str_concat(str79, this->name), str80->join(this->valid)), location); } return Alloc(arg); } @@ -1633,11 +1645,11 @@ bool SetAttachedBool::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_ StackRoot _root2(&out); if (attached_arg != nullptr) { - if ((str_equals(attached_arg, str79) || str_equals(attached_arg, str80) || str_equals(attached_arg, str81) || str_equals(attached_arg, str82))) { + if ((str_equals(attached_arg, str81) || str_equals(attached_arg, str82) || str_equals(attached_arg, str83) || str_equals(attached_arg, str84))) { b = false; } else { - if ((str_equals(attached_arg, str83) || str_equals(attached_arg, str84) || str_equals(attached_arg, str85) || str_equals(attached_arg, str86))) { + if ((str_equals(attached_arg, str85) || str_equals(attached_arg, str86) || str_equals(attached_arg, str87) || str_equals(attached_arg, str88))) { b = true; } else { @@ -1675,7 +1687,7 @@ bool SetOption::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attrib StackRoot _root1(&arg_r); StackRoot _root2(&out); - b = maybe_str_equals(attached_arg, str88); + b = maybe_str_equals(attached_arg, str90); out->opt_changes->append((Alloc>(this->name, b))); return false; } @@ -1703,7 +1715,7 @@ bool SetNamedOption::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_A StackRoot _root4(&attr_name); StackRoot _root5(&changes); - b = maybe_str_equals(attached_arg, str89); + b = maybe_str_equals(attached_arg, str91); arg_r->Next(); arg = arg_r->Peek(); if (arg == nullptr) { @@ -1754,7 +1766,7 @@ bool SetNamedAction::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_A arg_r->Next(); arg = arg_r->Peek(); if (arg == nullptr) { - e_usage(str91, loc::Missing); + e_usage(str93, loc::Missing); } attr_name = arg; if ((len(this->names) and !list_contains(this->names, attr_name))) { @@ -1787,13 +1799,13 @@ args::_Attributes* Parse(flag_spec::_FlagSpec* spec, args::Reader* arg_r) { out = Alloc<_Attributes>(spec->defaults); while (!arg_r->AtEnd()) { arg = arg_r->Peek(); - if (maybe_str_equals(arg, str93)) { + if (maybe_str_equals(arg, str95)) { out->saw_double_dash = true; arg_r->Next(); break; } - if ((len(spec->actions_long) and arg->startswith(str94))) { - pos = arg->find(str95, 2); + if ((len(spec->actions_long) and arg->startswith(str96))) { + pos = arg->find(str97, 2); if (pos == -1) { suffix = nullptr; flag_name = arg->slice(2); @@ -1811,15 +1823,15 @@ args::_Attributes* Parse(flag_spec::_FlagSpec* spec, args::Reader* arg_r) { continue; } else { - if ((arg->startswith(str97) and len(arg) > 1)) { + if ((arg->startswith(str99) and len(arg) > 1)) { n = len(arg); for (int i = 1; i < n; ++i) { ch = arg->at(i); - if (str_equals(ch, str98)) { - ch = str99; + if (str_equals(ch, str100)) { + ch = str101; } if (list_contains(spec->plus_flags, ch)) { - out->Set(ch, Alloc(str100)); + out->Set(ch, Alloc(str102)); continue; } if (list_contains(spec->arity0, ch)) { @@ -1832,20 +1844,20 @@ args::_Attributes* Parse(flag_spec::_FlagSpec* spec, args::Reader* arg_r) { action->OnMatch(attached_arg, arg_r, out); break; } - e_usage(StrFormat("doesn't accept flag %s", str_concat(str102, ch)), arg_r->Location()); + e_usage(StrFormat("doesn't accept flag %s", str_concat(str104, ch)), arg_r->Location()); } arg_r->Next(); } else { - if ((len(spec->plus_flags) and (arg->startswith(str103) and len(arg) > 1))) { + if ((len(spec->plus_flags) and (arg->startswith(str105) and len(arg) > 1))) { n = len(arg); for (int i = 1; i < n; ++i) { ch = arg->at(i); if (list_contains(spec->plus_flags, ch)) { - out->Set(ch, Alloc(str104)); + out->Set(ch, Alloc(str106)); continue; } - e_usage(StrFormat("doesn't accept option %s", str_concat(str106, ch)), arg_r->Location()); + e_usage(StrFormat("doesn't accept option %s", str_concat(str108, ch)), arg_r->Location()); } arg_r->Next(); } @@ -1873,7 +1885,7 @@ args::_Attributes* ParseLikeEcho(flag_spec::_FlagSpec* spec, args::Reader* arg_r while (!arg_r->AtEnd()) { arg = arg_r->Peek(); chars = arg->slice(1); - if ((arg->startswith(str107) and len(chars))) { + if ((arg->startswith(str109) and len(chars))) { done = false; for (StrIter it(chars); !it.Done(); it.Next()) { BigStr* c = it.Value(); @@ -1919,12 +1931,12 @@ args::_Attributes* ParseMore(flag_spec::_FlagSpecAndMore* spec, args::Reader* ar quit = false; while (!arg_r->AtEnd()) { arg = arg_r->Peek(); - if (maybe_str_equals(arg, str108)) { + if (maybe_str_equals(arg, str110)) { out->saw_double_dash = true; arg_r->Next(); break; } - if (arg->startswith(str109)) { + if (arg->startswith(str111)) { action = spec->actions_long->get(arg->slice(2)); if (action == nullptr) { e_usage(StrFormat("got invalid flag %r", arg), arg_r->Location()); @@ -1933,14 +1945,14 @@ args::_Attributes* ParseMore(flag_spec::_FlagSpecAndMore* spec, args::Reader* ar arg_r->Next(); continue; } - if (((arg->startswith(str111) or arg->startswith(str112)) and len(arg) > 1)) { + if (((arg->startswith(str113) or arg->startswith(str114)) and len(arg) > 1)) { char0 = arg->at(0); for (StrIter it(arg->slice(1)); !it.Done(); it.Next()) { BigStr* ch = it.Value(); StackRoot _for(&ch ); action = spec->actions_short->get(ch); if (action == nullptr) { - e_usage(StrFormat("got invalid flag %r", str_concat(str114, ch)), arg_r->Location()); + e_usage(StrFormat("got invalid flag %r", str_concat(str116, ch)), arg_r->Location()); } attached_arg = list_contains(spec->plus_flags, ch) ? char0 : nullptr; quit = action->OnMatch(attached_arg, arg_r, out); diff --git a/prebuilt/frontend/args.mycpp.h b/prebuilt/frontend/args.mycpp.h index b9eb2c12fd..dafe4d6905 100644 --- a/prebuilt/frontend/args.mycpp.h +++ b/prebuilt/frontend/args.mycpp.h @@ -70,7 +70,6 @@ class TraversalState { extern BigStr* TRUE_STR; extern BigStr* FALSE_STR; - } // declare namespace runtime namespace format { // declare @@ -103,7 +102,7 @@ class ColorOutput { DISALLOW_COPY_AND_ASSIGN(ColorOutput) }; -class TextOutput : public ColorOutput { +class TextOutput : public ::format::ColorOutput { public: TextOutput(mylib::Writer* f); virtual format::TextOutput* NewTempBuffer(); @@ -111,7 +110,7 @@ class TextOutput : public ColorOutput { virtual void PopColor(); static constexpr uint32_t field_mask() { - return ColorOutput::field_mask(); + return ::format::ColorOutput::field_mask(); } static constexpr ObjHeader obj_header() { @@ -121,7 +120,7 @@ class TextOutput : public ColorOutput { DISALLOW_COPY_AND_ASSIGN(TextOutput) }; -class HtmlOutput : public ColorOutput { +class HtmlOutput : public ::format::ColorOutput { public: HtmlOutput(mylib::Writer* f); virtual format::HtmlOutput* NewTempBuffer(); @@ -132,7 +131,7 @@ class HtmlOutput : public ColorOutput { virtual void write(BigStr* s); static constexpr uint32_t field_mask() { - return ColorOutput::field_mask(); + return ::format::ColorOutput::field_mask(); } static constexpr ObjHeader obj_header() { @@ -142,7 +141,7 @@ class HtmlOutput : public ColorOutput { DISALLOW_COPY_AND_ASSIGN(HtmlOutput) }; -class AnsiOutput : public ColorOutput { +class AnsiOutput : public ::format::ColorOutput { public: AnsiOutput(mylib::Writer* f); virtual format::AnsiOutput* NewTempBuffer(); @@ -150,7 +149,7 @@ class AnsiOutput : public ColorOutput { virtual void PopColor(); static constexpr uint32_t field_mask() { - return ColorOutput::field_mask(); + return ::format::ColorOutput::field_mask(); } static constexpr ObjHeader obj_header() { @@ -181,7 +180,6 @@ bool _TrySingleLineObj(hnode::Record* node, format::ColorOutput* f, int max_char bool _TrySingleLine(hnode_asdl::hnode_t* node, format::ColorOutput* f, int max_chars); void PrintTree(hnode_asdl::hnode_t* node, format::ColorOutput* f); - } // declare namespace format namespace args { // declare @@ -221,6 +219,7 @@ class Reader { List* Rest(); Tuple2*, List*> Rest2(); bool AtEnd(); + void Done(); syntax_asdl::loc_t* _FirstLocation(); syntax_asdl::loc_t* Location(); List* argv; @@ -251,7 +250,7 @@ class _Action { DISALLOW_COPY_AND_ASSIGN(_Action) }; -class _ArgAction : public _Action { +class _ArgAction : public ::args::_Action { public: _ArgAction(BigStr* name, bool quit_parsing_flags, List* valid = nullptr); virtual value_asdl::value_t* _Value(BigStr* arg, syntax_asdl::loc_t* location); @@ -262,7 +261,7 @@ class _ArgAction : public _Action { List* valid; static constexpr uint32_t field_mask() { - return _Action::field_mask() + return ::args::_Action::field_mask() | maskbit(offsetof(_ArgAction, name)) | maskbit(offsetof(_ArgAction, valid)); } @@ -274,13 +273,13 @@ class _ArgAction : public _Action { DISALLOW_COPY_AND_ASSIGN(_ArgAction) }; -class SetToInt : public _ArgAction { +class SetToInt : public ::args::_ArgAction { public: SetToInt(BigStr* name); virtual value_asdl::value_t* _Value(BigStr* arg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ArgAction::field_mask(); + return ::args::_ArgAction::field_mask(); } static constexpr ObjHeader obj_header() { @@ -290,13 +289,13 @@ class SetToInt : public _ArgAction { DISALLOW_COPY_AND_ASSIGN(SetToInt) }; -class SetToFloat : public _ArgAction { +class SetToFloat : public ::args::_ArgAction { public: SetToFloat(BigStr* name); virtual value_asdl::value_t* _Value(BigStr* arg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ArgAction::field_mask(); + return ::args::_ArgAction::field_mask(); } static constexpr ObjHeader obj_header() { @@ -306,13 +305,13 @@ class SetToFloat : public _ArgAction { DISALLOW_COPY_AND_ASSIGN(SetToFloat) }; -class SetToString : public _ArgAction { +class SetToString : public ::args::_ArgAction { public: SetToString(BigStr* name, bool quit_parsing_flags, List* valid = nullptr); virtual value_asdl::value_t* _Value(BigStr* arg, syntax_asdl::loc_t* location); static constexpr uint32_t field_mask() { - return _ArgAction::field_mask(); + return ::args::_ArgAction::field_mask(); } static constexpr ObjHeader obj_header() { @@ -322,7 +321,7 @@ class SetToString : public _ArgAction { DISALLOW_COPY_AND_ASSIGN(SetToString) }; -class SetAttachedBool : public _Action { +class SetAttachedBool : public ::args::_Action { public: SetAttachedBool(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); @@ -330,7 +329,7 @@ class SetAttachedBool : public _Action { BigStr* name; static constexpr uint32_t field_mask() { - return _Action::field_mask() + return ::args::_Action::field_mask() | maskbit(offsetof(SetAttachedBool, name)); } @@ -341,7 +340,7 @@ class SetAttachedBool : public _Action { DISALLOW_COPY_AND_ASSIGN(SetAttachedBool) }; -class SetToTrue : public _Action { +class SetToTrue : public ::args::_Action { public: SetToTrue(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); @@ -349,7 +348,7 @@ class SetToTrue : public _Action { BigStr* name; static constexpr uint32_t field_mask() { - return _Action::field_mask() + return ::args::_Action::field_mask() | maskbit(offsetof(SetToTrue, name)); } @@ -360,7 +359,7 @@ class SetToTrue : public _Action { DISALLOW_COPY_AND_ASSIGN(SetToTrue) }; -class SetOption : public _Action { +class SetOption : public ::args::_Action { public: SetOption(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); @@ -368,7 +367,7 @@ class SetOption : public _Action { BigStr* name; static constexpr uint32_t field_mask() { - return _Action::field_mask() + return ::args::_Action::field_mask() | maskbit(offsetof(SetOption, name)); } @@ -379,7 +378,7 @@ class SetOption : public _Action { DISALLOW_COPY_AND_ASSIGN(SetOption) }; -class SetNamedOption : public _Action { +class SetNamedOption : public ::args::_Action { public: SetNamedOption(bool shopt = false); void ArgName(BigStr* name); @@ -389,7 +388,7 @@ class SetNamedOption : public _Action { bool shopt; static constexpr uint32_t field_mask() { - return _Action::field_mask() + return ::args::_Action::field_mask() | maskbit(offsetof(SetNamedOption, names)); } @@ -400,7 +399,7 @@ class SetNamedOption : public _Action { DISALLOW_COPY_AND_ASSIGN(SetNamedOption) }; -class SetAction : public _Action { +class SetAction : public ::args::_Action { public: SetAction(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); @@ -408,7 +407,7 @@ class SetAction : public _Action { BigStr* name; static constexpr uint32_t field_mask() { - return _Action::field_mask() + return ::args::_Action::field_mask() | maskbit(offsetof(SetAction, name)); } @@ -419,7 +418,7 @@ class SetAction : public _Action { DISALLOW_COPY_AND_ASSIGN(SetAction) }; -class SetNamedAction : public _Action { +class SetNamedAction : public ::args::_Action { public: SetNamedAction(); void ArgName(BigStr* name); @@ -428,7 +427,7 @@ class SetNamedAction : public _Action { List* names; static constexpr uint32_t field_mask() { - return _Action::field_mask() + return ::args::_Action::field_mask() | maskbit(offsetof(SetNamedAction, names)); } @@ -443,7 +442,6 @@ args::_Attributes* Parse(flag_spec::_FlagSpec* spec, args::Reader* arg_r); args::_Attributes* ParseLikeEcho(flag_spec::_FlagSpec* spec, args::Reader* arg_r); args::_Attributes* ParseMore(flag_spec::_FlagSpecAndMore* spec, args::Reader* arg_r); - } // declare namespace args #endif // FRONTEND_ARGS_MYCPP_H diff --git a/prebuilt/translate.sh b/prebuilt/translate.sh index 32a59a460d..c9d0c8685d 100755 --- a/prebuilt/translate.sh +++ b/prebuilt/translate.sh @@ -69,7 +69,7 @@ EOF } readonly -a ASDL_FILES=( - $REPO_ROOT/{asdl/runtime,asdl/format,core/ansi,pylib/cgi,data_lang/j8_lite}.py \ + $REPO_ROOT/{asdl/runtime,asdl/format,display/ansi,pylib/cgi,data_lang/j8_lite}.py \ ) asdl-runtime() { diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 77c190362d..35f92135ee 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -944,6 +944,10 @@ test-pp() { _ysh-expr-error 'pp (42/0)' _ysh-expr-error 'pp [42/0]' + # Multiple lines + _ysh-expr-error 'pp [42 +/0]' + _ysh-expr-error 'pp [5, 6]' _ysh-should-run 'pp (42)' From 06c63f8e6734cbeb5f85e9b3cac2a48fe9f3c3c8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 13:47:20 -0400 Subject: [PATCH 062/506] [soil] Try Mythic Beasts again Since both Dreamhost and OpalStack have SSH failures We really need 'wwup' though --- soil/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/soil/common.sh b/soil/common.sh index c85d39b4c8..1dc85b71ae 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -20,12 +20,12 @@ dump-env() { env | grep -v '^encrypted_' | sort } -if true; then +if false; then readonly SOIL_USER='travis_admin' readonly SOIL_HOST='travis-ci.oilshell.org' readonly SOIL_HOST_DIR=~/travis-ci.oilshell.org # used on server readonly SOIL_REMOTE_DIR=travis-ci.oilshell.org # used on client -elif false; then +elif true; then readonly SOIL_USER='oils' readonly SOIL_HOST='mb.oils.pub' # Extra level From 1205e4f0a824ae76e5c8f4fe23277c63ca815255 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 14:29:50 -0400 Subject: [PATCH 063/506] [refactor] Move pretty printer to display/ dir --- README.md | 3 ++- bin/NINJA_subgraph.py | 2 +- build/ninja_main.py | 4 ++++ build/py.sh | 2 +- core/comp_ui.py | 2 +- cpp/preamble.h | 2 +- data_lang/NINJA_subgraph.py | 1 - display/NINJA_subgraph.py | 16 ++++++++++++++++ {data_lang => display}/pretty-benchmark.sh | 0 {data_lang => display}/pretty.asdl | 0 {data_lang => display}/pretty.py | 0 {data_lang => display}/pretty_test.py | 2 +- {data_lang => display}/pretty_test.txt | 0 display/ui.py | 2 +- metrics/source-code.sh | 2 +- 15 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 display/NINJA_subgraph.py rename {data_lang => display}/pretty-benchmark.sh (100%) rename {data_lang => display}/pretty.asdl (100%) rename {data_lang => display}/pretty.py (100%) rename {data_lang => display}/pretty_test.py (98%) rename {data_lang => display}/pretty_test.txt (100%) diff --git a/README.md b/README.md index 0977f8dc56..31a09293f2 100644 --- a/README.md +++ b/README.md @@ -143,11 +143,12 @@ languages, Zephyr ASDL, and a statically-typed subset of Python. osh/ # OSH parsers and evaluators (cmd, word, sh_expr) ysh/ # YSH parser and evaluator data_lang/ # Languages based on JSON - builtin/ # Builtin commands and functions core/ # Other code shared between OSH and YSH + builtin/ # Builtin commands and functions pyext/ # Python extension modules, e.g. libc.c pylib/ # Borrowed from the Python standard library. tools/ # User-facing tools, e.g. the osh2oil translator + display/ # User interface ### DSLs / Code Generators diff --git a/bin/NINJA_subgraph.py b/bin/NINJA_subgraph.py index 71a0577dca..991606c06c 100644 --- a/bin/NINJA_subgraph.py +++ b/bin/NINJA_subgraph.py @@ -107,7 +107,7 @@ def NinjaGraph(ru): '//cpp/frontend_match', '//cpp/frontend_pyreadline', '//data_lang/nil8.asdl', - '//data_lang/pretty.asdl', + '//display/pretty.asdl', '//frontend/arg_types', '//frontend/consts', '//frontend/help_meta', diff --git a/build/ninja_main.py b/build/ninja_main.py index d76fe6fcba..8f89da692c 100755 --- a/build/ninja_main.py +++ b/build/ninja_main.py @@ -20,6 +20,7 @@ from core import NINJA_subgraph as core_subgraph from cpp import NINJA_subgraph as cpp_subgraph from data_lang import NINJA_subgraph as data_lang_subgraph +from display import NINJA_subgraph as display_subgraph from frontend import NINJA_subgraph as frontend_subgraph from ysh import NINJA_subgraph as ysh_subgraph from osh import NINJA_subgraph as osh_subgraph @@ -364,6 +365,9 @@ def main(argv): data_lang_subgraph.NinjaGraph(ru) ru.comment('') + display_subgraph.NinjaGraph(ru) + ru.comment('') + frontend_subgraph.NinjaGraph(ru) ru.comment('') diff --git a/build/py.sh b/build/py.sh index dff65ca581..2f4d71904d 100755 --- a/build/py.sh +++ b/build/py.sh @@ -140,7 +140,7 @@ py-codegen() { gen-asdl-py 'core/runtime.asdl' gen-asdl-py 'core/value.asdl' gen-asdl-py 'data_lang/nil8.asdl' - gen-asdl-py 'data_lang/pretty.asdl' + gen-asdl-py 'display/pretty.asdl' gen-asdl-py 'tools/find/find.asdl' diff --git a/core/comp_ui.py b/core/comp_ui.py index 5d71ef387d..4f9ed44d2f 100644 --- a/core/comp_ui.py +++ b/core/comp_ui.py @@ -3,7 +3,7 @@ from display import ansi from core import completion -from data_lang import pretty +from display import pretty import libc from mycpp import mylib diff --git a/cpp/preamble.h b/cpp/preamble.h index a6bdaecd15..08305e38f9 100644 --- a/cpp/preamble.h +++ b/cpp/preamble.h @@ -11,7 +11,7 @@ #include "_gen/core/runtime.asdl.h" #include "_gen/core/value.asdl.h" #include "_gen/data_lang/nil8.asdl.h" -#include "_gen/data_lang/pretty.asdl.h" +#include "_gen/display/pretty.asdl.h" #include "_gen/frontend/arg_types.h" #include "_gen/frontend/consts.h" #include "_gen/frontend/help_meta.h" diff --git a/data_lang/NINJA_subgraph.py b/data_lang/NINJA_subgraph.py index 0ea05376d2..7de9e16345 100644 --- a/data_lang/NINJA_subgraph.py +++ b/data_lang/NINJA_subgraph.py @@ -14,7 +14,6 @@ def NinjaGraph(ru): ru.comment('Generated by %s' % __name__) ru.asdl_library('data_lang/nil8.asdl') - ru.asdl_library('data_lang/pretty.asdl') ru.cc_binary( 'data_lang/utf8_test.cc', diff --git a/display/NINJA_subgraph.py b/display/NINJA_subgraph.py new file mode 100644 index 0000000000..1485a9de70 --- /dev/null +++ b/display/NINJA_subgraph.py @@ -0,0 +1,16 @@ +""" +display/NINJA_subgraph.py +""" + +from __future__ import print_function + +from build import ninja_lib +from build.ninja_lib import log + + +def NinjaGraph(ru): + n = ru.n + + ru.comment('Generated by %s' % __name__) + + ru.asdl_library('display/pretty.asdl') diff --git a/data_lang/pretty-benchmark.sh b/display/pretty-benchmark.sh similarity index 100% rename from data_lang/pretty-benchmark.sh rename to display/pretty-benchmark.sh diff --git a/data_lang/pretty.asdl b/display/pretty.asdl similarity index 100% rename from data_lang/pretty.asdl rename to display/pretty.asdl diff --git a/data_lang/pretty.py b/display/pretty.py similarity index 100% rename from data_lang/pretty.py rename to display/pretty.py diff --git a/data_lang/pretty_test.py b/display/pretty_test.py similarity index 98% rename from data_lang/pretty_test.py rename to display/pretty_test.py index 0d4ece6294..ce5fd6dba7 100755 --- a/data_lang/pretty_test.py +++ b/display/pretty_test.py @@ -7,7 +7,7 @@ from display import ansi from display import ui from data_lang import j8 -from data_lang import pretty # module under test +from display import pretty # module under test from mycpp import mylib from typing import Optional diff --git a/data_lang/pretty_test.txt b/display/pretty_test.txt similarity index 100% rename from data_lang/pretty_test.txt rename to display/pretty_test.txt diff --git a/display/ui.py b/display/ui.py index 5eeb1d3272..8d83e59e6a 100644 --- a/display/ui.py +++ b/display/ui.py @@ -23,7 +23,7 @@ ) from _devbuild.gen.value_asdl import value_e, value_t from asdl import format as fmt -from data_lang import pretty +from display import pretty from frontend import lexer from frontend import location from mycpp import mylib diff --git a/metrics/source-code.sh b/metrics/source-code.sh index 223f037d71..69d6c5f0f7 100755 --- a/metrics/source-code.sh +++ b/metrics/source-code.sh @@ -19,7 +19,7 @@ filter-py() { grep -E -v '__init__.py$|_gen.py|_test.py|_tests.py|NINJA_subgraph.py$' } -readonly -a OSH_ASDL=( {frontend,core}/*.asdl ) +readonly -a OSH_ASDL=( {frontend,core,display}/*.asdl ) oils-files() { # what's in the runtime From ce40460268ad6e6c13320fc0f9201fd4c279eaab Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 14:41:28 -0400 Subject: [PATCH 064/506] [build] Regenerate dynamic deps --- pea/oils-typecheck.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pea/oils-typecheck.txt b/pea/oils-typecheck.txt index c523d8d057..f640aa493b 100644 --- a/pea/oils-typecheck.txt +++ b/pea/oils-typecheck.txt @@ -59,9 +59,9 @@ core/util.py core/vm.py data_lang/j8.py data_lang/j8_lite.py -data_lang/pretty.py data_lang/pyj8.py display/ansi.py +display/pretty.py display/ui.py frontend/args.py frontend/builtin_def.py From 49a1ee44393dba029eb036ddef35a5f4f4959ae3 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 14:50:05 -0400 Subject: [PATCH 065/506] [display refactor] Separate pretty.py and enc_value.py So we can have enc_hnode.py for ASDL. Most "constructors" like _Text() are still "private" --- display/enc_value.py | 440 +++++++++++++++++++++++++++++++++++++++++ display/pretty.py | 436 +--------------------------------------- display/pretty_test.py | 5 +- display/ui.py | 9 +- test/lint.sh | 2 +- 5 files changed, 452 insertions(+), 440 deletions(-) create mode 100644 display/enc_value.py diff --git a/display/enc_value.py b/display/enc_value.py new file mode 100644 index 0000000000..10772535e1 --- /dev/null +++ b/display/enc_value.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python2 +""" +Render Oils value_t -> doc_t, so it can be pretty printed +""" + +from __future__ import print_function + +import math + +from _devbuild.gen.pretty_asdl import (doc, MeasuredDoc) +from _devbuild.gen.value_asdl import value, value_e, value_t, value_str +from data_lang import j8 +from data_lang import j8_lite +from display.pretty import (_Break, _Concat, _Flat, _Group, _IfFlat, _Indent, _Text, _EmptyMeasure, TryUnicodeWidth) +from display import ansi +from frontend import match +from mycpp import mops +from mycpp.mylib import log, tagswitch, iteritems +from typing import cast, List, Dict + +_ = log + + +def ValType(val): + # type: (value_t) -> str + """Returns a user-facing string like Int, Eggex, BashArray, etc.""" + return value_str(val.tag(), dot=False) + + +def _FloatString(fl): + # type: (float) -> str + + # Print in YSH syntax, similar to data_lang/j8.py + if math.isinf(fl): + s = 'INFINITY' + if fl < 0: + s = '-' + s + elif math.isnan(fl): + s = 'NAN' + else: + s = str(fl) + return s + + +class ValueEncoder: + """Converts Oils values into `doc`s, which can then be pretty printed.""" + + def __init__(self): + # type: () -> None + + # Default values + self.indent = 4 + self.use_styles = True + # Tuned for 'data_lang/pretty-benchmark.sh float-demo' + # TODO: might want options for float width + self.max_tabular_width = 22 + + self.ysh_style = True + + self.visiting = {} # type: Dict[int, bool] + + # These can be configurable later + self.int_style = ansi.YELLOW + self.float_style = ansi.BLUE + self.null_style = ansi.RED + self.bool_style = ansi.CYAN + self.string_style = ansi.GREEN + self.cycle_style = ansi.BOLD + ansi.BLUE + self.type_style = ansi.MAGENTA + + def SetIndent(self, indent): + # type: (int) -> None + """Set the number of spaces per indent.""" + self.indent = indent + + def SetUseStyles(self, use_styles): + # type: (bool) -> None + """Print with ansi colors and styles, rather than plain text.""" + self.use_styles = use_styles + + def SetMaxTabularWidth(self, max_tabular_width): + # type: (int) -> None + """Set the maximum width that list elements can be, for them to be + vertically aligned.""" + self.max_tabular_width = max_tabular_width + + def TypePrefix(self, type_str): + # type: (str) -> List[MeasuredDoc] + """Return docs for type string "(List)", which may break afterward.""" + type_name = self._Styled(self.type_style, _Text(type_str)) + + n = len(type_str) + # Our maximum string is "Float" + assert n <= 5, type_str + + # Start printing in column 8. Adjust to 6 because () takes 2 spaces. + spaces = ' ' * (6 - n) + + mdocs = [_Text("("), type_name, _Text(")"), _Break(spaces)] + return mdocs + + def Value(self, val): + # type: (value_t) -> MeasuredDoc + """Convert an Oils value into a `doc`, which can then be pretty printed.""" + self.visiting.clear() + return self._Value(val) + + def _Styled(self, style, mdoc): + # type: (str, MeasuredDoc) -> MeasuredDoc + """Apply the ANSI style string to the given node, if use_styles is set.""" + if self.use_styles: + return _Concat([ + MeasuredDoc(doc.Text(style), _EmptyMeasure()), mdoc, + MeasuredDoc(doc.Text(ansi.RESET), _EmptyMeasure()) + ]) + else: + return mdoc + + def _Surrounded(self, open, mdoc, close): + # type: (str, MeasuredDoc, str) -> MeasuredDoc + """Print one of two options (using '[', ']' for open, close): + + ``` + [mdoc] + ------ + [ + mdoc + ] + ``` + """ + return _Group( + _Concat([ + _Text(open), + _Indent(self.indent, _Concat([_Break(""), mdoc])), + _Break(""), + _Text(close) + ])) + + def _SurroundedAndPrefixed(self, open, prefix, sep, mdoc, close): + # type: (str, MeasuredDoc, str, MeasuredDoc, str) -> MeasuredDoc + """Print one of two options + (using '[', 'prefix', ':', 'mdoc', ']' for open, prefix, sep, mdoc, close): + + ``` + [prefix:mdoc] + ------ + [prefix + mdoc + ] + ``` + """ + return _Group( + _Concat([ + _Text(open), prefix, + _Indent(self.indent, _Concat([_Break(sep), mdoc])), + _Break(""), + _Text(close) + ])) + + def _Join(self, items, sep, space): + # type: (List[MeasuredDoc], str, str) -> MeasuredDoc + """Join `items`, using either 'sep+space' or 'sep+newline' between them. + + E.g., if sep and space are ',' and '_', print one of these two cases: + ``` + first,_second,_third + ------ + first, + second, + third + ``` + """ + seq = [] # type: List[MeasuredDoc] + for i, item in enumerate(items): + if i != 0: + seq.append(_Text(sep)) + seq.append(_Break(space)) + seq.append(item) + return _Concat(seq) + + def _Tabular(self, items, sep): + # type: (List[MeasuredDoc], str) -> MeasuredDoc + """Join `items` together, using one of three styles: + + (showing spaces as underscores for clarity) + ``` + first,_second,_third,_fourth,_fifth,_sixth,_seventh,_eighth + ------ + first,___second,__third, + fourth,__fifth,___sixth, + seventh,_eighth + ------ + first, + second, + third, + fourth, + fifth, + sixth, + seventh, + eighth + ``` + + The first "single line" style is used if the items fit on one line. The + second "tabular' style is used if the flat width of all items is no + greater than `self.max_tabular_width`. The third "multi line" style is + used otherwise. + """ + + # Why not "just" use tabular alignment so long as two items fit on every + # line? Because it isn't possible to check for that in the pretty + # printing language. There are two sorts of conditionals we can do: + # + # A. Inside the pretty printing language, which supports exactly one + # conditional: "does it fit on one line?". + # B. Outside the pretty printing language we can run arbitrary Python + # code, but we don't know how much space is available on the line + # because it depends on the context in which we're printed, which may + # vary. + # + # We're picking between the three styles, by using (A) to check if the + # first style fits on one line, then using (B) with "are all the items + # smaller than `self.max_tabular_width`?" to pick between style 2 and + # style 3. + + if len(items) == 0: + return _Text("") + + max_flat_len = 0 + seq = [] # type: List[MeasuredDoc] + for i, item in enumerate(items): + if i != 0: + seq.append(_Text(sep)) + seq.append(_Break(" ")) + seq.append(item) + max_flat_len = max(max_flat_len, item.measure.flat) + non_tabular = _Concat(seq) + + sep_width = TryUnicodeWidth(sep) + if max_flat_len + sep_width + 1 <= self.max_tabular_width: + tabular_seq = [] # type: List[MeasuredDoc] + for i, item in enumerate(items): + tabular_seq.append(_Flat(item)) + if i != len(items) - 1: + padding = max_flat_len - item.measure.flat + 1 + tabular_seq.append(_Text(sep)) + tabular_seq.append(_Group(_Break(" " * padding))) + tabular = _Concat(tabular_seq) + return _Group(_IfFlat(non_tabular, tabular)) + else: + return non_tabular + + def _DictKey(self, s): + # type: (str) -> MeasuredDoc + if match.IsValidVarName(s): + encoded = s + else: + if self.ysh_style: + encoded = j8_lite.YshEncodeString(s) + else: + # TODO: remove this dead branch after fixing tests + encoded = j8_lite.EncodeString(s) + return _Text(encoded) + + def _StringLiteral(self, s): + # type: (str) -> MeasuredDoc + if self.ysh_style: + # YSH r'' or b'' style + encoded = j8_lite.YshEncodeString(s) + else: + # TODO: remove this dead branch after fixing tests + encoded = j8_lite.EncodeString(s) + return self._Styled(self.string_style, _Text(encoded)) + + def _BashStringLiteral(self, s): + # type: (str) -> MeasuredDoc + + # '' or $'' style + # + # We mimic bash syntax by using $'\\' instead of b'\\' + # + # $ declare -a array=($'\\') + # $ = array + # (BashArray) (BashArray $'\\') + # + # $ declare -A assoc=([k]=$'\\') + # $ = assoc + # (BashAssoc) (BashAssoc ['k']=$'\\') + + encoded = j8_lite.ShellEncode(s) + return self._Styled(self.string_style, _Text(encoded)) + + def _YshList(self, vlist): + # type: (value.List) -> MeasuredDoc + """Print a string literal.""" + if len(vlist.items) == 0: + return _Text("[]") + mdocs = [self._Value(item) for item in vlist.items] + return self._Surrounded("[", self._Tabular(mdocs, ","), "]") + + def _YshDict(self, vdict): + # type: (value.Dict) -> MeasuredDoc + if len(vdict.d) == 0: + return _Text("{}") + mdocs = [] # type: List[MeasuredDoc] + for k, v in iteritems(vdict.d): + mdocs.append( + _Concat([self._DictKey(k), + _Text(": "), + self._Value(v)])) + return self._Surrounded("{", self._Join(mdocs, ",", " "), "}") + + def _BashArray(self, varray): + # type: (value.BashArray) -> MeasuredDoc + type_name = self._Styled(self.type_style, _Text("BashArray")) + if len(varray.strs) == 0: + return _Concat([_Text("("), type_name, _Text(")")]) + mdocs = [] # type: List[MeasuredDoc] + for s in varray.strs: + if s is None: + mdocs.append(_Text("null")) + else: + mdocs.append(self._BashStringLiteral(s)) + return self._SurroundedAndPrefixed("(", type_name, " ", + self._Tabular(mdocs, ""), ")") + + def _BashAssoc(self, vassoc): + # type: (value.BashAssoc) -> MeasuredDoc + type_name = self._Styled(self.type_style, _Text("BashAssoc")) + if len(vassoc.d) == 0: + return _Concat([_Text("("), type_name, _Text(")")]) + mdocs = [] # type: List[MeasuredDoc] + for k2, v2 in iteritems(vassoc.d): + mdocs.append( + _Concat([ + _Text("["), + self._BashStringLiteral(k2), + _Text("]="), + self._BashStringLiteral(v2) + ])) + return self._SurroundedAndPrefixed("(", type_name, " ", + self._Join(mdocs, "", " "), ")") + + def _SparseArray(self, val): + # type: (value.SparseArray) -> MeasuredDoc + type_name = self._Styled(self.type_style, _Text("SparseArray")) + if len(val.d) == 0: + return _Concat([_Text("("), type_name, _Text(")")]) + mdocs = [] # type: List[MeasuredDoc] + for k2, v2 in iteritems(val.d): + mdocs.append( + _Concat([ + _Text("["), + self._Styled(self.int_style, _Text(mops.ToStr(k2))), + _Text("]="), + self._BashStringLiteral(v2) + ])) + return self._SurroundedAndPrefixed("(", type_name, " ", + self._Join(mdocs, "", " "), ")") + + def _Value(self, val): + # type: (value_t) -> MeasuredDoc + + with tagswitch(val) as case: + if case(value_e.Null): + return self._Styled(self.null_style, _Text("null")) + + elif case(value_e.Bool): + b = cast(value.Bool, val).b + return self._Styled(self.bool_style, + _Text("true" if b else "false")) + + elif case(value_e.Int): + i = cast(value.Int, val).i + return self._Styled(self.int_style, _Text(mops.ToStr(i))) + + elif case(value_e.Float): + f = cast(value.Float, val).f + return self._Styled(self.float_style, _Text(_FloatString(f))) + + elif case(value_e.Str): + s = cast(value.Str, val).s + return self._StringLiteral(s) + + elif case(value_e.Range): + r = cast(value.Range, val) + type_name = self._Styled(self.type_style, _Text(ValType(r))) + mdocs = [_Text(str(r.lower)), _Text(".."), _Text(str(r.upper))] + return self._SurroundedAndPrefixed("(", type_name, " ", + self._Join(mdocs, "", " "), + ")") + + elif case(value_e.List): + vlist = cast(value.List, val) + heap_id = j8.HeapValueId(vlist) + if self.visiting.get(heap_id, False): + return _Concat([ + _Text("["), + self._Styled(self.cycle_style, _Text("...")), + _Text("]") + ]) + else: + self.visiting[heap_id] = True + result = self._YshList(vlist) + self.visiting[heap_id] = False + return result + + elif case(value_e.Dict): + vdict = cast(value.Dict, val) + heap_id = j8.HeapValueId(vdict) + if self.visiting.get(heap_id, False): + return _Concat([ + _Text("{"), + self._Styled(self.cycle_style, _Text("...")), + _Text("}") + ]) + else: + self.visiting[heap_id] = True + result = self._YshDict(vdict) + self.visiting[heap_id] = False + return result + + elif case(value_e.SparseArray): + sparse = cast(value.SparseArray, val) + return self._SparseArray(sparse) + + elif case(value_e.BashArray): + varray = cast(value.BashArray, val) + return self._BashArray(varray) + + elif case(value_e.BashAssoc): + vassoc = cast(value.BashAssoc, val) + return self._BashAssoc(vassoc) + + else: + type_name = self._Styled(self.type_style, _Text(ValType(val))) + id_str = j8.ValueIdString(val) + return _Concat([_Text("<"), type_name, _Text(id_str + ">")]) + + +# vim: sw=4 diff --git a/display/pretty.py b/display/pretty.py index e58777c197..74f1bbebc3 100644 --- a/display/pretty.py +++ b/display/pretty.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2 """ -Pretty print Oils values (and later other data/languages as well). +Pretty printing library. Pretty printing means intelligently choosing whitespace including indentation and newline placement, to attempt to display data nicely while staying within a @@ -100,43 +100,13 @@ from __future__ import print_function -import math - from _devbuild.gen.pretty_asdl import doc, doc_e, DocFragment, Measure, MeasuredDoc -from _devbuild.gen.value_asdl import value, value_e, value_t, value_str -from data_lang import j8 -from data_lang import j8_lite -from display import ansi -from frontend import match -from mycpp import mops -from mycpp.mylib import log, tagswitch, BufWriter, iteritems -from typing import cast, List, Dict +from mycpp.mylib import log, tagswitch, BufWriter +from typing import cast, List import libc _ = log - -def ValType(val): - # type: (value_t) -> str - """Returns a user-facing string like Int, Eggex, BashArray, etc.""" - return value_str(val.tag(), dot=False) - - -def _FloatString(fl): - # type: (float) -> str - - # Print in YSH syntax, similar to data_lang/j8.py - if math.isinf(fl): - s = 'INFINITY' - if fl < 0: - s = '-' + s - elif math.isnan(fl): - s = 'NAN' - else: - s = str(fl) - return s - - ################ # Measurements # ################ @@ -342,404 +312,4 @@ def PrintDoc(self, document, buf): frag.measure)) -################ -# Value -> Doc # -################ - - -class ValueEncoder: - """Converts Oils values into `doc`s, which can then be pretty printed.""" - - def __init__(self): - # type: () -> None - - # Default values - self.indent = 4 - self.use_styles = True - # Tuned for 'data_lang/pretty-benchmark.sh float-demo' - # TODO: might want options for float width - self.max_tabular_width = 22 - - self.ysh_style = True - - self.visiting = {} # type: Dict[int, bool] - - # These can be configurable later - self.int_style = ansi.YELLOW - self.float_style = ansi.BLUE - self.null_style = ansi.RED - self.bool_style = ansi.CYAN - self.string_style = ansi.GREEN - self.cycle_style = ansi.BOLD + ansi.BLUE - self.type_style = ansi.MAGENTA - - def SetIndent(self, indent): - # type: (int) -> None - """Set the number of spaces per indent.""" - self.indent = indent - - def SetUseStyles(self, use_styles): - # type: (bool) -> None - """Print with ansi colors and styles, rather than plain text.""" - self.use_styles = use_styles - - def SetMaxTabularWidth(self, max_tabular_width): - # type: (int) -> None - """Set the maximum width that list elements can be, for them to be - vertically aligned.""" - self.max_tabular_width = max_tabular_width - - def TypePrefix(self, type_str): - # type: (str) -> List[MeasuredDoc] - """Return docs for type string "(List)", which may break afterward.""" - type_name = self._Styled(self.type_style, _Text(type_str)) - - n = len(type_str) - # Our maximum string is "Float" - assert n <= 5, type_str - - # Start printing in column 8. Adjust to 6 because () takes 2 spaces. - spaces = ' ' * (6 - n) - - mdocs = [_Text("("), type_name, _Text(")"), _Break(spaces)] - return mdocs - - def Value(self, val): - # type: (value_t) -> MeasuredDoc - """Convert an Oils value into a `doc`, which can then be pretty printed.""" - self.visiting.clear() - return self._Value(val) - - def _Styled(self, style, mdoc): - # type: (str, MeasuredDoc) -> MeasuredDoc - """Apply the ANSI style string to the given node, if use_styles is set.""" - if self.use_styles: - return _Concat([ - MeasuredDoc(doc.Text(style), _EmptyMeasure()), mdoc, - MeasuredDoc(doc.Text(ansi.RESET), _EmptyMeasure()) - ]) - else: - return mdoc - - def _Surrounded(self, open, mdoc, close): - # type: (str, MeasuredDoc, str) -> MeasuredDoc - """Print one of two options (using '[', ']' for open, close): - - ``` - [mdoc] - ------ - [ - mdoc - ] - ``` - """ - return _Group( - _Concat([ - _Text(open), - _Indent(self.indent, _Concat([_Break(""), mdoc])), - _Break(""), - _Text(close) - ])) - - def _SurroundedAndPrefixed(self, open, prefix, sep, mdoc, close): - # type: (str, MeasuredDoc, str, MeasuredDoc, str) -> MeasuredDoc - """Print one of two options - (using '[', 'prefix', ':', 'mdoc', ']' for open, prefix, sep, mdoc, close): - - ``` - [prefix:mdoc] - ------ - [prefix - mdoc - ] - ``` - """ - return _Group( - _Concat([ - _Text(open), prefix, - _Indent(self.indent, _Concat([_Break(sep), mdoc])), - _Break(""), - _Text(close) - ])) - - def _Join(self, items, sep, space): - # type: (List[MeasuredDoc], str, str) -> MeasuredDoc - """Join `items`, using either 'sep+space' or 'sep+newline' between them. - - E.g., if sep and space are ',' and '_', print one of these two cases: - ``` - first,_second,_third - ------ - first, - second, - third - ``` - """ - seq = [] # type: List[MeasuredDoc] - for i, item in enumerate(items): - if i != 0: - seq.append(_Text(sep)) - seq.append(_Break(space)) - seq.append(item) - return _Concat(seq) - - def _Tabular(self, items, sep): - # type: (List[MeasuredDoc], str) -> MeasuredDoc - """Join `items` together, using one of three styles: - - (showing spaces as underscores for clarity) - ``` - first,_second,_third,_fourth,_fifth,_sixth,_seventh,_eighth - ------ - first,___second,__third, - fourth,__fifth,___sixth, - seventh,_eighth - ------ - first, - second, - third, - fourth, - fifth, - sixth, - seventh, - eighth - ``` - - The first "single line" style is used if the items fit on one line. The - second "tabular' style is used if the flat width of all items is no - greater than `self.max_tabular_width`. The third "multi line" style is - used otherwise. - """ - - # Why not "just" use tabular alignment so long as two items fit on every - # line? Because it isn't possible to check for that in the pretty - # printing language. There are two sorts of conditionals we can do: - # - # A. Inside the pretty printing language, which supports exactly one - # conditional: "does it fit on one line?". - # B. Outside the pretty printing language we can run arbitrary Python - # code, but we don't know how much space is available on the line - # because it depends on the context in which we're printed, which may - # vary. - # - # We're picking between the three styles, by using (A) to check if the - # first style fits on one line, then using (B) with "are all the items - # smaller than `self.max_tabular_width`?" to pick between style 2 and - # style 3. - - if len(items) == 0: - return _Text("") - - max_flat_len = 0 - seq = [] # type: List[MeasuredDoc] - for i, item in enumerate(items): - if i != 0: - seq.append(_Text(sep)) - seq.append(_Break(" ")) - seq.append(item) - max_flat_len = max(max_flat_len, item.measure.flat) - non_tabular = _Concat(seq) - - sep_width = TryUnicodeWidth(sep) - if max_flat_len + sep_width + 1 <= self.max_tabular_width: - tabular_seq = [] # type: List[MeasuredDoc] - for i, item in enumerate(items): - tabular_seq.append(_Flat(item)) - if i != len(items) - 1: - padding = max_flat_len - item.measure.flat + 1 - tabular_seq.append(_Text(sep)) - tabular_seq.append(_Group(_Break(" " * padding))) - tabular = _Concat(tabular_seq) - return _Group(_IfFlat(non_tabular, tabular)) - else: - return non_tabular - - def _DictKey(self, s): - # type: (str) -> MeasuredDoc - if match.IsValidVarName(s): - encoded = s - else: - if self.ysh_style: - encoded = j8_lite.YshEncodeString(s) - else: - # TODO: remove this dead branch after fixing tests - encoded = j8_lite.EncodeString(s) - return _Text(encoded) - - def _StringLiteral(self, s): - # type: (str) -> MeasuredDoc - if self.ysh_style: - # YSH r'' or b'' style - encoded = j8_lite.YshEncodeString(s) - else: - # TODO: remove this dead branch after fixing tests - encoded = j8_lite.EncodeString(s) - return self._Styled(self.string_style, _Text(encoded)) - - def _BashStringLiteral(self, s): - # type: (str) -> MeasuredDoc - - # '' or $'' style - # - # We mimic bash syntax by using $'\\' instead of b'\\' - # - # $ declare -a array=($'\\') - # $ = array - # (BashArray) (BashArray $'\\') - # - # $ declare -A assoc=([k]=$'\\') - # $ = assoc - # (BashAssoc) (BashAssoc ['k']=$'\\') - - encoded = j8_lite.ShellEncode(s) - return self._Styled(self.string_style, _Text(encoded)) - - def _YshList(self, vlist): - # type: (value.List) -> MeasuredDoc - """Print a string literal.""" - if len(vlist.items) == 0: - return _Text("[]") - mdocs = [self._Value(item) for item in vlist.items] - return self._Surrounded("[", self._Tabular(mdocs, ","), "]") - - def _YshDict(self, vdict): - # type: (value.Dict) -> MeasuredDoc - if len(vdict.d) == 0: - return _Text("{}") - mdocs = [] # type: List[MeasuredDoc] - for k, v in iteritems(vdict.d): - mdocs.append( - _Concat([self._DictKey(k), - _Text(": "), - self._Value(v)])) - return self._Surrounded("{", self._Join(mdocs, ",", " "), "}") - - def _BashArray(self, varray): - # type: (value.BashArray) -> MeasuredDoc - type_name = self._Styled(self.type_style, _Text("BashArray")) - if len(varray.strs) == 0: - return _Concat([_Text("("), type_name, _Text(")")]) - mdocs = [] # type: List[MeasuredDoc] - for s in varray.strs: - if s is None: - mdocs.append(_Text("null")) - else: - mdocs.append(self._BashStringLiteral(s)) - return self._SurroundedAndPrefixed("(", type_name, " ", - self._Tabular(mdocs, ""), ")") - - def _BashAssoc(self, vassoc): - # type: (value.BashAssoc) -> MeasuredDoc - type_name = self._Styled(self.type_style, _Text("BashAssoc")) - if len(vassoc.d) == 0: - return _Concat([_Text("("), type_name, _Text(")")]) - mdocs = [] # type: List[MeasuredDoc] - for k2, v2 in iteritems(vassoc.d): - mdocs.append( - _Concat([ - _Text("["), - self._BashStringLiteral(k2), - _Text("]="), - self._BashStringLiteral(v2) - ])) - return self._SurroundedAndPrefixed("(", type_name, " ", - self._Join(mdocs, "", " "), ")") - - def _SparseArray(self, val): - # type: (value.SparseArray) -> MeasuredDoc - type_name = self._Styled(self.type_style, _Text("SparseArray")) - if len(val.d) == 0: - return _Concat([_Text("("), type_name, _Text(")")]) - mdocs = [] # type: List[MeasuredDoc] - for k2, v2 in iteritems(val.d): - mdocs.append( - _Concat([ - _Text("["), - self._Styled(self.int_style, _Text(mops.ToStr(k2))), - _Text("]="), - self._BashStringLiteral(v2) - ])) - return self._SurroundedAndPrefixed("(", type_name, " ", - self._Join(mdocs, "", " "), ")") - - def _Value(self, val): - # type: (value_t) -> MeasuredDoc - - with tagswitch(val) as case: - if case(value_e.Null): - return self._Styled(self.null_style, _Text("null")) - - elif case(value_e.Bool): - b = cast(value.Bool, val).b - return self._Styled(self.bool_style, - _Text("true" if b else "false")) - - elif case(value_e.Int): - i = cast(value.Int, val).i - return self._Styled(self.int_style, _Text(mops.ToStr(i))) - - elif case(value_e.Float): - f = cast(value.Float, val).f - return self._Styled(self.float_style, _Text(_FloatString(f))) - - elif case(value_e.Str): - s = cast(value.Str, val).s - return self._StringLiteral(s) - - elif case(value_e.Range): - r = cast(value.Range, val) - type_name = self._Styled(self.type_style, _Text(ValType(r))) - mdocs = [_Text(str(r.lower)), _Text(".."), _Text(str(r.upper))] - return self._SurroundedAndPrefixed("(", type_name, " ", - self._Join(mdocs, "", " "), - ")") - - elif case(value_e.List): - vlist = cast(value.List, val) - heap_id = j8.HeapValueId(vlist) - if self.visiting.get(heap_id, False): - return _Concat([ - _Text("["), - self._Styled(self.cycle_style, _Text("...")), - _Text("]") - ]) - else: - self.visiting[heap_id] = True - result = self._YshList(vlist) - self.visiting[heap_id] = False - return result - - elif case(value_e.Dict): - vdict = cast(value.Dict, val) - heap_id = j8.HeapValueId(vdict) - if self.visiting.get(heap_id, False): - return _Concat([ - _Text("{"), - self._Styled(self.cycle_style, _Text("...")), - _Text("}") - ]) - else: - self.visiting[heap_id] = True - result = self._YshDict(vdict) - self.visiting[heap_id] = False - return result - - elif case(value_e.SparseArray): - sparse = cast(value.SparseArray, val) - return self._SparseArray(sparse) - - elif case(value_e.BashArray): - varray = cast(value.BashArray, val) - return self._BashArray(varray) - - elif case(value_e.BashAssoc): - vassoc = cast(value.BashAssoc, val) - return self._BashAssoc(vassoc) - - else: - type_name = self._Styled(self.type_style, _Text(ValType(val))) - id_str = j8.ValueIdString(val) - return _Concat([_Text("<"), type_name, _Text(id_str + ">")]) - - # vim: sw=4 diff --git a/display/pretty_test.py b/display/pretty_test.py index ce5fd6dba7..9a643dca37 100755 --- a/display/pretty_test.py +++ b/display/pretty_test.py @@ -5,9 +5,10 @@ import unittest from display import ansi +from display import pretty # module under test +from display import enc_value from display import ui from data_lang import j8 -from display import pretty # module under test from mycpp import mylib from typing import Optional @@ -52,7 +53,7 @@ class PrettyTest(unittest.TestCase): def setUp(self): # Use settings that make testing easier. - self.encoder = pretty.ValueEncoder() + self.encoder = enc_value.ValueEncoder() self.encoder.SetUseStyles(False) def assertPretty(self, width, value_str, expected, lineno=None): diff --git a/display/ui.py b/display/ui.py index 8d83e59e6a..6af8fd6bb9 100644 --- a/display/ui.py +++ b/display/ui.py @@ -23,12 +23,13 @@ ) from _devbuild.gen.value_asdl import value_e, value_t from asdl import format as fmt +from data_lang import j8_lite +from display import enc_value from display import pretty from frontend import lexer from frontend import location from mycpp import mylib from mycpp.mylib import print_stderr, tagswitch, log -from data_lang import j8_lite import libc from typing import List, Tuple, Optional, Any, cast, TYPE_CHECKING @@ -45,7 +46,7 @@ def ValType(val): """For displaying type errors in the UI.""" # TODO: consolidate these functions - return pretty.ValType(val) + return enc_value.ValType(val) def CommandType(cmd): @@ -569,12 +570,12 @@ def PrettyPrintValue(prefix, val, f, max_width=-1): # type: (str, value_t, mylib.Writer, int) -> None """For the = keyword""" - encoder = pretty.ValueEncoder() + encoder = enc_value.ValueEncoder() encoder.SetUseStyles(f.isatty()) # TODO: pretty._Concat, etc. shouldn't be private if TypeNotPrinted(val): - mdocs = encoder.TypePrefix(pretty.ValType(val)) + mdocs = encoder.TypePrefix(enc_value.ValType(val)) mdocs.append(encoder.Value(val)) doc = pretty._Concat(mdocs) else: diff --git a/test/lint.sh b/test/lint.sh index 7558ac60ee..4ad67b3a49 100755 --- a/test/lint.sh +++ b/test/lint.sh @@ -71,7 +71,7 @@ py3-lint() { # TODO: Use devtools/repo.sh instead of this hard-coded list readonly -a CODE_DIRS=( - asdl bin builtin core data_lang doctools frontend osh tools yaks ysh + asdl bin builtin core data_lang display doctools frontend osh tools yaks ysh prebuilt pyext From 52f552c70e5d025d0258cb44fc29b2cb81cc927c Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 15:01:08 -0400 Subject: [PATCH 066/506] [cleanup] Reformat some files --- builtin/meta_osh.py | 9 +++++---- builtin/printf_osh.py | 4 ++-- devtools/format.sh | 2 +- display/enc_value.py | 3 ++- frontend/lexer_def.py | 1 - 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index 8a0dee52fa..29bf756320 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -142,9 +142,9 @@ def Run(self, cmd_val): load_path = os_path.join("stdlib", builtin_path) contents = self.loader.Get(load_path) except (IOError, OSError): - self.errfmt.Print_( - 'source failed: No builtin file %r' % load_path, - blame_loc=cmd_val.arg_locs[2]) + self.errfmt.Print_('source failed: No builtin file %r' % + load_path, + blame_loc=cmd_val.arg_locs[2]) return 2 line_reader = reader.StringLineReader(contents, self.arena) @@ -153,7 +153,8 @@ def Run(self, cmd_val): else: # 'source' respects $PATH - resolved = self.search_path.LookupOne(path_arg, exec_required=False) + resolved = self.search_path.LookupOne(path_arg, + exec_required=False) if resolved is None: resolved = path_arg diff --git a/builtin/printf_osh.py b/builtin/printf_osh.py index 20a9ded37e..7c7c441ba0 100644 --- a/builtin/printf_osh.py +++ b/builtin/printf_osh.py @@ -322,8 +322,8 @@ def _Percent(self, pr, part, varargs, locs): # the rest of the bytes. # Something like strict_arith or strict_printf # could throw an error in this case. - self.errfmt.Print_('Warning: %s' % - e.UserErrorString(), word_loc) + self.errfmt.Print_( + 'Warning: %s' % e.UserErrorString(), word_loc) small_i = ord(s[1]) d = mops.IntWiden(small_i) diff --git a/devtools/format.sh b/devtools/format.sh index 68722c1f16..20b4eef927 100755 --- a/devtools/format.sh +++ b/devtools/format.sh @@ -57,7 +57,7 @@ yapf-known() { ### yapf some files that have been normalized time yapf-files \ - {asdl,benchmarks,builtin,core,data_lang,doctools,frontend,lazylex,mycpp,mycpp/examples,osh,spec/*,yaks,ysh}/*.py \ + {asdl,benchmarks,builtin,core,data_lang,display,doctools,frontend,lazylex,mycpp,mycpp/examples,osh,spec/*,yaks,ysh}/*.py \ */NINJA_subgraph.py } diff --git a/display/enc_value.py b/display/enc_value.py index 10772535e1..bea2a66d49 100644 --- a/display/enc_value.py +++ b/display/enc_value.py @@ -11,7 +11,8 @@ from _devbuild.gen.value_asdl import value, value_e, value_t, value_str from data_lang import j8 from data_lang import j8_lite -from display.pretty import (_Break, _Concat, _Flat, _Group, _IfFlat, _Indent, _Text, _EmptyMeasure, TryUnicodeWidth) +from display.pretty import (_Break, _Concat, _Flat, _Group, _IfFlat, _Indent, + _Text, _EmptyMeasure, TryUnicodeWidth) from display import ansi from frontend import match from mycpp import mops diff --git a/frontend/lexer_def.py b/frontend/lexer_def.py index c137bab0a4..4b5ce6437d 100644 --- a/frontend/lexer_def.py +++ b/frontend/lexer_def.py @@ -503,7 +503,6 @@ def R(pat, tok_type): # Eggex. This is a LITERAL translation to \xff in ERE? So it's not \yff # It doesn't have semantics; it's just syntax. R(r'\\x[0-9a-fA-F]{2}', Id.Char_Hex), - _U_BRACED_CHAR, ] From 54a6ee26ae81599adc0e4b75565e74f366deba19 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 15:22:21 -0400 Subject: [PATCH 067/506] [soil] Use OILS_GITHUB_KEY Not the old one --- soil/maybe-merge.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/soil/maybe-merge.sh b/soil/maybe-merge.sh index cab1bb2c2e..00a6f9fe1f 100755 --- a/soil/maybe-merge.sh +++ b/soil/maybe-merge.sh @@ -23,7 +23,7 @@ fast-forward() { # local testing if test -z "$github_token"; then # set by YAML - github_token=${SOIL_GITHUB_API_TOKEN:-} + github_token=${OILS_GITHUB_KEY:-} # Local testing if test -z "$github_token"; then @@ -97,7 +97,7 @@ all-status-zero() { } soil-run() { - local github_token=${1:-} # SOIL_GITHUB_API_TOKEN + local github_token=${1:-} # OILS_GITHUB_KEY local run_id=${2:-} # $GITHUB_RUN_ID local commit_hash=${3:-} # GITHUB_SHA local to_branch=${4:-} # defaults to master From 7799c71acf1cc13c35d50f91f225cc7426923ff0 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 15:35:49 -0400 Subject: [PATCH 068/506] [soil] Fixes after accidentally deleting Github API token --- soil/README.md | 7 +++++++ soil/maybe-merge.sh | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/soil/README.md b/soil/README.md index e4ba28bd72..e841744b36 100644 --- a/soil/README.md +++ b/soil/README.md @@ -44,6 +44,13 @@ Continuous testing on many platforms. 09ab09ab.html # links to ../345/dev-minimal.wwz/ 1010abab.html +## Tokens / Authentication + +- `SOIL_GITHUB_API_TOKEN` - used by `maybe-merge` task, to use Github API to fast forward + - appears in `.github/workflows/all-builds.yml` for **only** the `maybe-merge` task +- `OILS_GITHUB_KEY` - used by all tasks to publish HTML + - - should really be called `OILS_SSH_FROM_GITHUB_ACTIONS` + ## Code Running a job starts at either: diff --git a/soil/maybe-merge.sh b/soil/maybe-merge.sh index 00a6f9fe1f..cab1bb2c2e 100755 --- a/soil/maybe-merge.sh +++ b/soil/maybe-merge.sh @@ -23,7 +23,7 @@ fast-forward() { # local testing if test -z "$github_token"; then # set by YAML - github_token=${OILS_GITHUB_KEY:-} + github_token=${SOIL_GITHUB_API_TOKEN:-} # Local testing if test -z "$github_token"; then @@ -97,7 +97,7 @@ all-status-zero() { } soil-run() { - local github_token=${1:-} # OILS_GITHUB_KEY + local github_token=${1:-} # SOIL_GITHUB_API_TOKEN local run_id=${2:-} # $GITHUB_RUN_ID local commit_hash=${3:-} # GITHUB_SHA local to_branch=${4:-} # defaults to master From b91793469d2de5941b3007e9b053399cf4a5be5a Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 16:46:24 -0400 Subject: [PATCH 069/506] [ASDL] Add dependency on display/pretty.py As a result, factor out libc Unicode from pretty.py ASDL data can have unicode strings, but for now we don't mind if the computed width is wrong. This also caused another hard-coded ASDL dependency like hnode.asdl - we need pretty.asdl too --- asdl/format.py | 19 +- build/ninja_lib.py | 1 + build/ninja_lib_test.py | 16 +- core/comp_ui.py | 6 +- display/{enc_value.py => pp_value.py} | 115 +++--- display/pretty.py | 26 +- display/pretty_test.py | 4 +- display/ui.py | 10 +- mycpp/examples.sh | 11 +- mycpp/examples/parse.translate.txt | 1 + mycpp/examples/parse.typecheck.txt | 2 + mycpp/examples/parse_preamble.h | 5 + pea/oils-typecheck.txt | 1 + prebuilt/NINJA_subgraph.py | 4 + prebuilt/asdl/runtime.mycpp.cc | 327 +++++++++++++++-- prebuilt/asdl/runtime.mycpp.h | 6 + prebuilt/core/error.mycpp.h | 1 + prebuilt/frontend/args.mycpp.cc | 499 ++++++++++++++++++++------ prebuilt/frontend/args.mycpp.h | 4 + prebuilt/translate.sh | 15 +- yaks/preamble.h | 2 + 21 files changed, 838 insertions(+), 237 deletions(-) rename display/{enc_value.py => pp_value.py} (82%) diff --git a/asdl/format.py b/asdl/format.py index 15545866f7..bb6f915e2b 100644 --- a/asdl/format.py +++ b/asdl/format.py @@ -15,8 +15,9 @@ from _devbuild.gen.hnode_asdl import (hnode, hnode_e, hnode_t, color_e, color_t) -from display import ansi from data_lang import j8_lite +from display import ansi +from display import pretty from pylib import cgi from mycpp import mylib @@ -516,3 +517,19 @@ def PrintTree(node, f): # type: (hnode_t, ColorOutput) -> None pp = _PrettyPrinter(100) # max_col pp.PrintNode(node, f, 0) # indent + + +def PrintTree2(node, f): + # type: (hnode_t, ColorOutput) -> None + """ + Make sure dependencies aren't a problem + + TODO: asdl/pp_hnode.py, which is like display/pp_value.py + """ + doc = pretty.AsciiText('foo') + printer = pretty.PrettyPrinter(20) + + buf = mylib.BufWriter() + printer.PrintDoc(doc, buf) + f.write(buf.getvalue()) + f.write('\n') diff --git a/build/ninja_lib.py b/build/ninja_lib.py index 7e452bdc34..9a5b042d39 100644 --- a/build/ninja_lib.py +++ b/build/ninja_lib.py @@ -430,6 +430,7 @@ def asdl_library(self, asdl_path, deps = None, # SYSTEM header, _gen/asdl/hnode.asdl.h deps.append('//asdl/hnode.asdl') + deps.append('//display/pretty.asdl') # to create _gen/mycpp/examples/expr.asdl.h prefix = '_gen/%s' % asdl_path diff --git a/build/ninja_lib_test.py b/build/ninja_lib_test.py index e0daf235f5..292e0a69bd 100755 --- a/build/ninja_lib_test.py +++ b/build/ninja_lib_test.py @@ -155,6 +155,7 @@ def test_cc_binary_to_asdl(self): n, ru = self._Rules() ru.asdl_library('asdl/hnode.asdl', pretty_print_methods = False) # REQUIRED + ru.asdl_library('display/pretty.asdl') ru.asdl_library('mycpp/examples/expr.asdl') @@ -170,6 +171,8 @@ def test_cc_binary_to_asdl(self): self.assertEqual([ 'asdl-cpp', 'asdl-cpp', + 'asdl-cpp', + 'compile_one', 'compile_one', 'compile_one', 'link'], @@ -180,6 +183,7 @@ def test_cc_binary_to_asdl(self): # Important implicit dependencies on generated headers! self.assertEqual([ '_gen/asdl/hnode.asdl.h', + '_gen/display/pretty.asdl.h', '_gen/mycpp/examples/expr.asdl.h', ], compile_parse.implicit) @@ -188,6 +192,7 @@ def test_cc_binary_to_asdl(self): self.assertEqual([ '_build/obj/cxx-dbg/_gen/mycpp/examples/parse.mycpp.o', + '_build/obj/cxx-dbg/_gen/display/pretty.asdl.o', '_build/obj/cxx-dbg/_gen/mycpp/examples/expr.asdl.o', ], last.inputs) @@ -196,6 +201,7 @@ def test_asdl_to_asdl(self): n, ru = self._Rules() ru.asdl_library('asdl/hnode.asdl', pretty_print_methods = False) # REQUIRED + ru.asdl_library('display/pretty.asdl') ru.asdl_library('asdl/examples/demo_lib.asdl') @@ -205,7 +211,7 @@ def test_asdl_to_asdl(self): deps = ['//asdl/examples/demo_lib.asdl']) actions = [call.rule for call in n.build_calls] - self.assertEqual(['asdl-cpp', 'asdl-cpp', 'asdl-cpp'], actions) + self.assertEqual(['asdl-cpp', 'asdl-cpp', 'asdl-cpp', 'asdl-cpp'], actions) ru.cc_binary( 'asdl/gen_cpp_test.cc', @@ -215,8 +221,10 @@ def test_asdl_to_asdl(self): ru.WriteRules() actions = [call.rule for call in n.build_calls] + print(actions) self.assertEqual([ - 'asdl-cpp', 'asdl-cpp', 'asdl-cpp', + 'asdl-cpp', 'asdl-cpp', 'asdl-cpp', 'asdl-cpp', + 'compile_one', 'compile_one', # compile demo_lib 'compile_one', # compile typed_demo 'compile_one', # compile gen_cpp_test @@ -231,7 +239,8 @@ def test_asdl_to_asdl(self): # the header demo_lib.asdl.h self.assertEqual( [ '_gen/asdl/examples/demo_lib.asdl.h', - '_gen/asdl/hnode.asdl.h' ], + '_gen/asdl/hnode.asdl.h', + '_gen/display/pretty.asdl.h' ], sorted(c.implicit)) c = CallFor(n, '_build/obj/cxx-dbg/asdl/gen_cpp_test.o') @@ -241,6 +250,7 @@ def test_asdl_to_asdl(self): [ '_gen/asdl/examples/demo_lib.asdl.h', '_gen/asdl/examples/typed_demo.asdl.h', '_gen/asdl/hnode.asdl.h', + '_gen/display/pretty.asdl.h', ], sorted(c.implicit)) diff --git a/core/comp_ui.py b/core/comp_ui.py index 4f9ed44d2f..8db9dd0c63 100644 --- a/core/comp_ui.py +++ b/core/comp_ui.py @@ -1,9 +1,9 @@ """comp_ui.py.""" from __future__ import print_function -from display import ansi from core import completion -from display import pretty +from display import ansi +from display import pp_value import libc from mycpp import mylib @@ -48,7 +48,7 @@ def _PromptLen(prompt_str): # mycpp: rewrite of += display_str = display_str + c last_line = display_str.split('\n')[-1] - return pretty.TryUnicodeWidth(last_line) + return pp_value.TryUnicodeWidth(last_line) class PromptState(object): diff --git a/display/enc_value.py b/display/pp_value.py similarity index 82% rename from display/enc_value.py rename to display/pp_value.py index bea2a66d49..edcf5823d3 100644 --- a/display/enc_value.py +++ b/display/pp_value.py @@ -7,18 +7,20 @@ import math -from _devbuild.gen.pretty_asdl import (doc, MeasuredDoc) +from _devbuild.gen.pretty_asdl import (doc, Measure, MeasuredDoc) from _devbuild.gen.value_asdl import value, value_e, value_t, value_str from data_lang import j8 from data_lang import j8_lite from display.pretty import (_Break, _Concat, _Flat, _Group, _IfFlat, _Indent, - _Text, _EmptyMeasure, TryUnicodeWidth) + _EmptyMeasure) from display import ansi from frontend import match from mycpp import mops from mycpp.mylib import log, tagswitch, iteritems from typing import cast, List, Dict +import libc + _ = log @@ -43,6 +45,31 @@ def _FloatString(fl): return s +# +# Unicode Helpers +# + + +def TryUnicodeWidth(s): + # type: (str) -> int + try: + width = libc.wcswidth(s) + except UnicodeError: + # e.g. en_US.UTF-8 locale missing, just return the number of bytes + width = len(s) + + if width == -1: # non-printable wide char + return len(s) + + return width + + +def UText(string): + # type: (str) -> MeasuredDoc + """Print `string` (which must not contain a newline).""" + return MeasuredDoc(doc.Text(string), Measure(TryUnicodeWidth(string), -1)) + + class ValueEncoder: """Converts Oils values into `doc`s, which can then be pretty printed.""" @@ -88,7 +115,7 @@ def SetMaxTabularWidth(self, max_tabular_width): def TypePrefix(self, type_str): # type: (str) -> List[MeasuredDoc] """Return docs for type string "(List)", which may break afterward.""" - type_name = self._Styled(self.type_style, _Text(type_str)) + type_name = self._Styled(self.type_style, UText(type_str)) n = len(type_str) # Our maximum string is "Float" @@ -97,7 +124,7 @@ def TypePrefix(self, type_str): # Start printing in column 8. Adjust to 6 because () takes 2 spaces. spaces = ' ' * (6 - n) - mdocs = [_Text("("), type_name, _Text(")"), _Break(spaces)] + mdocs = [UText("("), type_name, UText(")"), _Break(spaces)] return mdocs def Value(self, val): @@ -131,10 +158,10 @@ def _Surrounded(self, open, mdoc, close): """ return _Group( _Concat([ - _Text(open), + UText(open), _Indent(self.indent, _Concat([_Break(""), mdoc])), _Break(""), - _Text(close) + UText(close) ])) def _SurroundedAndPrefixed(self, open, prefix, sep, mdoc, close): @@ -152,10 +179,10 @@ def _SurroundedAndPrefixed(self, open, prefix, sep, mdoc, close): """ return _Group( _Concat([ - _Text(open), prefix, + UText(open), prefix, _Indent(self.indent, _Concat([_Break(sep), mdoc])), _Break(""), - _Text(close) + UText(close) ])) def _Join(self, items, sep, space): @@ -174,7 +201,7 @@ def _Join(self, items, sep, space): seq = [] # type: List[MeasuredDoc] for i, item in enumerate(items): if i != 0: - seq.append(_Text(sep)) + seq.append(UText(sep)) seq.append(_Break(space)) seq.append(item) return _Concat(seq) @@ -224,13 +251,13 @@ def _Tabular(self, items, sep): # style 3. if len(items) == 0: - return _Text("") + return UText("") max_flat_len = 0 seq = [] # type: List[MeasuredDoc] for i, item in enumerate(items): if i != 0: - seq.append(_Text(sep)) + seq.append(UText(sep)) seq.append(_Break(" ")) seq.append(item) max_flat_len = max(max_flat_len, item.measure.flat) @@ -243,7 +270,7 @@ def _Tabular(self, items, sep): tabular_seq.append(_Flat(item)) if i != len(items) - 1: padding = max_flat_len - item.measure.flat + 1 - tabular_seq.append(_Text(sep)) + tabular_seq.append(UText(sep)) tabular_seq.append(_Group(_Break(" " * padding))) tabular = _Concat(tabular_seq) return _Group(_IfFlat(non_tabular, tabular)) @@ -260,7 +287,7 @@ def _DictKey(self, s): else: # TODO: remove this dead branch after fixing tests encoded = j8_lite.EncodeString(s) - return _Text(encoded) + return UText(encoded) def _StringLiteral(self, s): # type: (str) -> MeasuredDoc @@ -270,7 +297,7 @@ def _StringLiteral(self, s): else: # TODO: remove this dead branch after fixing tests encoded = j8_lite.EncodeString(s) - return self._Styled(self.string_style, _Text(encoded)) + return self._Styled(self.string_style, UText(encoded)) def _BashStringLiteral(self, s): # type: (str) -> MeasuredDoc @@ -288,37 +315,37 @@ def _BashStringLiteral(self, s): # (BashAssoc) (BashAssoc ['k']=$'\\') encoded = j8_lite.ShellEncode(s) - return self._Styled(self.string_style, _Text(encoded)) + return self._Styled(self.string_style, UText(encoded)) def _YshList(self, vlist): # type: (value.List) -> MeasuredDoc """Print a string literal.""" if len(vlist.items) == 0: - return _Text("[]") + return UText("[]") mdocs = [self._Value(item) for item in vlist.items] return self._Surrounded("[", self._Tabular(mdocs, ","), "]") def _YshDict(self, vdict): # type: (value.Dict) -> MeasuredDoc if len(vdict.d) == 0: - return _Text("{}") + return UText("{}") mdocs = [] # type: List[MeasuredDoc] for k, v in iteritems(vdict.d): mdocs.append( _Concat([self._DictKey(k), - _Text(": "), + UText(": "), self._Value(v)])) return self._Surrounded("{", self._Join(mdocs, ",", " "), "}") def _BashArray(self, varray): # type: (value.BashArray) -> MeasuredDoc - type_name = self._Styled(self.type_style, _Text("BashArray")) + type_name = self._Styled(self.type_style, UText("BashArray")) if len(varray.strs) == 0: - return _Concat([_Text("("), type_name, _Text(")")]) + return _Concat([UText("("), type_name, UText(")")]) mdocs = [] # type: List[MeasuredDoc] for s in varray.strs: if s is None: - mdocs.append(_Text("null")) + mdocs.append(UText("null")) else: mdocs.append(self._BashStringLiteral(s)) return self._SurroundedAndPrefixed("(", type_name, " ", @@ -326,16 +353,16 @@ def _BashArray(self, varray): def _BashAssoc(self, vassoc): # type: (value.BashAssoc) -> MeasuredDoc - type_name = self._Styled(self.type_style, _Text("BashAssoc")) + type_name = self._Styled(self.type_style, UText("BashAssoc")) if len(vassoc.d) == 0: - return _Concat([_Text("("), type_name, _Text(")")]) + return _Concat([UText("("), type_name, UText(")")]) mdocs = [] # type: List[MeasuredDoc] for k2, v2 in iteritems(vassoc.d): mdocs.append( _Concat([ - _Text("["), + UText("["), self._BashStringLiteral(k2), - _Text("]="), + UText("]="), self._BashStringLiteral(v2) ])) return self._SurroundedAndPrefixed("(", type_name, " ", @@ -343,16 +370,16 @@ def _BashAssoc(self, vassoc): def _SparseArray(self, val): # type: (value.SparseArray) -> MeasuredDoc - type_name = self._Styled(self.type_style, _Text("SparseArray")) + type_name = self._Styled(self.type_style, UText("SparseArray")) if len(val.d) == 0: - return _Concat([_Text("("), type_name, _Text(")")]) + return _Concat([UText("("), type_name, UText(")")]) mdocs = [] # type: List[MeasuredDoc] for k2, v2 in iteritems(val.d): mdocs.append( _Concat([ - _Text("["), - self._Styled(self.int_style, _Text(mops.ToStr(k2))), - _Text("]="), + UText("["), + self._Styled(self.int_style, UText(mops.ToStr(k2))), + UText("]="), self._BashStringLiteral(v2) ])) return self._SurroundedAndPrefixed("(", type_name, " ", @@ -363,20 +390,20 @@ def _Value(self, val): with tagswitch(val) as case: if case(value_e.Null): - return self._Styled(self.null_style, _Text("null")) + return self._Styled(self.null_style, UText("null")) elif case(value_e.Bool): b = cast(value.Bool, val).b return self._Styled(self.bool_style, - _Text("true" if b else "false")) + UText("true" if b else "false")) elif case(value_e.Int): i = cast(value.Int, val).i - return self._Styled(self.int_style, _Text(mops.ToStr(i))) + return self._Styled(self.int_style, UText(mops.ToStr(i))) elif case(value_e.Float): f = cast(value.Float, val).f - return self._Styled(self.float_style, _Text(_FloatString(f))) + return self._Styled(self.float_style, UText(_FloatString(f))) elif case(value_e.Str): s = cast(value.Str, val).s @@ -384,8 +411,8 @@ def _Value(self, val): elif case(value_e.Range): r = cast(value.Range, val) - type_name = self._Styled(self.type_style, _Text(ValType(r))) - mdocs = [_Text(str(r.lower)), _Text(".."), _Text(str(r.upper))] + type_name = self._Styled(self.type_style, UText(ValType(r))) + mdocs = [UText(str(r.lower)), UText(".."), UText(str(r.upper))] return self._SurroundedAndPrefixed("(", type_name, " ", self._Join(mdocs, "", " "), ")") @@ -395,9 +422,9 @@ def _Value(self, val): heap_id = j8.HeapValueId(vlist) if self.visiting.get(heap_id, False): return _Concat([ - _Text("["), - self._Styled(self.cycle_style, _Text("...")), - _Text("]") + UText("["), + self._Styled(self.cycle_style, UText("...")), + UText("]") ]) else: self.visiting[heap_id] = True @@ -410,9 +437,9 @@ def _Value(self, val): heap_id = j8.HeapValueId(vdict) if self.visiting.get(heap_id, False): return _Concat([ - _Text("{"), - self._Styled(self.cycle_style, _Text("...")), - _Text("}") + UText("{"), + self._Styled(self.cycle_style, UText("...")), + UText("}") ]) else: self.visiting[heap_id] = True @@ -433,9 +460,9 @@ def _Value(self, val): return self._BashAssoc(vassoc) else: - type_name = self._Styled(self.type_style, _Text(ValType(val))) + type_name = self._Styled(self.type_style, UText(ValType(val))) id_str = j8.ValueIdString(val) - return _Concat([_Text("<"), type_name, _Text(id_str + ">")]) + return _Concat([UText("<"), type_name, UText(id_str + ">")]) # vim: sw=4 diff --git a/display/pretty.py b/display/pretty.py index 74f1bbebc3..a6aedbee65 100644 --- a/display/pretty.py +++ b/display/pretty.py @@ -103,7 +103,6 @@ from _devbuild.gen.pretty_asdl import doc, doc_e, DocFragment, Measure, MeasuredDoc from mycpp.mylib import log, tagswitch, BufWriter from typing import cast, List -import libc _ = log @@ -112,20 +111,6 @@ ################ -def TryUnicodeWidth(s): - # type: (str) -> int - try: - width = libc.wcswidth(s) - except UnicodeError: - # e.g. en_US.UTF-8 locale missing, just return the number of bytes - width = len(s) - - if width == -1: # non-printable wide char - return len(s) - - return width - - def _EmptyMeasure(): # type: () -> Measure """The measure of an empty doc.""" @@ -167,16 +152,19 @@ def _SuffixLen(measure): #################### -def _Text(string): +def AsciiText(string): # type: (str) -> MeasuredDoc """Print `string` (which must not contain a newline).""" - return MeasuredDoc(doc.Text(string), Measure(TryUnicodeWidth(string), -1)) + return MeasuredDoc(doc.Text(string), Measure(len(string), -1)) def _Break(string): # type: (str) -> MeasuredDoc - """If in `flat` mode, print `string`, otherwise print `\n`.""" - return MeasuredDoc(doc.Break(string), Measure(TryUnicodeWidth(string), 0)) + """If in `flat` mode, print `string`, otherwise print `\n`. + + Note: Doesn't try to compute Unicode width, since we control these strings. + """ + return MeasuredDoc(doc.Break(string), Measure(len(string), 0)) def _Indent(indent, mdoc): diff --git a/display/pretty_test.py b/display/pretty_test.py index 9a643dca37..c64ae7190d 100755 --- a/display/pretty_test.py +++ b/display/pretty_test.py @@ -6,7 +6,7 @@ from display import ansi from display import pretty # module under test -from display import enc_value +from display import pp_value from display import ui from data_lang import j8 from mycpp import mylib @@ -53,7 +53,7 @@ class PrettyTest(unittest.TestCase): def setUp(self): # Use settings that make testing easier. - self.encoder = enc_value.ValueEncoder() + self.encoder = pp_value.ValueEncoder() self.encoder.SetUseStyles(False) def assertPretty(self, width, value_str, expected, lineno=None): diff --git a/display/ui.py b/display/ui.py index 6af8fd6bb9..9a91ed1346 100644 --- a/display/ui.py +++ b/display/ui.py @@ -24,7 +24,7 @@ from _devbuild.gen.value_asdl import value_e, value_t from asdl import format as fmt from data_lang import j8_lite -from display import enc_value +from display import pp_value from display import pretty from frontend import lexer from frontend import location @@ -46,7 +46,7 @@ def ValType(val): """For displaying type errors in the UI.""" # TODO: consolidate these functions - return enc_value.ValType(val) + return pp_value.ValType(val) def CommandType(cmd): @@ -570,12 +570,12 @@ def PrettyPrintValue(prefix, val, f, max_width=-1): # type: (str, value_t, mylib.Writer, int) -> None """For the = keyword""" - encoder = enc_value.ValueEncoder() + encoder = pp_value.ValueEncoder() encoder.SetUseStyles(f.isatty()) # TODO: pretty._Concat, etc. shouldn't be private if TypeNotPrinted(val): - mdocs = encoder.TypePrefix(enc_value.ValType(val)) + mdocs = encoder.TypePrefix(pp_value.ValType(val)) mdocs.append(encoder.Value(val)) doc = pretty._Concat(mdocs) else: @@ -586,7 +586,7 @@ def PrettyPrintValue(prefix, val, f, max_width=-1): # inner = pretty._Concat([pretty._Break(""), doc]) doc = pretty._Concat([ - pretty._Text(prefix), + pretty.AsciiText(prefix), #pretty._Break(""), pretty._Indent(4, doc) ]) diff --git a/mycpp/examples.sh b/mycpp/examples.sh index 8a5b73f7e3..d13de05ea9 100644 --- a/mycpp/examples.sh +++ b/mycpp/examples.sh @@ -93,17 +93,8 @@ translate-parse() { # Need this otherwise we get type errors codegen-parse - local snippet=' - -#include "expr_asdl.h" - -Str* repr(void* obj) { - return StrFromC("TODO: repr()"); -} - -' # TODO: This is similar to prebuilt/translate.sh ASDL_FILES - translate-ordered parse "$snippet" \ + translate-ordered parse '' \ $REPO_ROOT/pylib/cgi.py \ $REPO_ROOT/asdl/runtime.py \ $REPO_ROOT/asdl/format.py \ diff --git a/mycpp/examples/parse.translate.txt b/mycpp/examples/parse.translate.txt index 4d762274d3..2b861ccc12 100644 --- a/mycpp/examples/parse.translate.txt +++ b/mycpp/examples/parse.translate.txt @@ -2,5 +2,6 @@ asdl/format.py asdl/runtime.py data_lang/j8_lite.py display/ansi.py +display/pretty.py mycpp/examples/parse.py pylib/cgi.py diff --git a/mycpp/examples/parse.typecheck.txt b/mycpp/examples/parse.typecheck.txt index 9c81a7aea1..149d3b6f9a 100644 --- a/mycpp/examples/parse.typecheck.txt +++ b/mycpp/examples/parse.typecheck.txt @@ -1,10 +1,12 @@ _devbuild/gen/expr_asdl.py _devbuild/gen/hnode_asdl.py +_devbuild/gen/pretty_asdl.py asdl/format.py asdl/pybase.py asdl/runtime.py data_lang/j8_lite.py display/ansi.py +display/pretty.py mycpp/examples/parse.py mycpp/mops.py pylib/cgi.py diff --git a/mycpp/examples/parse_preamble.h b/mycpp/examples/parse_preamble.h index eb2b13d62b..bbb00dfd5f 100644 --- a/mycpp/examples/parse_preamble.h +++ b/mycpp/examples/parse_preamble.h @@ -1,2 +1,7 @@ +// like cpp/preamble.h + +#include "_gen/display/pretty.asdl.h" #include "_gen/mycpp/examples/expr.asdl.h" #include "cpp/data_lang.h" + +using pretty_asdl::doc; diff --git a/pea/oils-typecheck.txt b/pea/oils-typecheck.txt index f640aa493b..f5f7ec9f8c 100644 --- a/pea/oils-typecheck.txt +++ b/pea/oils-typecheck.txt @@ -61,6 +61,7 @@ data_lang/j8.py data_lang/j8_lite.py data_lang/pyj8.py display/ansi.py +display/pp_value.py display/pretty.py display/ui.py frontend/args.py diff --git a/prebuilt/NINJA_subgraph.py b/prebuilt/NINJA_subgraph.py index 04dc7a4bcc..7d93a9dc78 100644 --- a/prebuilt/NINJA_subgraph.py +++ b/prebuilt/NINJA_subgraph.py @@ -48,9 +48,11 @@ def NinjaGraph(ru): ru.cc_library( '//prebuilt/asdl/runtime.mycpp', srcs=['prebuilt/asdl/runtime.mycpp.cc'], + # TODO: make a common library for these deps? deps=[ '//asdl/hnode.asdl', '//cpp/data_lang', # for fastfunc + '//display/pretty.asdl', ]) ru.cc_library( @@ -61,6 +63,7 @@ def NinjaGraph(ru): '//core/value.asdl', '//frontend/syntax.asdl', '//cpp/data_lang', # for fastfunc + '//display/pretty.asdl', ]) ru.cc_library( @@ -72,4 +75,5 @@ def NinjaGraph(ru): '//frontend/syntax.asdl', '//cpp/data_lang', # for fastfunc '//cpp/frontend_flag_spec', + '//display/pretty.asdl', ]) diff --git a/prebuilt/asdl/runtime.mycpp.cc b/prebuilt/asdl/runtime.mycpp.cc index 8f2be2a3d6..722fadca59 100644 --- a/prebuilt/asdl/runtime.mycpp.cc +++ b/prebuilt/asdl/runtime.mycpp.cc @@ -46,29 +46,38 @@ GLOBAL_STR(str37, "["); GLOBAL_STR(str38, " "); GLOBAL_STR(str39, "]"); GLOBAL_STR(str40, "...0x%s"); -GLOBAL_STR(str41, "\u001b[0;0m"); -GLOBAL_STR(str42, "\u001b[1m"); -GLOBAL_STR(str43, "\u001b[4m"); -GLOBAL_STR(str44, "\u001b[7m"); -GLOBAL_STR(str45, "\u001b[31m"); -GLOBAL_STR(str46, "\u001b[32m"); -GLOBAL_STR(str47, "\u001b[33m"); -GLOBAL_STR(str48, "\u001b[34m"); -GLOBAL_STR(str49, "\u001b[35m"); -GLOBAL_STR(str50, "\u001b[36m"); -GLOBAL_STR(str51, "\u001b[37m"); -GLOBAL_STR(str52, "&"); -GLOBAL_STR(str53, "&"); -GLOBAL_STR(str54, "<"); -GLOBAL_STR(str55, "<"); -GLOBAL_STR(str56, ">"); -GLOBAL_STR(str57, ">"); +GLOBAL_STR(str41, "foo"); +GLOBAL_STR(str42, "\n"); +GLOBAL_STR(str43, "\u001b[0;0m"); +GLOBAL_STR(str44, "\u001b[1m"); +GLOBAL_STR(str45, "\u001b[4m"); +GLOBAL_STR(str46, "\u001b[7m"); +GLOBAL_STR(str47, "\u001b[31m"); +GLOBAL_STR(str48, "\u001b[32m"); +GLOBAL_STR(str49, "\u001b[33m"); +GLOBAL_STR(str50, "\u001b[34m"); +GLOBAL_STR(str51, "\u001b[35m"); +GLOBAL_STR(str52, "\u001b[36m"); +GLOBAL_STR(str53, "\u001b[37m"); +GLOBAL_STR(str54, "\n"); +GLOBAL_STR(str55, "&"); +GLOBAL_STR(str56, "&"); +GLOBAL_STR(str57, "<"); +GLOBAL_STR(str58, "<"); +GLOBAL_STR(str59, ">"); +GLOBAL_STR(str60, ">"); namespace ansi { // forward declare } // forward declare namespace ansi +namespace pretty { // forward declare + + class PrettyPrinter; + +} // forward declare namespace pretty + namespace cgi { // forward declare @@ -95,6 +104,36 @@ extern BigStr* WHITE; } // declare namespace ansi +namespace pretty { // declare + +pretty_asdl::Measure* _EmptyMeasure(); +pretty_asdl::Measure* _FlattenMeasure(pretty_asdl::Measure* measure); +pretty_asdl::Measure* _ConcatMeasure(pretty_asdl::Measure* m1, pretty_asdl::Measure* m2); +int _SuffixLen(pretty_asdl::Measure* measure); +pretty_asdl::MeasuredDoc* AsciiText(BigStr* string); +pretty_asdl::MeasuredDoc* _Break(BigStr* string); +pretty_asdl::MeasuredDoc* _Indent(int indent, pretty_asdl::MeasuredDoc* mdoc); +pretty_asdl::MeasuredDoc* _Concat(List* mdocs); +pretty_asdl::MeasuredDoc* _Group(pretty_asdl::MeasuredDoc* mdoc); +pretty_asdl::MeasuredDoc* _IfFlat(pretty_asdl::MeasuredDoc* flat_mdoc, pretty_asdl::MeasuredDoc* nonflat_mdoc); +pretty_asdl::MeasuredDoc* _Flat(pretty_asdl::MeasuredDoc* mdoc); +class PrettyPrinter { + public: + PrettyPrinter(int max_width); + bool _Fits(int prefix_len, doc::Group* group, pretty_asdl::Measure* suffix_measure); + void PrintDoc(pretty_asdl::MeasuredDoc* document, mylib::BufWriter* buf); + int max_width; + + static constexpr ObjHeader obj_header() { + return ObjHeader::ClassScanned(0, sizeof(PrettyPrinter)); + } + + DISALLOW_COPY_AND_ASSIGN(PrettyPrinter) +}; + + +} // declare namespace pretty + namespace cgi { // declare BigStr* escape(BigStr* s); @@ -704,33 +743,259 @@ void PrintTree(hnode_asdl::hnode_t* node, format::ColorOutput* f) { pp->PrintNode(node, f, 0); } +void PrintTree2(hnode_asdl::hnode_t* node, format::ColorOutput* f) { + pretty_asdl::MeasuredDoc* doc = nullptr; + pretty::PrettyPrinter* printer = nullptr; + mylib::BufWriter* buf = nullptr; + StackRoot _root0(&node); + StackRoot _root1(&f); + StackRoot _root2(&doc); + StackRoot _root3(&printer); + StackRoot _root4(&buf); + + doc = pretty::AsciiText(str41); + printer = Alloc(20); + buf = Alloc(); + printer->PrintDoc(doc, buf); + f->write(buf->getvalue()); + f->write(str42); +} + } // define namespace format namespace ansi { // define -BigStr* RESET = str41; -BigStr* BOLD = str42; -BigStr* UNDERLINE = str43; -BigStr* REVERSE = str44; -BigStr* RED = str45; -BigStr* GREEN = str46; -BigStr* YELLOW = str47; -BigStr* BLUE = str48; -BigStr* MAGENTA = str49; -BigStr* CYAN = str50; -BigStr* WHITE = str51; +BigStr* RESET = str43; +BigStr* BOLD = str44; +BigStr* UNDERLINE = str45; +BigStr* REVERSE = str46; +BigStr* RED = str47; +BigStr* GREEN = str48; +BigStr* YELLOW = str49; +BigStr* BLUE = str50; +BigStr* MAGENTA = str51; +BigStr* CYAN = str52; +BigStr* WHITE = str53; } // define namespace ansi +namespace pretty { // define + +using pretty_asdl::doc; +using pretty_asdl::doc_e; +using pretty_asdl::DocFragment; +using pretty_asdl::Measure; +using pretty_asdl::MeasuredDoc; +using mylib::BufWriter; + +pretty_asdl::Measure* _EmptyMeasure() { + return Alloc(0, -1); +} + +pretty_asdl::Measure* _FlattenMeasure(pretty_asdl::Measure* measure) { + StackRoot _root0(&measure); + + return Alloc(measure->flat, -1); +} + +pretty_asdl::Measure* _ConcatMeasure(pretty_asdl::Measure* m1, pretty_asdl::Measure* m2) { + StackRoot _root0(&m1); + StackRoot _root1(&m2); + + if (m1->nonflat != -1) { + return Alloc((m1->flat + m2->flat), m1->nonflat); + } + else { + if (m2->nonflat != -1) { + return Alloc((m1->flat + m2->flat), (m1->flat + m2->nonflat)); + } + else { + return Alloc((m1->flat + m2->flat), -1); + } + } +} + +int _SuffixLen(pretty_asdl::Measure* measure) { + StackRoot _root0(&measure); + + if (measure->nonflat != -1) { + return measure->nonflat; + } + else { + return measure->flat; + } +} + +pretty_asdl::MeasuredDoc* AsciiText(BigStr* string) { + StackRoot _root0(&string); + + return Alloc(Alloc(string), Alloc(len(string), -1)); +} + +pretty_asdl::MeasuredDoc* _Break(BigStr* string) { + StackRoot _root0(&string); + + return Alloc(Alloc(string), Alloc(len(string), 0)); +} + +pretty_asdl::MeasuredDoc* _Indent(int indent, pretty_asdl::MeasuredDoc* mdoc) { + StackRoot _root0(&mdoc); + + return Alloc(Alloc(indent, mdoc), mdoc->measure); +} + +pretty_asdl::MeasuredDoc* _Concat(List* mdocs) { + pretty_asdl::Measure* measure = nullptr; + StackRoot _root0(&mdocs); + StackRoot _root1(&measure); + + measure = _EmptyMeasure(); + for (ListIter it(mdocs); !it.Done(); it.Next()) { + pretty_asdl::MeasuredDoc* mdoc = it.Value(); + StackRoot _for(&mdoc ); + measure = _ConcatMeasure(measure, mdoc->measure); + } + return Alloc(Alloc(mdocs), measure); +} + +pretty_asdl::MeasuredDoc* _Group(pretty_asdl::MeasuredDoc* mdoc) { + StackRoot _root0(&mdoc); + + return Alloc(Alloc(mdoc), mdoc->measure); +} + +pretty_asdl::MeasuredDoc* _IfFlat(pretty_asdl::MeasuredDoc* flat_mdoc, pretty_asdl::MeasuredDoc* nonflat_mdoc) { + StackRoot _root0(&flat_mdoc); + StackRoot _root1(&nonflat_mdoc); + + return Alloc(Alloc(flat_mdoc, nonflat_mdoc), Alloc(flat_mdoc->measure->flat, nonflat_mdoc->measure->nonflat)); +} + +pretty_asdl::MeasuredDoc* _Flat(pretty_asdl::MeasuredDoc* mdoc) { + StackRoot _root0(&mdoc); + + return Alloc(Alloc(mdoc), _FlattenMeasure(mdoc->measure)); +} + +PrettyPrinter::PrettyPrinter(int max_width) { + this->max_width = max_width; +} + +bool PrettyPrinter::_Fits(int prefix_len, doc::Group* group, pretty_asdl::Measure* suffix_measure) { + pretty_asdl::Measure* measure = nullptr; + StackRoot _root0(&group); + StackRoot _root1(&suffix_measure); + StackRoot _root2(&measure); + + measure = _ConcatMeasure(_FlattenMeasure(group->mdoc->measure), suffix_measure); + return (prefix_len + _SuffixLen(measure)) <= this->max_width; +} + +void PrettyPrinter::PrintDoc(pretty_asdl::MeasuredDoc* document, mylib::BufWriter* buf) { + int prefix_len; + List* fragments = nullptr; + pretty_asdl::DocFragment* frag = nullptr; + doc::Text* text = nullptr; + BigStr* break_str = nullptr; + doc::Indent* indented = nullptr; + doc::Concat* concat = nullptr; + pretty_asdl::Measure* measure = nullptr; + doc::Group* group = nullptr; + bool flat; + doc::IfFlat* if_flat = nullptr; + pretty_asdl::MeasuredDoc* subdoc = nullptr; + doc::Flat* flat_doc = nullptr; + StackRoot _root0(&document); + StackRoot _root1(&buf); + StackRoot _root2(&fragments); + StackRoot _root3(&frag); + StackRoot _root4(&text); + StackRoot _root5(&break_str); + StackRoot _root6(&indented); + StackRoot _root7(&concat); + StackRoot _root8(&measure); + StackRoot _root9(&group); + StackRoot _root10(&if_flat); + StackRoot _root11(&subdoc); + StackRoot _root12(&flat_doc); + + prefix_len = 0; + fragments = NewList(std::initializer_list{Alloc(_Group(document), 0, false, _EmptyMeasure())}); + while (len(fragments) > 0) { + frag = fragments->pop(); + switch (frag->mdoc->doc->tag()) { + case doc_e::Text: { + text = static_cast(frag->mdoc->doc); + buf->write(text->string); + prefix_len += frag->mdoc->measure->flat; + } + break; + case doc_e::Break: { + if (frag->is_flat) { + break_str = static_cast(frag->mdoc->doc)->string; + buf->write(break_str); + prefix_len += frag->mdoc->measure->flat; + } + else { + buf->write(str54); + buf->write_spaces(frag->indent); + prefix_len = frag->indent; + } + } + break; + case doc_e::Indent: { + indented = static_cast(frag->mdoc->doc); + fragments->append(Alloc(indented->mdoc, (frag->indent + indented->indent), frag->is_flat, frag->measure)); + } + break; + case doc_e::Concat: { + concat = static_cast(frag->mdoc->doc); + measure = frag->measure; + for (ReverseListIter it(concat->mdocs); !it.Done(); it.Next()) { + pretty_asdl::MeasuredDoc* mdoc = it.Value(); + StackRoot _for(&mdoc ); + fragments->append(Alloc(mdoc, frag->indent, frag->is_flat, measure)); + measure = _ConcatMeasure(mdoc->measure, measure); + } + } + break; + case doc_e::Group: { + group = static_cast(frag->mdoc->doc); + flat = this->_Fits(prefix_len, group, frag->measure); + fragments->append(Alloc(group->mdoc, frag->indent, flat, frag->measure)); + } + break; + case doc_e::IfFlat: { + if_flat = static_cast(frag->mdoc->doc); + if (frag->is_flat) { + subdoc = if_flat->flat_mdoc; + } + else { + subdoc = if_flat->nonflat_mdoc; + } + fragments->append(Alloc(subdoc, frag->indent, frag->is_flat, frag->measure)); + } + break; + case doc_e::Flat: { + flat_doc = static_cast(frag->mdoc->doc); + fragments->append(Alloc(flat_doc->mdoc, frag->indent, true, frag->measure)); + } + break; + } + } +} + +} // define namespace pretty + namespace cgi { // define BigStr* escape(BigStr* s) { StackRoot _root0(&s); - s = s->replace(str52, str53); - s = s->replace(str54, str55); - s = s->replace(str56, str57); + s = s->replace(str55, str56); + s = s->replace(str57, str58); + s = s->replace(str59, str60); return s; } diff --git a/prebuilt/asdl/runtime.mycpp.h b/prebuilt/asdl/runtime.mycpp.h index a417a8e5f1..60dfd3d46d 100644 --- a/prebuilt/asdl/runtime.mycpp.h +++ b/prebuilt/asdl/runtime.mycpp.h @@ -4,9 +4,14 @@ #define ASDL_RUNTIME_MYCPP_H #include "_gen/asdl/hnode.asdl.h" +#include "_gen/display/pretty.asdl.h" #include "cpp/data_lang.h" #include "mycpp/runtime.h" +#include "_gen/display/pretty.asdl.h" + +using pretty_asdl::doc; // ad hoc + namespace runtime { // forward declare class TraversalState; @@ -154,6 +159,7 @@ class _PrettyPrinter { bool _TrySingleLineObj(hnode::Record* node, format::ColorOutput* f, int max_chars); bool _TrySingleLine(hnode_asdl::hnode_t* node, format::ColorOutput* f, int max_chars); void PrintTree(hnode_asdl::hnode_t* node, format::ColorOutput* f); +void PrintTree2(hnode_asdl::hnode_t* node, format::ColorOutput* f); } // declare namespace format diff --git a/prebuilt/core/error.mycpp.h b/prebuilt/core/error.mycpp.h index fb627067a6..584979c783 100644 --- a/prebuilt/core/error.mycpp.h +++ b/prebuilt/core/error.mycpp.h @@ -4,6 +4,7 @@ #define CORE_ERROR_MYCPP_H #include "_gen/asdl/hnode.asdl.h" +#include "_gen/display/pretty.asdl.h" #include "cpp/data_lang.h" #include "mycpp/runtime.h" diff --git a/prebuilt/frontend/args.mycpp.cc b/prebuilt/frontend/args.mycpp.cc index 01dda3d948..dc3c03ee28 100644 --- a/prebuilt/frontend/args.mycpp.cc +++ b/prebuilt/frontend/args.mycpp.cc @@ -46,88 +46,97 @@ GLOBAL_STR(str37, "["); GLOBAL_STR(str38, " "); GLOBAL_STR(str39, "]"); GLOBAL_STR(str40, "...0x%s"); -GLOBAL_STR(str41, "\u001b[0;0m"); -GLOBAL_STR(str42, "\u001b[1m"); -GLOBAL_STR(str43, "\u001b[4m"); -GLOBAL_STR(str44, "\u001b[7m"); -GLOBAL_STR(str45, "\u001b[31m"); -GLOBAL_STR(str46, "\u001b[32m"); -GLOBAL_STR(str47, "\u001b[33m"); -GLOBAL_STR(str48, "\u001b[34m"); -GLOBAL_STR(str49, "\u001b[35m"); -GLOBAL_STR(str50, "\u001b[36m"); -GLOBAL_STR(str51, "\u001b[37m"); -GLOBAL_STR(str52, "&"); -GLOBAL_STR(str53, "&"); -GLOBAL_STR(str54, "<"); -GLOBAL_STR(str55, "<"); -GLOBAL_STR(str56, ">"); -GLOBAL_STR(str57, ">"); -GLOBAL_STR(str58, "<%s %r>"); -GLOBAL_STR(str59, "code"); -GLOBAL_STR(str60, "message"); -GLOBAL_STR(str61, "%s, got %s"); -GLOBAL_STR(str62, " (line %d, offset %d-%d: %r)"); -GLOBAL_STR(str63, "-"); -GLOBAL_STR(str64, "_"); -GLOBAL_STR(str65, "<_Attributes %s>"); -GLOBAL_STR(str66, ""); -GLOBAL_STR(str67, "got too many arguments"); -GLOBAL_STR(str68, "expected argument to %r"); -GLOBAL_STR(str69, "-"); -GLOBAL_STR(str70, "expected integer after %s, got %r"); -GLOBAL_STR(str71, "-"); -GLOBAL_STR(str72, "got invalid integer for %s: %s"); -GLOBAL_STR(str73, "-"); -GLOBAL_STR(str74, "expected number after %r, got %r"); -GLOBAL_STR(str75, "-"); -GLOBAL_STR(str76, "got invalid float for %s: %s"); -GLOBAL_STR(str77, "-"); -GLOBAL_STR(str78, "got invalid argument %r to %r, expected one of: %s"); -GLOBAL_STR(str79, "-"); -GLOBAL_STR(str80, "|"); -GLOBAL_STR(str81, "0"); -GLOBAL_STR(str82, "F"); -GLOBAL_STR(str83, "false"); -GLOBAL_STR(str84, "False"); -GLOBAL_STR(str85, "1"); -GLOBAL_STR(str86, "T"); -GLOBAL_STR(str87, "true"); -GLOBAL_STR(str88, "Talse"); -GLOBAL_STR(str89, "got invalid argument to boolean flag: %r"); -GLOBAL_STR(str90, "-"); -GLOBAL_STR(str91, "-"); -GLOBAL_STR(str92, "Invalid option %r"); -GLOBAL_STR(str93, "Expected argument for action"); -GLOBAL_STR(str94, "Invalid action name %r"); -GLOBAL_STR(str95, "--"); -GLOBAL_STR(str96, "--"); -GLOBAL_STR(str97, "="); -GLOBAL_STR(str98, "got invalid flag %r"); -GLOBAL_STR(str99, "-"); -GLOBAL_STR(str100, "0"); -GLOBAL_STR(str101, "Z"); +GLOBAL_STR(str41, "foo"); +GLOBAL_STR(str42, "\n"); +GLOBAL_STR(str43, "\u001b[0;0m"); +GLOBAL_STR(str44, "\u001b[1m"); +GLOBAL_STR(str45, "\u001b[4m"); +GLOBAL_STR(str46, "\u001b[7m"); +GLOBAL_STR(str47, "\u001b[31m"); +GLOBAL_STR(str48, "\u001b[32m"); +GLOBAL_STR(str49, "\u001b[33m"); +GLOBAL_STR(str50, "\u001b[34m"); +GLOBAL_STR(str51, "\u001b[35m"); +GLOBAL_STR(str52, "\u001b[36m"); +GLOBAL_STR(str53, "\u001b[37m"); +GLOBAL_STR(str54, "\n"); +GLOBAL_STR(str55, "&"); +GLOBAL_STR(str56, "&"); +GLOBAL_STR(str57, "<"); +GLOBAL_STR(str58, "<"); +GLOBAL_STR(str59, ">"); +GLOBAL_STR(str60, ">"); +GLOBAL_STR(str61, "<%s %r>"); +GLOBAL_STR(str62, "code"); +GLOBAL_STR(str63, "message"); +GLOBAL_STR(str64, "%s, got %s"); +GLOBAL_STR(str65, " (line %d, offset %d-%d: %r)"); +GLOBAL_STR(str66, "-"); +GLOBAL_STR(str67, "_"); +GLOBAL_STR(str68, "<_Attributes %s>"); +GLOBAL_STR(str69, ""); +GLOBAL_STR(str70, "got too many arguments"); +GLOBAL_STR(str71, "expected argument to %r"); +GLOBAL_STR(str72, "-"); +GLOBAL_STR(str73, "expected integer after %s, got %r"); +GLOBAL_STR(str74, "-"); +GLOBAL_STR(str75, "got invalid integer for %s: %s"); +GLOBAL_STR(str76, "-"); +GLOBAL_STR(str77, "expected number after %r, got %r"); +GLOBAL_STR(str78, "-"); +GLOBAL_STR(str79, "got invalid float for %s: %s"); +GLOBAL_STR(str80, "-"); +GLOBAL_STR(str81, "got invalid argument %r to %r, expected one of: %s"); +GLOBAL_STR(str82, "-"); +GLOBAL_STR(str83, "|"); +GLOBAL_STR(str84, "0"); +GLOBAL_STR(str85, "F"); +GLOBAL_STR(str86, "false"); +GLOBAL_STR(str87, "False"); +GLOBAL_STR(str88, "1"); +GLOBAL_STR(str89, "T"); +GLOBAL_STR(str90, "true"); +GLOBAL_STR(str91, "Talse"); +GLOBAL_STR(str92, "got invalid argument to boolean flag: %r"); +GLOBAL_STR(str93, "-"); +GLOBAL_STR(str94, "-"); +GLOBAL_STR(str95, "Invalid option %r"); +GLOBAL_STR(str96, "Expected argument for action"); +GLOBAL_STR(str97, "Invalid action name %r"); +GLOBAL_STR(str98, "--"); +GLOBAL_STR(str99, "--"); +GLOBAL_STR(str100, "="); +GLOBAL_STR(str101, "got invalid flag %r"); GLOBAL_STR(str102, "-"); -GLOBAL_STR(str103, "doesn't accept flag %s"); -GLOBAL_STR(str104, "-"); -GLOBAL_STR(str105, "+"); -GLOBAL_STR(str106, "+"); -GLOBAL_STR(str107, "doesn't accept option %s"); +GLOBAL_STR(str103, "0"); +GLOBAL_STR(str104, "Z"); +GLOBAL_STR(str105, "-"); +GLOBAL_STR(str106, "doesn't accept flag %s"); +GLOBAL_STR(str107, "-"); GLOBAL_STR(str108, "+"); -GLOBAL_STR(str109, "-"); -GLOBAL_STR(str110, "--"); -GLOBAL_STR(str111, "--"); -GLOBAL_STR(str112, "got invalid flag %r"); -GLOBAL_STR(str113, "-"); -GLOBAL_STR(str114, "+"); +GLOBAL_STR(str109, "+"); +GLOBAL_STR(str110, "doesn't accept option %s"); +GLOBAL_STR(str111, "+"); +GLOBAL_STR(str112, "-"); +GLOBAL_STR(str113, "--"); +GLOBAL_STR(str114, "--"); GLOBAL_STR(str115, "got invalid flag %r"); GLOBAL_STR(str116, "-"); +GLOBAL_STR(str117, "+"); +GLOBAL_STR(str118, "got invalid flag %r"); +GLOBAL_STR(str119, "-"); namespace ansi { // forward declare } // forward declare namespace ansi +namespace pretty { // forward declare + + class PrettyPrinter; + +} // forward declare namespace pretty + namespace cgi { // forward declare @@ -180,6 +189,36 @@ extern BigStr* WHITE; } // declare namespace ansi +namespace pretty { // declare + +pretty_asdl::Measure* _EmptyMeasure(); +pretty_asdl::Measure* _FlattenMeasure(pretty_asdl::Measure* measure); +pretty_asdl::Measure* _ConcatMeasure(pretty_asdl::Measure* m1, pretty_asdl::Measure* m2); +int _SuffixLen(pretty_asdl::Measure* measure); +pretty_asdl::MeasuredDoc* AsciiText(BigStr* string); +pretty_asdl::MeasuredDoc* _Break(BigStr* string); +pretty_asdl::MeasuredDoc* _Indent(int indent, pretty_asdl::MeasuredDoc* mdoc); +pretty_asdl::MeasuredDoc* _Concat(List* mdocs); +pretty_asdl::MeasuredDoc* _Group(pretty_asdl::MeasuredDoc* mdoc); +pretty_asdl::MeasuredDoc* _IfFlat(pretty_asdl::MeasuredDoc* flat_mdoc, pretty_asdl::MeasuredDoc* nonflat_mdoc); +pretty_asdl::MeasuredDoc* _Flat(pretty_asdl::MeasuredDoc* mdoc); +class PrettyPrinter { + public: + PrettyPrinter(int max_width); + bool _Fits(int prefix_len, doc::Group* group, pretty_asdl::Measure* suffix_measure); + void PrintDoc(pretty_asdl::MeasuredDoc* document, mylib::BufWriter* buf); + int max_width; + + static constexpr ObjHeader obj_header() { + return ObjHeader::ClassScanned(0, sizeof(PrettyPrinter)); + } + + DISALLOW_COPY_AND_ASSIGN(PrettyPrinter) +}; + + +} // declare namespace pretty + namespace cgi { // declare BigStr* escape(BigStr* s); @@ -1066,33 +1105,259 @@ void PrintTree(hnode_asdl::hnode_t* node, format::ColorOutput* f) { pp->PrintNode(node, f, 0); } +void PrintTree2(hnode_asdl::hnode_t* node, format::ColorOutput* f) { + pretty_asdl::MeasuredDoc* doc = nullptr; + pretty::PrettyPrinter* printer = nullptr; + mylib::BufWriter* buf = nullptr; + StackRoot _root0(&node); + StackRoot _root1(&f); + StackRoot _root2(&doc); + StackRoot _root3(&printer); + StackRoot _root4(&buf); + + doc = pretty::AsciiText(str41); + printer = Alloc(20); + buf = Alloc(); + printer->PrintDoc(doc, buf); + f->write(buf->getvalue()); + f->write(str42); +} + } // define namespace format namespace ansi { // define -BigStr* RESET = str41; -BigStr* BOLD = str42; -BigStr* UNDERLINE = str43; -BigStr* REVERSE = str44; -BigStr* RED = str45; -BigStr* GREEN = str46; -BigStr* YELLOW = str47; -BigStr* BLUE = str48; -BigStr* MAGENTA = str49; -BigStr* CYAN = str50; -BigStr* WHITE = str51; +BigStr* RESET = str43; +BigStr* BOLD = str44; +BigStr* UNDERLINE = str45; +BigStr* REVERSE = str46; +BigStr* RED = str47; +BigStr* GREEN = str48; +BigStr* YELLOW = str49; +BigStr* BLUE = str50; +BigStr* MAGENTA = str51; +BigStr* CYAN = str52; +BigStr* WHITE = str53; } // define namespace ansi +namespace pretty { // define + +using pretty_asdl::doc; +using pretty_asdl::doc_e; +using pretty_asdl::DocFragment; +using pretty_asdl::Measure; +using pretty_asdl::MeasuredDoc; +using mylib::BufWriter; + +pretty_asdl::Measure* _EmptyMeasure() { + return Alloc(0, -1); +} + +pretty_asdl::Measure* _FlattenMeasure(pretty_asdl::Measure* measure) { + StackRoot _root0(&measure); + + return Alloc(measure->flat, -1); +} + +pretty_asdl::Measure* _ConcatMeasure(pretty_asdl::Measure* m1, pretty_asdl::Measure* m2) { + StackRoot _root0(&m1); + StackRoot _root1(&m2); + + if (m1->nonflat != -1) { + return Alloc((m1->flat + m2->flat), m1->nonflat); + } + else { + if (m2->nonflat != -1) { + return Alloc((m1->flat + m2->flat), (m1->flat + m2->nonflat)); + } + else { + return Alloc((m1->flat + m2->flat), -1); + } + } +} + +int _SuffixLen(pretty_asdl::Measure* measure) { + StackRoot _root0(&measure); + + if (measure->nonflat != -1) { + return measure->nonflat; + } + else { + return measure->flat; + } +} + +pretty_asdl::MeasuredDoc* AsciiText(BigStr* string) { + StackRoot _root0(&string); + + return Alloc(Alloc(string), Alloc(len(string), -1)); +} + +pretty_asdl::MeasuredDoc* _Break(BigStr* string) { + StackRoot _root0(&string); + + return Alloc(Alloc(string), Alloc(len(string), 0)); +} + +pretty_asdl::MeasuredDoc* _Indent(int indent, pretty_asdl::MeasuredDoc* mdoc) { + StackRoot _root0(&mdoc); + + return Alloc(Alloc(indent, mdoc), mdoc->measure); +} + +pretty_asdl::MeasuredDoc* _Concat(List* mdocs) { + pretty_asdl::Measure* measure = nullptr; + StackRoot _root0(&mdocs); + StackRoot _root1(&measure); + + measure = _EmptyMeasure(); + for (ListIter it(mdocs); !it.Done(); it.Next()) { + pretty_asdl::MeasuredDoc* mdoc = it.Value(); + StackRoot _for(&mdoc ); + measure = _ConcatMeasure(measure, mdoc->measure); + } + return Alloc(Alloc(mdocs), measure); +} + +pretty_asdl::MeasuredDoc* _Group(pretty_asdl::MeasuredDoc* mdoc) { + StackRoot _root0(&mdoc); + + return Alloc(Alloc(mdoc), mdoc->measure); +} + +pretty_asdl::MeasuredDoc* _IfFlat(pretty_asdl::MeasuredDoc* flat_mdoc, pretty_asdl::MeasuredDoc* nonflat_mdoc) { + StackRoot _root0(&flat_mdoc); + StackRoot _root1(&nonflat_mdoc); + + return Alloc(Alloc(flat_mdoc, nonflat_mdoc), Alloc(flat_mdoc->measure->flat, nonflat_mdoc->measure->nonflat)); +} + +pretty_asdl::MeasuredDoc* _Flat(pretty_asdl::MeasuredDoc* mdoc) { + StackRoot _root0(&mdoc); + + return Alloc(Alloc(mdoc), _FlattenMeasure(mdoc->measure)); +} + +PrettyPrinter::PrettyPrinter(int max_width) { + this->max_width = max_width; +} + +bool PrettyPrinter::_Fits(int prefix_len, doc::Group* group, pretty_asdl::Measure* suffix_measure) { + pretty_asdl::Measure* measure = nullptr; + StackRoot _root0(&group); + StackRoot _root1(&suffix_measure); + StackRoot _root2(&measure); + + measure = _ConcatMeasure(_FlattenMeasure(group->mdoc->measure), suffix_measure); + return (prefix_len + _SuffixLen(measure)) <= this->max_width; +} + +void PrettyPrinter::PrintDoc(pretty_asdl::MeasuredDoc* document, mylib::BufWriter* buf) { + int prefix_len; + List* fragments = nullptr; + pretty_asdl::DocFragment* frag = nullptr; + doc::Text* text = nullptr; + BigStr* break_str = nullptr; + doc::Indent* indented = nullptr; + doc::Concat* concat = nullptr; + pretty_asdl::Measure* measure = nullptr; + doc::Group* group = nullptr; + bool flat; + doc::IfFlat* if_flat = nullptr; + pretty_asdl::MeasuredDoc* subdoc = nullptr; + doc::Flat* flat_doc = nullptr; + StackRoot _root0(&document); + StackRoot _root1(&buf); + StackRoot _root2(&fragments); + StackRoot _root3(&frag); + StackRoot _root4(&text); + StackRoot _root5(&break_str); + StackRoot _root6(&indented); + StackRoot _root7(&concat); + StackRoot _root8(&measure); + StackRoot _root9(&group); + StackRoot _root10(&if_flat); + StackRoot _root11(&subdoc); + StackRoot _root12(&flat_doc); + + prefix_len = 0; + fragments = NewList(std::initializer_list{Alloc(_Group(document), 0, false, _EmptyMeasure())}); + while (len(fragments) > 0) { + frag = fragments->pop(); + switch (frag->mdoc->doc->tag()) { + case doc_e::Text: { + text = static_cast(frag->mdoc->doc); + buf->write(text->string); + prefix_len += frag->mdoc->measure->flat; + } + break; + case doc_e::Break: { + if (frag->is_flat) { + break_str = static_cast(frag->mdoc->doc)->string; + buf->write(break_str); + prefix_len += frag->mdoc->measure->flat; + } + else { + buf->write(str54); + buf->write_spaces(frag->indent); + prefix_len = frag->indent; + } + } + break; + case doc_e::Indent: { + indented = static_cast(frag->mdoc->doc); + fragments->append(Alloc(indented->mdoc, (frag->indent + indented->indent), frag->is_flat, frag->measure)); + } + break; + case doc_e::Concat: { + concat = static_cast(frag->mdoc->doc); + measure = frag->measure; + for (ReverseListIter it(concat->mdocs); !it.Done(); it.Next()) { + pretty_asdl::MeasuredDoc* mdoc = it.Value(); + StackRoot _for(&mdoc ); + fragments->append(Alloc(mdoc, frag->indent, frag->is_flat, measure)); + measure = _ConcatMeasure(mdoc->measure, measure); + } + } + break; + case doc_e::Group: { + group = static_cast(frag->mdoc->doc); + flat = this->_Fits(prefix_len, group, frag->measure); + fragments->append(Alloc(group->mdoc, frag->indent, flat, frag->measure)); + } + break; + case doc_e::IfFlat: { + if_flat = static_cast(frag->mdoc->doc); + if (frag->is_flat) { + subdoc = if_flat->flat_mdoc; + } + else { + subdoc = if_flat->nonflat_mdoc; + } + fragments->append(Alloc(subdoc, frag->indent, frag->is_flat, frag->measure)); + } + break; + case doc_e::Flat: { + flat_doc = static_cast(frag->mdoc->doc); + fragments->append(Alloc(flat_doc->mdoc, frag->indent, true, frag->measure)); + } + break; + } + } +} + +} // define namespace pretty + namespace cgi { // define BigStr* escape(BigStr* s) { StackRoot _root0(&s); - s = s->replace(str52, str53); - s = s->replace(str54, str55); - s = s->replace(str56, str57); + s = s->replace(str55, str56); + s = s->replace(str57, str58); + s = s->replace(str59, str60); return s; } @@ -1213,8 +1478,8 @@ value::Dict* Structured::ToDict() { if (this->properties == nullptr) { this->properties = Alloc>(); } - this->properties->set(str59, num::ToBig(this->ExitStatus())); - this->properties->set(str60, Alloc(this->msg)); + this->properties->set(str62, num::ToBig(this->ExitStatus())); + this->properties->set(str63, Alloc(this->msg)); return Alloc(this->properties); } @@ -1422,7 +1687,7 @@ void _Attributes::Set(BigStr* name, value_asdl::value_t* val) { StackRoot _root0(&name); StackRoot _root1(&val); - name = name->replace(str63, str64); + name = name->replace(str66, str67); this->attrs->set(name, val); } @@ -1498,7 +1763,7 @@ bool Reader::AtEnd() { void Reader::Done() { if (!this->AtEnd()) { - e_usage(str67, this->Location()); + e_usage(str70, this->Location()); } } @@ -1573,7 +1838,7 @@ bool _ArgAction::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attri arg_r->Next(); arg = arg_r->Peek(); if (arg == nullptr) { - e_usage(StrFormat("expected argument to %r", str_concat(str69, this->name)), arg_r->Location()); + e_usage(StrFormat("expected argument to %r", str_concat(str72, this->name)), arg_r->Location()); } } val = this->_Value(arg, arg_r->Location()); @@ -1593,10 +1858,10 @@ value_asdl::value_t* SetToInt::_Value(BigStr* arg, syntax_asdl::loc_t* location) i = mops::FromStr(arg); } catch (ValueError*) { - e_usage(StrFormat("expected integer after %s, got %r", str_concat(str71, this->name), arg), location); + e_usage(StrFormat("expected integer after %s, got %r", str_concat(str74, this->name), arg), location); } if (mops::Greater(mops::BigInt(0), i)) { - e_usage(StrFormat("got invalid integer for %s: %s", str_concat(str73, this->name), arg), location); + e_usage(StrFormat("got invalid integer for %s: %s", str_concat(str76, this->name), arg), location); } return Alloc(i); } @@ -1613,10 +1878,10 @@ value_asdl::value_t* SetToFloat::_Value(BigStr* arg, syntax_asdl::loc_t* locatio f = to_float(arg); } catch (ValueError*) { - e_usage(StrFormat("expected number after %r, got %r", str_concat(str75, this->name), arg), location); + e_usage(StrFormat("expected number after %r, got %r", str_concat(str78, this->name), arg), location); } if (f < 0) { - e_usage(StrFormat("got invalid float for %s: %s", str_concat(str77, this->name), arg), location); + e_usage(StrFormat("got invalid float for %s: %s", str_concat(str80, this->name), arg), location); } return Alloc(f); } @@ -1629,7 +1894,7 @@ value_asdl::value_t* SetToString::_Value(BigStr* arg, syntax_asdl::loc_t* locati StackRoot _root1(&location); if ((this->valid != nullptr and !list_contains(this->valid, arg))) { - e_usage(StrFormat("got invalid argument %r to %r, expected one of: %s", arg, str_concat(str79, this->name), str80->join(this->valid)), location); + e_usage(StrFormat("got invalid argument %r to %r, expected one of: %s", arg, str_concat(str82, this->name), str83->join(this->valid)), location); } return Alloc(arg); } @@ -1645,11 +1910,11 @@ bool SetAttachedBool::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_ StackRoot _root2(&out); if (attached_arg != nullptr) { - if ((str_equals(attached_arg, str81) || str_equals(attached_arg, str82) || str_equals(attached_arg, str83) || str_equals(attached_arg, str84))) { + if ((str_equals(attached_arg, str84) || str_equals(attached_arg, str85) || str_equals(attached_arg, str86) || str_equals(attached_arg, str87))) { b = false; } else { - if ((str_equals(attached_arg, str85) || str_equals(attached_arg, str86) || str_equals(attached_arg, str87) || str_equals(attached_arg, str88))) { + if ((str_equals(attached_arg, str88) || str_equals(attached_arg, str89) || str_equals(attached_arg, str90) || str_equals(attached_arg, str91))) { b = true; } else { @@ -1687,7 +1952,7 @@ bool SetOption::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attrib StackRoot _root1(&arg_r); StackRoot _root2(&out); - b = maybe_str_equals(attached_arg, str90); + b = maybe_str_equals(attached_arg, str93); out->opt_changes->append((Alloc>(this->name, b))); return false; } @@ -1715,7 +1980,7 @@ bool SetNamedOption::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_A StackRoot _root4(&attr_name); StackRoot _root5(&changes); - b = maybe_str_equals(attached_arg, str91); + b = maybe_str_equals(attached_arg, str94); arg_r->Next(); arg = arg_r->Peek(); if (arg == nullptr) { @@ -1766,7 +2031,7 @@ bool SetNamedAction::OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_A arg_r->Next(); arg = arg_r->Peek(); if (arg == nullptr) { - e_usage(str93, loc::Missing); + e_usage(str96, loc::Missing); } attr_name = arg; if ((len(this->names) and !list_contains(this->names, attr_name))) { @@ -1799,13 +2064,13 @@ args::_Attributes* Parse(flag_spec::_FlagSpec* spec, args::Reader* arg_r) { out = Alloc<_Attributes>(spec->defaults); while (!arg_r->AtEnd()) { arg = arg_r->Peek(); - if (maybe_str_equals(arg, str95)) { + if (maybe_str_equals(arg, str98)) { out->saw_double_dash = true; arg_r->Next(); break; } - if ((len(spec->actions_long) and arg->startswith(str96))) { - pos = arg->find(str97, 2); + if ((len(spec->actions_long) and arg->startswith(str99))) { + pos = arg->find(str100, 2); if (pos == -1) { suffix = nullptr; flag_name = arg->slice(2); @@ -1823,15 +2088,15 @@ args::_Attributes* Parse(flag_spec::_FlagSpec* spec, args::Reader* arg_r) { continue; } else { - if ((arg->startswith(str99) and len(arg) > 1)) { + if ((arg->startswith(str102) and len(arg) > 1)) { n = len(arg); for (int i = 1; i < n; ++i) { ch = arg->at(i); - if (str_equals(ch, str100)) { - ch = str101; + if (str_equals(ch, str103)) { + ch = str104; } if (list_contains(spec->plus_flags, ch)) { - out->Set(ch, Alloc(str102)); + out->Set(ch, Alloc(str105)); continue; } if (list_contains(spec->arity0, ch)) { @@ -1844,20 +2109,20 @@ args::_Attributes* Parse(flag_spec::_FlagSpec* spec, args::Reader* arg_r) { action->OnMatch(attached_arg, arg_r, out); break; } - e_usage(StrFormat("doesn't accept flag %s", str_concat(str104, ch)), arg_r->Location()); + e_usage(StrFormat("doesn't accept flag %s", str_concat(str107, ch)), arg_r->Location()); } arg_r->Next(); } else { - if ((len(spec->plus_flags) and (arg->startswith(str105) and len(arg) > 1))) { + if ((len(spec->plus_flags) and (arg->startswith(str108) and len(arg) > 1))) { n = len(arg); for (int i = 1; i < n; ++i) { ch = arg->at(i); if (list_contains(spec->plus_flags, ch)) { - out->Set(ch, Alloc(str106)); + out->Set(ch, Alloc(str109)); continue; } - e_usage(StrFormat("doesn't accept option %s", str_concat(str108, ch)), arg_r->Location()); + e_usage(StrFormat("doesn't accept option %s", str_concat(str111, ch)), arg_r->Location()); } arg_r->Next(); } @@ -1885,7 +2150,7 @@ args::_Attributes* ParseLikeEcho(flag_spec::_FlagSpec* spec, args::Reader* arg_r while (!arg_r->AtEnd()) { arg = arg_r->Peek(); chars = arg->slice(1); - if ((arg->startswith(str109) and len(chars))) { + if ((arg->startswith(str112) and len(chars))) { done = false; for (StrIter it(chars); !it.Done(); it.Next()) { BigStr* c = it.Value(); @@ -1931,12 +2196,12 @@ args::_Attributes* ParseMore(flag_spec::_FlagSpecAndMore* spec, args::Reader* ar quit = false; while (!arg_r->AtEnd()) { arg = arg_r->Peek(); - if (maybe_str_equals(arg, str110)) { + if (maybe_str_equals(arg, str113)) { out->saw_double_dash = true; arg_r->Next(); break; } - if (arg->startswith(str111)) { + if (arg->startswith(str114)) { action = spec->actions_long->get(arg->slice(2)); if (action == nullptr) { e_usage(StrFormat("got invalid flag %r", arg), arg_r->Location()); @@ -1945,14 +2210,14 @@ args::_Attributes* ParseMore(flag_spec::_FlagSpecAndMore* spec, args::Reader* ar arg_r->Next(); continue; } - if (((arg->startswith(str113) or arg->startswith(str114)) and len(arg) > 1)) { + if (((arg->startswith(str116) or arg->startswith(str117)) and len(arg) > 1)) { char0 = arg->at(0); for (StrIter it(arg->slice(1)); !it.Done(); it.Next()) { BigStr* ch = it.Value(); StackRoot _for(&ch ); action = spec->actions_short->get(ch); if (action == nullptr) { - e_usage(StrFormat("got invalid flag %r", str_concat(str116, ch)), arg_r->Location()); + e_usage(StrFormat("got invalid flag %r", str_concat(str119, ch)), arg_r->Location()); } attached_arg = list_contains(spec->plus_flags, ch) ? char0 : nullptr; quit = action->OnMatch(attached_arg, arg_r, out); diff --git a/prebuilt/frontend/args.mycpp.h b/prebuilt/frontend/args.mycpp.h index dafe4d6905..5db44edaee 100644 --- a/prebuilt/frontend/args.mycpp.h +++ b/prebuilt/frontend/args.mycpp.h @@ -4,15 +4,18 @@ #define FRONTEND_ARGS_MYCPP_H #include "_gen/asdl/hnode.asdl.h" +#include "_gen/display/pretty.asdl.h" #include "cpp/data_lang.h" #include "mycpp/runtime.h" #include "_gen/core/runtime.asdl.h" #include "_gen/core/value.asdl.h" +#include "_gen/display/pretty.asdl.h" #include "_gen/frontend/syntax.asdl.h" #include "cpp/frontend_flag_spec.h" using value_asdl::value; // This is a bit ad hoc +using pretty_asdl::doc; namespace runtime { // forward declare @@ -179,6 +182,7 @@ class _PrettyPrinter { bool _TrySingleLineObj(hnode::Record* node, format::ColorOutput* f, int max_chars); bool _TrySingleLine(hnode_asdl::hnode_t* node, format::ColorOutput* f, int max_chars); void PrintTree(hnode_asdl::hnode_t* node, format::ColorOutput* f); +void PrintTree2(hnode_asdl::hnode_t* node, format::ColorOutput* f); } // declare namespace format diff --git a/prebuilt/translate.sh b/prebuilt/translate.sh index c9d0c8685d..033ab21b7c 100755 --- a/prebuilt/translate.sh +++ b/prebuilt/translate.sh @@ -48,6 +48,7 @@ oils-part() { echo "#define $guard" echo echo '#include "_gen/asdl/hnode.asdl.h"' + echo '#include "_gen/display/pretty.asdl.h"' echo '#include "cpp/data_lang.h"' echo '#include "mycpp/runtime.h"' echo "$more_include" @@ -69,7 +70,7 @@ EOF } readonly -a ASDL_FILES=( - $REPO_ROOT/{asdl/runtime,asdl/format,display/ansi,pylib/cgi,data_lang/j8_lite}.py \ + $REPO_ROOT/{asdl/runtime,asdl/format,display/ansi,display/pretty,pylib/cgi,data_lang/j8_lite}.py \ ) asdl-runtime() { @@ -78,13 +79,19 @@ asdl-runtime() { prebuilt/asdl/runtime.mycpp \ $TEMP_DIR/asdl/runtime_raw.mycpp.h \ ASDL_RUNTIME_MYCPP_H \ - '' \ + ' +#include "_gen/display/pretty.asdl.h" + +using pretty_asdl::doc; // ad hoc + ' \ --to-header asdl.runtime \ --to-header asdl.format \ "${ASDL_FILES[@]}" } core-error() { + ### For cpp/osh_test.cc + # Depends on frontend/syntax_asdl mkdir -p prebuilt/core $TEMP_DIR/core @@ -105,6 +112,8 @@ using value_asdl::value; // This is a bit ad hoc } frontend-args() { + ### For cpp/frontend_args_test.cc + # Depends on core/runtime_asdl mkdir -p prebuilt/frontend $TEMP_DIR/frontend @@ -115,10 +124,12 @@ frontend-args() { ' #include "_gen/core/runtime.asdl.h" #include "_gen/core/value.asdl.h" +#include "_gen/display/pretty.asdl.h" #include "_gen/frontend/syntax.asdl.h" #include "cpp/frontend_flag_spec.h" using value_asdl::value; // This is a bit ad hoc +using pretty_asdl::doc; ' \ --to-header asdl.runtime \ --to-header asdl.format \ diff --git a/yaks/preamble.h b/yaks/preamble.h index d766288f8e..5363c45618 100644 --- a/yaks/preamble.h +++ b/yaks/preamble.h @@ -4,6 +4,7 @@ #include "_gen/core/value.asdl.h" // could break this dep from j8? #include "_gen/data_lang/nil8.asdl.h" +#include "_gen/display/pretty.asdl.h" #include "_gen/frontend/consts.h" #include "_gen/frontend/id_kind.asdl.h" // syntax.asdl depends on this #include "_gen/yaks/yaks.asdl.h" @@ -13,5 +14,6 @@ #include "mycpp/runtime.h" // runtime library e.g. with Python data structures // TODO: Why do we need these? +using pretty_asdl::doc; using value_asdl::value; using yaks_asdl::mod_def; From 36c2140dd0d07652e9f628903e48eca59a01a8f7 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 27 Jul 2024 20:21:38 -0400 Subject: [PATCH 070/506] [reformat] build/ninja*.py Manually fixed up docstrings. [demo] Knocked off TODOs in url-search-params.ysh --- build/ninja_lib.py | 834 +++++++++++++++++++------------------ build/ninja_lib_test.py | 420 +++++++++---------- build/ninja_main.py | 557 +++++++++++++------------ demo/url-search-params.ysh | 6 +- 4 files changed, 921 insertions(+), 896 deletions(-) diff --git a/build/ninja_lib.py b/build/ninja_lib.py index 9a5b042d39..1030c916bf 100644 --- a/build/ninja_lib.py +++ b/build/ninja_lib.py @@ -31,9 +31,9 @@ def log(msg, *args): - if args: - msg = msg % args - print(msg, file=sys.stderr) + if args: + msg = msg % args + print(msg, file=sys.stderr) # Matrix of configurations @@ -42,10 +42,8 @@ def log(msg, *args): ('cxx', 'dbg'), ('cxx', 'opt'), ('cxx', 'asan'), - ('cxx', 'asan+gcalways'), ('cxx', 'asan32+gcalways'), - ('cxx', 'ubsan'), #('clang', 'asan'), @@ -58,15 +56,12 @@ def log(msg, *args): GC_PERF_VARIANTS = [ ('cxx', 'opt+bumpleak'), ('cxx', 'opt+bumproot'), - ('cxx', 'opt+bumpsmall'), ('cxx', 'asan+bumpsmall'), - ('cxx', 'opt+nopool'), # TODO: should be binary with different files ('cxx', 'opt+cheney'), - ('cxx', 'opt+tcmalloc'), # For tracing allocations, or debugging @@ -78,25 +73,25 @@ def log(msg, *args): ] SMALL_TEST_MATRIX = [ - ('cxx', 'asan'), - ('cxx', 'ubsan'), - ('clang', 'coverage'), + ('cxx', 'asan'), + ('cxx', 'ubsan'), + ('clang', 'coverage'), ] def ConfigDir(config): - compiler, variant, more_cxx_flags = config - if more_cxx_flags is None: - return '%s-%s' % (compiler, variant) - else: - # -D CPP_UNIT_TEST -> D_CPP_UNIT_TEST - flags_str = more_cxx_flags.replace('-', '').replace(' ', '_') - return '%s-%s-%s' % (compiler, variant, flags_str) + compiler, variant, more_cxx_flags = config + if more_cxx_flags is None: + return '%s-%s' % (compiler, variant) + else: + # -D CPP_UNIT_TEST -> D_CPP_UNIT_TEST + flags_str = more_cxx_flags.replace('-', '').replace(' ', '_') + return '%s-%s-%s' % (compiler, variant, flags_str) def ObjPath(src_path, config): - rel_path, _ = os.path.splitext(src_path) - return '_build/obj/%s/%s.o' % (ConfigDir(config), rel_path) + rel_path, _ = os.path.splitext(src_path) + return '_build/obj/%s/%s.o' % (ConfigDir(config), rel_path) # Used namedtuple since it doesn't have any state @@ -106,414 +101,445 @@ def ObjPath(src_path, config): class CcLibrary(object): - """ - Life cycle: - - 1. A cc_library is first created - 2. A cc_binary can depend on it - - maybe writing rules, and ensuring uniques per configuration - 3. The link step needs the list of objects - 4. The tarball needs the list of sources for binary - """ - - def __init__(self, label, srcs, implicit, deps, headers, generated_headers): - self.label = label - self.srcs = srcs # queried by SourcesForBinary - self.implicit = implicit - self.deps = deps - self.headers = headers - # TODO: asdl() rule should add to this. - # Generated headers are different than regular headers. The former need an - # implicit dep in Ninja, while the latter can rely on the .d mechanism. - self.generated_headers = generated_headers - - self.obj_lookup = {} # config -> list of objects - self.preprocessed_lookup = {} # config -> boolean - - def _CalculateImplicit(self, ru): - """ Compile actions for cc_library() also need implicit deps on generated headers""" - - out_deps = set() - ru._TransitiveClosure(self.label, self.deps, out_deps) - unique_deps = sorted(out_deps) - - implicit = list(self.implicit) # copy - for label in unique_deps: - cc_lib = ru.cc_libs[label] - implicit.extend(cc_lib.generated_headers) - return implicit - - def MaybeWrite(self, ru, config, preprocessed): - if config not in self.obj_lookup: # already written by some other cc_binary() - implicit = self._CalculateImplicit(ru) - - objects = [] - for src in self.srcs: - obj = ObjPath(src, config) - ru.compile(obj, src, self.deps, config, implicit=implicit) - objects.append(obj) - - self.obj_lookup[config] = objects - - if preprocessed and config not in self.preprocessed_lookup: - implicit = self._CalculateImplicit(ru) - - for src in self.srcs: - # no output needed - ru.compile('', src, self.deps, config, implicit=implicit, - maybe_preprocess=True) - self.preprocessed_lookup[config] = True + """ + Life cycle: + + 1. A cc_library is first created + 2. A cc_binary can depend on it + - maybe writing rules, and ensuring uniques per configuration + 3. The link step needs the list of objects + 4. The tarball needs the list of sources for binary + """ + def __init__(self, label, srcs, implicit, deps, headers, + generated_headers): + self.label = label + self.srcs = srcs # queried by SourcesForBinary + self.implicit = implicit + self.deps = deps + self.headers = headers + # TODO: asdl() rule should add to this. + # Generated headers are different than regular headers. The former need an + # implicit dep in Ninja, while the latter can rely on the .d mechanism. + self.generated_headers = generated_headers + + self.obj_lookup = {} # config -> list of objects + self.preprocessed_lookup = {} # config -> boolean + + def _CalculateImplicit(self, ru): + """ Compile actions for cc_library() also need implicit deps on generated headers""" + + out_deps = set() + ru._TransitiveClosure(self.label, self.deps, out_deps) + unique_deps = sorted(out_deps) + + implicit = list(self.implicit) # copy + for label in unique_deps: + cc_lib = ru.cc_libs[label] + implicit.extend(cc_lib.generated_headers) + return implicit + + def MaybeWrite(self, ru, config, preprocessed): + if config not in self.obj_lookup: # already written by some other cc_binary() + implicit = self._CalculateImplicit(ru) + + objects = [] + for src in self.srcs: + obj = ObjPath(src, config) + ru.compile(obj, src, self.deps, config, implicit=implicit) + objects.append(obj) + + self.obj_lookup[config] = objects + + if preprocessed and config not in self.preprocessed_lookup: + implicit = self._CalculateImplicit(ru) + + for src in self.srcs: + # no output needed + ru.compile('', + src, + self.deps, + config, + implicit=implicit, + maybe_preprocess=True) + self.preprocessed_lookup[config] = True -class Rules(object): - """High-level wrapper for NinjaWriter - What should it handle? +class Rules(object): + """High-level wrapper for NinjaWriter - - The (compiler, variant) matrix loop - - Implicit deps for generated code - - Phony convenience targets + What should it handle? - Maybe: exporting data to test runner + - The (compiler, variant) matrix loop + - Implicit deps for generated code + - Phony convenience targets - Terminology: + Maybe: exporting data to test runner - Ninja has - - rules, which are like Bazel "actions" - - build targets + Terminology: - Our library has: - - Build config: (compiler, variant), and more later + Ninja has + - rules, which are like Bazel "actions" + - build targets - - Labels: identifiers starting with //, which are higher level than Ninja - "targets" - cc_library: - //mycpp/runtime + Our library has: + - Build config: (compiler, variant), and more later - //mycpp/examples/expr.asdl - //frontend/syntax.asdl + - Labels: identifiers starting with //, which are higher level than Ninja + "targets" + cc_library: + //mycpp/runtime - - Deps are lists of labels, and have a transitive closure + //mycpp/examples/expr.asdl + //frontend/syntax.asdl - - H Rules / High level rules? B rules / Boil? - cc_binary, cc_library, asdl, etc. - """ - def __init__(self, n): - self.n = n # direct ninja writer + - Deps are lists of labels, and have a transitive closure - self.cc_bins = [] # list of CcBinary() objects to write - self.cc_libs = {} # label -> CcLibrary object - self.cc_binary_deps = {} # main_cc -> list of LABELS - self.phony = {} # list of phony targets + - H Rules / High level rules? B rules / Boil? + cc_binary, cc_library, asdl, etc. + """ - def AddPhony(self, phony_to_add): - self.phony.update(phony_to_add) + def __init__(self, n): + self.n = n # direct ninja writer + + self.cc_bins = [] # list of CcBinary() objects to write + self.cc_libs = {} # label -> CcLibrary object + self.cc_binary_deps = {} # main_cc -> list of LABELS + self.phony = {} # list of phony targets + + def AddPhony(self, phony_to_add): + self.phony.update(phony_to_add) + + def WritePhony(self): + for name in sorted(self.phony): + targets = self.phony[name] + if targets: + self.n.build([name], 'phony', targets) + self.n.newline() + + def WriteRules(self): + for cc_bin in self.cc_bins: + self.WriteCcBinary(cc_bin) + + def compile(self, + out_obj, + in_cc, + deps, + config, + implicit=None, + maybe_preprocess=False): + """ .cc -> compiler -> .o """ + + implicit = implicit or [] + + compiler, variant, more_cxx_flags = config + if more_cxx_flags is None: + flags_str = "''" + else: + assert "'" not in more_cxx_flags, more_cxx_flags # can't handle single quotes + flags_str = "'%s'" % more_cxx_flags + + v = [('compiler', compiler), ('variant', variant), + ('more_cxx_flags', flags_str)] + if maybe_preprocess: + # Limit it to certain configs + if more_cxx_flags is None and variant in ('dbg', 'opt'): + pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, + in_cc) + self.n.build(pre, + 'preprocess', [in_cc], + implicit=implicit, + variables=v) + else: + self.n.build([out_obj], + 'compile_one', [in_cc], + implicit=implicit, + variables=v) - def WritePhony(self): - for name in sorted(self.phony): - targets = self.phony[name] - if targets: - self.n.build([name], 'phony', targets) self.n.newline() - def WriteRules(self): - for cc_bin in self.cc_bins: - self.WriteCcBinary(cc_bin) + def link(self, out_bin, main_obj, deps, config): + """ list of .o -> linker -> executable, along with stripped version """ + compiler, variant, _ = config - def compile(self, out_obj, in_cc, deps, config, implicit=None, maybe_preprocess=False): - """ .cc -> compiler -> .o """ + assert isinstance(out_bin, str), out_bin + assert isinstance(main_obj, str), main_obj - implicit = implicit or [] + objects = [main_obj] + for label in deps: + key = (label, compiler, variant) + try: + cc_lib = self.cc_libs[label] + except KeyError: + raise RuntimeError("Couldn't resolve label %r" % label) - compiler, variant, more_cxx_flags = config - if more_cxx_flags is None: - flags_str = "''" - else: - assert "'" not in more_cxx_flags, more_cxx_flags # can't handle single quotes - flags_str = "'%s'" % more_cxx_flags - - v = [('compiler', compiler), ('variant', variant), ('more_cxx_flags', flags_str)] - if maybe_preprocess: - # Limit it to certain configs - if more_cxx_flags is None and variant in ('dbg', 'opt'): - pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, in_cc) - self.n.build(pre, 'preprocess', [in_cc], implicit=implicit, variables=v) - else: - self.n.build([out_obj], 'compile_one', [in_cc], implicit=implicit, variables=v) - - self.n.newline() - - def link(self, out_bin, main_obj, deps, config): - """ list of .o -> linker -> executable, along with stripped version """ - compiler, variant, _ = config - - assert isinstance(out_bin, str), out_bin - assert isinstance(main_obj, str), main_obj - - objects = [main_obj] - for label in deps: - key = (label, compiler, variant) - try: - cc_lib = self.cc_libs[label] - except KeyError: - raise RuntimeError("Couldn't resolve label %r" % label) - - o = cc_lib.obj_lookup[config] - objects.extend(o) - - v = [('compiler', compiler), ('variant', variant), ('more_link_flags', "''")] - self.n.build([out_bin], 'link', objects, variables=v) - self.n.newline() - - # Strip any .opt binaries - if variant.startswith('opt') or variant.startswith('opt32'): - stripped = out_bin + '.stripped' - symbols = out_bin + '.symbols' - self.n.build([stripped, symbols], 'strip', [out_bin]) - self.n.newline() - - def comment(self, s): - self.n.comment(s) - self.n.newline() - - def cc_library(self, label, - srcs = None, - implicit = None, - deps = None, - # note: headers is only used for tarball manifest, not compiler command line - headers = None, - generated_headers = None): - - # srcs = [] is allowed for _gen/asdl/hnode.asdl.h - if srcs is None: - raise RuntimeError('cc_library %r requires srcs' % label) - - implicit = implicit or [] - deps = deps or [] - headers = headers or [] - generated_headers = generated_headers or [] - - if label in self.cc_libs: - raise RuntimeError('%s was already defined' % label) - - self.cc_libs[label] = CcLibrary(label, srcs, implicit, deps, - headers, generated_headers) - - def _TransitiveClosure(self, name, deps, unique_out): - """ - Args: - name: for error messages - """ - for label in deps: - if label in unique_out: - continue - unique_out.add(label) - - try: - cc_lib = self.cc_libs[label] - except KeyError: - raise RuntimeError('Undefined label %s in %s' % (label, name)) - - self._TransitiveClosure(cc_lib.label, cc_lib.deps, unique_out) - - def cc_binary(self, main_cc, - symlinks = None, - implicit = None, # for COMPILE action, not link action - deps = None, - matrix = None, # $compiler $variant - phony_prefix = None, - preprocessed = False, - bin_path = None, # default is _bin/$compiler-$variant/rel/path - ): - symlinks = symlinks or [] - implicit = implicit or [] - deps = deps or [] - if not matrix: - raise RuntimeError("Config matrix required") - - cc_bin = CcBinary(main_cc, symlinks, implicit, deps, matrix, phony_prefix, - preprocessed, bin_path) - - self.cc_bins.append(cc_bin) - - def WriteCcBinary(self, cc_bin): - c = cc_bin - - out_deps = set() - self._TransitiveClosure(c.main_cc, c.deps, out_deps) - unique_deps = sorted(out_deps) - - # save for SourcesForBinary() - self.cc_binary_deps[c.main_cc] = unique_deps - - compile_imp = list(c.implicit) - for label in unique_deps: - cc_lib = self.cc_libs[label] # should exit - # compile actions of binaries that have ASDL label deps need the - # generated header as implicit dep - compile_imp.extend(cc_lib.generated_headers) - - for config in c.matrix: - if len(config) == 2: - config = (config[0], config[1], None) - - for label in unique_deps: - cc_lib = self.cc_libs[label] # should exit - - cc_lib.MaybeWrite(self, config, c.preprocessed) - - # Compile main object, maybe with IMPLICIT headers deps - main_obj = ObjPath(c.main_cc, config) - self.compile(main_obj, c.main_cc, c.deps, config, implicit=compile_imp) - if c.preprocessed: - self.compile('', c.main_cc, c.deps, config, implicit=compile_imp, - maybe_preprocess=True) - - config_dir = ConfigDir(config) - bin_dir = '_bin/%s' % config_dir - - if c.bin_path: - # e.g. _bin/cxx-dbg/oils_for_unix - bin_ = '%s/%s' % (bin_dir, c.bin_path) - else: - # e.g. _gen/mycpp/examples/classes.mycpp - rel_path, _ = os.path.splitext(c.main_cc) - - # Put binary in _bin/cxx-dbg/mycpp/examples, not _bin/cxx-dbg/_gen/mycpp/examples - if rel_path.startswith('_gen/'): - rel_path = rel_path[len('_gen/'):] - - bin_= '%s/%s' % (bin_dir, rel_path) - - # Link with OBJECT deps - self.link(bin_, main_obj, unique_deps, config) - - # Make symlinks - for symlink in c.symlinks: - # Must explicitly specify bin_path to have a symlink, for now - assert c.bin_path is not None - self.n.build( - ['%s/%s' % (bin_dir, symlink)], - 'symlink', - [bin_], - variables = [('dir', bin_dir), ('target', c.bin_path), ('new', symlink)]) - self.n.newline() - - if c.phony_prefix: - key = '%s-%s' % (c.phony_prefix, config_dir) - if key not in self.phony: - self.phony[key] = [] - self.phony[key].append(bin_) - - def SourcesForBinary(self, main_cc): - """ - Used for preprocessed metrics, release tarball, _build/oils.sh, etc. - """ - deps = self.cc_binary_deps[main_cc] - sources = [main_cc] - for label in deps: - sources.extend(self.cc_libs[label].srcs) - return sources - - def HeadersForBinary(self, main_cc): - deps = self.cc_binary_deps[main_cc] - headers = [] - for label in deps: - headers.extend(self.cc_libs[label].headers) - headers.extend(self.cc_libs[label].generated_headers) - return headers + o = cc_lib.obj_lookup[config] + objects.extend(o) - def asdl_library(self, asdl_path, deps = None, - pretty_print_methods=True): - - deps = deps or [] - - # SYSTEM header, _gen/asdl/hnode.asdl.h - deps.append('//asdl/hnode.asdl') - deps.append('//display/pretty.asdl') - - # to create _gen/mycpp/examples/expr.asdl.h - prefix = '_gen/%s' % asdl_path - - out_cc = prefix + '.cc' - out_header = prefix + '.h' + v = [('compiler', compiler), ('variant', variant), + ('more_link_flags', "''")] + self.n.build([out_bin], 'link', objects, variables=v) + self.n.newline() - asdl_flags = '' + # Strip any .opt binaries + if variant.startswith('opt') or variant.startswith('opt32'): + stripped = out_bin + '.stripped' + symbols = out_bin + '.symbols' + self.n.build([stripped, symbols], 'strip', [out_bin]) + self.n.newline() - if pretty_print_methods: - outputs = [out_cc, out_header] - else: - outputs = [out_header] - asdl_flags += '--no-pretty-print-methods' - - debug_mod = prefix + '_debug.py' - outputs.append(debug_mod) - - # Generating syntax_asdl.h does NOT depend on hnode_asdl.h existing ... - self.n.build(outputs, 'asdl-cpp', [asdl_path], - implicit = ['_bin/shwrap/asdl_main'], - variables = [ - ('action', 'cpp'), - ('out_prefix', prefix), - ('asdl_flags', asdl_flags), - ('debug_mod', debug_mod), - ]) - self.n.newline() - - # ... But COMPILING anything that #includes it does. - # Note: assumes there's a build rule for this "system" ASDL schema - - srcs = [out_cc] if pretty_print_methods else [] - # Define lazy CC library - self.cc_library( - '//' + asdl_path, - srcs = srcs, - deps = deps, - # For compile_one steps of files that #include this ASDL file - generated_headers = [out_header], - ) - - def py_binary(self, main_py, deps_base_dir='_build/NINJA', template='py'): - """ - Wrapper for Python script with dynamically discovered deps - """ - rel_path, _ = os.path.splitext(main_py) - py_module = rel_path.replace('/', '.') # asdl/asdl_main.py -> asdl.asdl_main + def comment(self, s): + self.n.comment(s) + self.n.newline() - deps_path = os.path.join(deps_base_dir, py_module, 'deps.txt') - with open(deps_path) as f: - deps = [line.strip() for line in f] + def cc_library( + self, + label, + srcs=None, + implicit=None, + deps=None, + # note: headers is only used for tarball manifest, not compiler command line + headers=None, + generated_headers=None): + + # srcs = [] is allowed for _gen/asdl/hnode.asdl.h + if srcs is None: + raise RuntimeError('cc_library %r requires srcs' % label) + + implicit = implicit or [] + deps = deps or [] + headers = headers or [] + generated_headers = generated_headers or [] + + if label in self.cc_libs: + raise RuntimeError('%s was already defined' % label) + + self.cc_libs[label] = CcLibrary(label, srcs, implicit, deps, headers, + generated_headers) + + def _TransitiveClosure(self, name, deps, unique_out): + """ + Args: + name: for error messages + """ + for label in deps: + if label in unique_out: + continue + unique_out.add(label) + + try: + cc_lib = self.cc_libs[label] + except KeyError: + raise RuntimeError('Undefined label %s in %s' % (label, name)) + + self._TransitiveClosure(cc_lib.label, cc_lib.deps, unique_out) + + def cc_binary( + self, + main_cc, + symlinks=None, + implicit=None, # for COMPILE action, not link action + deps=None, + matrix=None, # $compiler $variant + phony_prefix=None, + preprocessed=False, + bin_path=None, # default is _bin/$compiler-$variant/rel/path + ): + symlinks = symlinks or [] + implicit = implicit or [] + deps = deps or [] + if not matrix: + raise RuntimeError("Config matrix required") + + cc_bin = CcBinary(main_cc, symlinks, implicit, deps, matrix, + phony_prefix, preprocessed, bin_path) + + self.cc_bins.append(cc_bin) + + def WriteCcBinary(self, cc_bin): + c = cc_bin + + out_deps = set() + self._TransitiveClosure(c.main_cc, c.deps, out_deps) + unique_deps = sorted(out_deps) + + # save for SourcesForBinary() + self.cc_binary_deps[c.main_cc] = unique_deps + + compile_imp = list(c.implicit) + for label in unique_deps: + cc_lib = self.cc_libs[label] # should exit + # compile actions of binaries that have ASDL label deps need the + # generated header as implicit dep + compile_imp.extend(cc_lib.generated_headers) + + for config in c.matrix: + if len(config) == 2: + config = (config[0], config[1], None) + + for label in unique_deps: + cc_lib = self.cc_libs[label] # should exit + + cc_lib.MaybeWrite(self, config, c.preprocessed) + + # Compile main object, maybe with IMPLICIT headers deps + main_obj = ObjPath(c.main_cc, config) + self.compile(main_obj, + c.main_cc, + c.deps, + config, + implicit=compile_imp) + if c.preprocessed: + self.compile('', + c.main_cc, + c.deps, + config, + implicit=compile_imp, + maybe_preprocess=True) + + config_dir = ConfigDir(config) + bin_dir = '_bin/%s' % config_dir + + if c.bin_path: + # e.g. _bin/cxx-dbg/oils_for_unix + bin_ = '%s/%s' % (bin_dir, c.bin_path) + else: + # e.g. _gen/mycpp/examples/classes.mycpp + rel_path, _ = os.path.splitext(c.main_cc) + + # Put binary in _bin/cxx-dbg/mycpp/examples, not _bin/cxx-dbg/_gen/mycpp/examples + if rel_path.startswith('_gen/'): + rel_path = rel_path[len('_gen/'):] + + bin_ = '%s/%s' % (bin_dir, rel_path) + + # Link with OBJECT deps + self.link(bin_, main_obj, unique_deps, config) + + # Make symlinks + for symlink in c.symlinks: + # Must explicitly specify bin_path to have a symlink, for now + assert c.bin_path is not None + self.n.build(['%s/%s' % (bin_dir, symlink)], + 'symlink', [bin_], + variables=[('dir', bin_dir), + ('target', c.bin_path), + ('new', symlink)]) + self.n.newline() + + if c.phony_prefix: + key = '%s-%s' % (c.phony_prefix, config_dir) + if key not in self.phony: + self.phony[key] = [] + self.phony[key].append(bin_) + + def SourcesForBinary(self, main_cc): + """ + Used for preprocessed metrics, release tarball, _build/oils.sh, etc. + """ + deps = self.cc_binary_deps[main_cc] + sources = [main_cc] + for label in deps: + sources.extend(self.cc_libs[label].srcs) + return sources + + def HeadersForBinary(self, main_cc): + deps = self.cc_binary_deps[main_cc] + headers = [] + for label in deps: + headers.extend(self.cc_libs[label].headers) + headers.extend(self.cc_libs[label].generated_headers) + return headers + + def asdl_library(self, asdl_path, deps=None, pretty_print_methods=True): + + deps = deps or [] + + # SYSTEM header, _gen/asdl/hnode.asdl.h + deps.append('//asdl/hnode.asdl') + deps.append('//display/pretty.asdl') + + # to create _gen/mycpp/examples/expr.asdl.h + prefix = '_gen/%s' % asdl_path + + out_cc = prefix + '.cc' + out_header = prefix + '.h' + + asdl_flags = '' + + if pretty_print_methods: + outputs = [out_cc, out_header] + else: + outputs = [out_header] + asdl_flags += '--no-pretty-print-methods' + + debug_mod = prefix + '_debug.py' + outputs.append(debug_mod) + + # Generating syntax_asdl.h does NOT depend on hnode_asdl.h existing ... + self.n.build(outputs, + 'asdl-cpp', [asdl_path], + implicit=['_bin/shwrap/asdl_main'], + variables=[ + ('action', 'cpp'), + ('out_prefix', prefix), + ('asdl_flags', asdl_flags), + ('debug_mod', debug_mod), + ]) + self.n.newline() - deps.remove(main_py) # raises ValueError if it's not there + # ... But COMPILING anything that #includes it does. + # Note: assumes there's a build rule for this "system" ASDL schema + + srcs = [out_cc] if pretty_print_methods else [] + # Define lazy CC library + self.cc_library( + '//' + asdl_path, + srcs=srcs, + deps=deps, + # For compile_one steps of files that #include this ASDL file + generated_headers=[out_header], + ) + + def py_binary(self, main_py, deps_base_dir='_build/NINJA', template='py'): + """ + Wrapper for Python script with dynamically discovered deps + """ + rel_path, _ = os.path.splitext(main_py) + py_module = rel_path.replace( + '/', '.') # asdl/asdl_main.py -> asdl.asdl_main + + deps_path = os.path.join(deps_base_dir, py_module, 'deps.txt') + with open(deps_path) as f: + deps = [line.strip() for line in f] + + deps.remove(main_py) # raises ValueError if it's not there + + basename = os.path.basename(rel_path) + self.n.build('_bin/shwrap/%s' % basename, + 'write-shwrap', [main_py] + deps, + variables=[('template', template)]) + self.n.newline() - basename = os.path.basename(rel_path) - self.n.build('_bin/shwrap/%s' % basename, 'write-shwrap', [main_py] + deps, - variables=[('template', template)]) - self.n.newline() + def souffle_binary(self, souffle_cpp): + """ + Compile souffle C++ into a native executable. + """ + rel_path, _ = os.path.splitext(souffle_cpp) + basename = os.path.basename(rel_path) + + souffle_obj = '_build/obj/datalog/%s.o' % basename + self.n.build([souffle_obj], + 'compile_one', + souffle_cpp, + variables=[('compiler', 'cxx'), ('variant', 'opt'), + ('more_cxx_flags', "'-Ivendor -std=c++17'")]) + + souffle_bin = '_bin/datalog/%s' % basename + self.n.build([souffle_bin], + 'link', + souffle_obj, + variables=[('compiler', 'cxx'), ('variant', 'opt'), + ('more_link_flags', "'-lstdc++fs'")]) - def souffle_binary(self, souffle_cpp): - """ - Compile a souffle C++ into a native executable. - """ - rel_path, _ = os.path.splitext(souffle_cpp) - basename = os.path.basename(rel_path) - - souffle_obj = '_build/obj/datalog/%s.o' % basename - self.n.build( - [souffle_obj], 'compile_one', souffle_cpp, - variables=[ - ('compiler', 'cxx'), - ('variant', 'opt'), - ('more_cxx_flags', "'-Ivendor -std=c++17'") - ]) - - souffle_bin = '_bin/datalog/%s' % basename - self.n.build( - [souffle_bin], 'link', souffle_obj, - variables=[ - ('compiler', 'cxx'), - ('variant', 'opt'), - ('more_link_flags', "'-lstdc++fs'") - ]) - - self.n.newline() + self.n.newline() diff --git a/build/ninja_lib_test.py b/build/ninja_lib_test.py index 292e0a69bd..8fd3df60cd 100755 --- a/build/ninja_lib_test.py +++ b/build/ninja_lib_test.py @@ -15,7 +15,6 @@ MATRIX1 = [CONFIG] - MATRIX = [ ('cxx', 'dbg'), ('cxx', 'opt'), @@ -23,241 +22,234 @@ def CallFor(n, output_name): - for b in n.build_calls: - if b.outputs[0] == output_name: - return b - else: - raise RuntimeError('%s not found' % output_name) + for b in n.build_calls: + if b.outputs[0] == output_name: + return b + else: + raise RuntimeError('%s not found' % output_name) class NinjaTest(unittest.TestCase): - def _Rules(self): - n = ninja_syntax.Writer(sys.stdout) - n = ninja_syntax.FakeWriter(n) + def _Rules(self): + n = ninja_syntax.Writer(sys.stdout) + n = ninja_syntax.FakeWriter(n) - ru = ninja_lib.Rules(n) - return n, ru + ru = ninja_lib.Rules(n) + return n, ru - def test_cc_library_IsLazy(self): - n, ru = self._Rules() + def test_cc_library_IsLazy(self): + n, ru = self._Rules() - ru.cc_library('//mycpp/ab', ['mycpp/a.cc', 'mycpp/b.cc']) - self.assertEqual(0, len(n.build_calls)) + ru.cc_library('//mycpp/ab', ['mycpp/a.cc', 'mycpp/b.cc']) + self.assertEqual(0, len(n.build_calls)) - ru.cc_binary( - 'mycpp/a_test.cc', - deps = ['//mycpp/ab'], - matrix = MATRIX1) + ru.cc_binary('mycpp/a_test.cc', deps=['//mycpp/ab'], matrix=MATRIX1) - ru.WriteRules() + ru.WriteRules() - actions = [b.rule for b in n.build_calls] - self.assertEqual([ - 'compile_one', - 'compile_one', - 'compile_one', - 'link'], - actions) + actions = [b.rule for b in n.build_calls] + self.assertEqual(['compile_one', 'compile_one', 'compile_one', 'link'], + actions) - last = n.build_calls[-1] - self.assertEqual([ - '_build/obj/cxx-dbg/mycpp/a_test.o', - '_build/obj/cxx-dbg/mycpp/a.o', - '_build/obj/cxx-dbg/mycpp/b.o', + last = n.build_calls[-1] + self.assertEqual([ + '_build/obj/cxx-dbg/mycpp/a_test.o', + '_build/obj/cxx-dbg/mycpp/a.o', + '_build/obj/cxx-dbg/mycpp/b.o', ], last.inputs) - # It's NOT used in a binary, so not instantiated - ru.cc_library('//mycpp/z', ['mycpp/z.cc']) - self.assertEqual(4, len(n.build_calls)) + # It's NOT used in a binary, so not instantiated + ru.cc_library('//mycpp/z', ['mycpp/z.cc']) + self.assertEqual(4, len(n.build_calls)) + + self.assertEqual(4, n.num_build_targets()) + + def testDiamondDeps(self): + n, ru = self._Rules() + + # e + # | + # d + # | \ + # b c + # | / + # a + + ru.cc_library('//mycpp/e', srcs=['mycpp/e.cc']) # leaf + ru.cc_library('//mycpp/d', srcs=['mycpp/d.cc'], + deps=['//mycpp/e']) # diamond + ru.cc_library('//mycpp/b', srcs=['mycpp/b.cc'], deps=['//mycpp/d']) + ru.cc_library('//mycpp/c', srcs=['mycpp/c.cc'], deps=['//mycpp/d']) + ru.cc_binary('mycpp/a.cc', + deps=['//mycpp/b', '//mycpp/c'], + matrix=MATRIX1) + + ru.WriteRules() + + actions = [b.rule for b in n.build_calls] + self.assertEqual( + [ + 'compile_one', # e + 'compile_one', # d + 'compile_one', # c + 'compile_one', # b + 'compile_one', # a + 'link' + ], + actions) + + b = CallFor(n, '_bin/cxx-dbg/mycpp/a') + print(b) + self.assertEqual([ + '_build/obj/cxx-dbg/mycpp/a.o', + '_build/obj/cxx-dbg/mycpp/b.o', + '_build/obj/cxx-dbg/mycpp/c.o', + '_build/obj/cxx-dbg/mycpp/d.o', + '_build/obj/cxx-dbg/mycpp/e.o', + ], sorted(b.inputs)) + + def testCircularDeps(self): + # Should be disallowed I think + pass + + def testSourcesForBinary(self): + n, ru = self._Rules() + + ru.cc_library('//mycpp/y', srcs=['mycpp/y.cc', 'mycpp/y2.cc']) + ru.cc_library('//mycpp/z', srcs=['mycpp/z.cc'], deps=['//mycpp/y']) - self.assertEqual(4, n.num_build_targets()) + # cc_library() is lazy + self.assertEqual(0, len(n.build_calls)) - def testDiamondDeps(self): - n, ru = self._Rules() + ru.cc_binary('mycpp/a_test.cc', deps=['//mycpp/z'], matrix=MATRIX) - # e - # | - # d - # | \ - # b c - # | / - # a + ru.WriteRules() - ru.cc_library('//mycpp/e', srcs = ['mycpp/e.cc']) # leaf - ru.cc_library('//mycpp/d', srcs = ['mycpp/d.cc'], deps = ['//mycpp/e']) # diamond - ru.cc_library('//mycpp/b', srcs = ['mycpp/b.cc'], deps = ['//mycpp/d']) - ru.cc_library('//mycpp/c', srcs = ['mycpp/c.cc'], deps = ['//mycpp/d']) - ru.cc_binary('mycpp/a.cc', deps = ['//mycpp/b', '//mycpp/c'], matrix = MATRIX1) + self.assertEqual(11, len(n.build_calls)) - ru.WriteRules() + srcs = ru.SourcesForBinary('mycpp/a_test.cc') + self.assertEqual( + ['mycpp/a_test.cc', 'mycpp/y.cc', 'mycpp/y2.cc', 'mycpp/z.cc'], + srcs) - actions = [b.rule for b in n.build_calls] - self.assertEqual([ - 'compile_one', # e - 'compile_one', # d - 'compile_one', # c - 'compile_one', # b - 'compile_one', # a - 'link'], - actions) - - b = CallFor(n, '_bin/cxx-dbg/mycpp/a') - print(b) - self.assertEqual([ - '_build/obj/cxx-dbg/mycpp/a.o', - '_build/obj/cxx-dbg/mycpp/b.o', - '_build/obj/cxx-dbg/mycpp/c.o', - '_build/obj/cxx-dbg/mycpp/d.o', - '_build/obj/cxx-dbg/mycpp/e.o', - ], - sorted(b.inputs)) + log('generated %d targets', n.num_build_targets()) - def testCircularDeps(self): - # Should be disallowed I think - pass - - def testSourcesForBinary(self): - n, ru = self._Rules() + def test_asdl(self): + n, ru = self._Rules() + ru.asdl_library('mycpp/examples/foo.asdl') - ru.cc_library('//mycpp/y', srcs = ['mycpp/y.cc', 'mycpp/y2.cc']) - ru.cc_library('//mycpp/z', srcs = ['mycpp/z.cc'], deps = ['//mycpp/y']) - - # cc_library() is lazy - self.assertEqual(0, len(n.build_calls)) - - ru.cc_binary( - 'mycpp/a_test.cc', deps = ['//mycpp/z'], matrix = MATRIX) - - ru.WriteRules() - - self.assertEqual(11, len(n.build_calls)) - - srcs = ru.SourcesForBinary('mycpp/a_test.cc') - self.assertEqual( - ['mycpp/a_test.cc', 'mycpp/y.cc', 'mycpp/y2.cc', 'mycpp/z.cc'], - srcs) - - log('generated %d targets', n.num_build_targets()) - - def test_asdl(self): - n, ru = self._Rules() - ru.asdl_library('mycpp/examples/foo.asdl') - - self.assertEqual(1, len(n.build_calls)) - - first = n.build_calls[0] - self.assertEqual('asdl-cpp', first.rule) - - # ru.asdl_library('mycpp/examples/foo.asdl', pretty_print_methods=False) - - def test_cc_binary_to_asdl(self): - n, ru = self._Rules() - - ru.asdl_library('asdl/hnode.asdl', pretty_print_methods = False) # REQUIRED - ru.asdl_library('display/pretty.asdl') - - ru.asdl_library('mycpp/examples/expr.asdl') - - ru.cc_binary( - '_gen/mycpp/examples/parse.mycpp.cc', - deps = ['//mycpp/examples/expr.asdl'], - matrix = MATRIX1) - - ru.WriteRules() - - actions = [b.rule for b in n.build_calls] - print(actions) - self.assertEqual([ - 'asdl-cpp', - 'asdl-cpp', - 'asdl-cpp', - 'compile_one', - 'compile_one', - 'compile_one', - 'link'], - actions) - - compile_parse = CallFor(n, '_build/obj/cxx-dbg/_gen/mycpp/examples/parse.mycpp.o') - - # Important implicit dependencies on generated headers! - self.assertEqual([ - '_gen/asdl/hnode.asdl.h', - '_gen/display/pretty.asdl.h', - '_gen/mycpp/examples/expr.asdl.h', - ], - compile_parse.implicit) - - last = n.build_calls[-1] - - self.assertEqual([ - '_build/obj/cxx-dbg/_gen/mycpp/examples/parse.mycpp.o', - '_build/obj/cxx-dbg/_gen/display/pretty.asdl.o', - '_build/obj/cxx-dbg/_gen/mycpp/examples/expr.asdl.o', - ], - last.inputs) - - def test_asdl_to_asdl(self): - n, ru = self._Rules() - - ru.asdl_library('asdl/hnode.asdl', pretty_print_methods = False) # REQUIRED - ru.asdl_library('display/pretty.asdl') - - ru.asdl_library('asdl/examples/demo_lib.asdl') - - # 'use' in ASDL creates this dependency - ru.asdl_library( - 'asdl/examples/typed_demo.asdl', - deps = ['//asdl/examples/demo_lib.asdl']) - - actions = [call.rule for call in n.build_calls] - self.assertEqual(['asdl-cpp', 'asdl-cpp', 'asdl-cpp', 'asdl-cpp'], actions) + self.assertEqual(1, len(n.build_calls)) + + first = n.build_calls[0] + self.assertEqual('asdl-cpp', first.rule) + + # ru.asdl_library('mycpp/examples/foo.asdl', pretty_print_methods=False) + + def test_cc_binary_to_asdl(self): + n, ru = self._Rules() + + ru.asdl_library('asdl/hnode.asdl', + pretty_print_methods=False) # REQUIRED + ru.asdl_library('display/pretty.asdl') + + ru.asdl_library('mycpp/examples/expr.asdl') + + ru.cc_binary('_gen/mycpp/examples/parse.mycpp.cc', + deps=['//mycpp/examples/expr.asdl'], + matrix=MATRIX1) + + ru.WriteRules() + + actions = [b.rule for b in n.build_calls] + print(actions) + self.assertEqual([ + 'asdl-cpp', 'asdl-cpp', 'asdl-cpp', 'compile_one', 'compile_one', + 'compile_one', 'link' + ], actions) + + compile_parse = CallFor( + n, '_build/obj/cxx-dbg/_gen/mycpp/examples/parse.mycpp.o') + + # Important implicit dependencies on generated headers! + self.assertEqual([ + '_gen/asdl/hnode.asdl.h', + '_gen/display/pretty.asdl.h', + '_gen/mycpp/examples/expr.asdl.h', + ], compile_parse.implicit) + + last = n.build_calls[-1] + + self.assertEqual([ + '_build/obj/cxx-dbg/_gen/mycpp/examples/parse.mycpp.o', + '_build/obj/cxx-dbg/_gen/display/pretty.asdl.o', + '_build/obj/cxx-dbg/_gen/mycpp/examples/expr.asdl.o', + ], last.inputs) - ru.cc_binary( - 'asdl/gen_cpp_test.cc', - deps = ['//asdl/examples/typed_demo.asdl'], - matrix = MATRIX1) - - ru.WriteRules() - - actions = [call.rule for call in n.build_calls] - print(actions) - self.assertEqual([ - 'asdl-cpp', 'asdl-cpp', 'asdl-cpp', 'asdl-cpp', - 'compile_one', - 'compile_one', # compile demo_lib - 'compile_one', # compile typed_demo - 'compile_one', # compile gen_cpp_test - 'link', - ], - actions) - - c = CallFor(n, '_build/obj/cxx-dbg/_gen/asdl/examples/typed_demo.asdl.o') - print(c) - - # typed_demo depends on demo_lib, so compiling typed_demo.asdl.c depends on - # the header demo_lib.asdl.h - self.assertEqual( - [ '_gen/asdl/examples/demo_lib.asdl.h', - '_gen/asdl/hnode.asdl.h', - '_gen/display/pretty.asdl.h' ], - sorted(c.implicit)) - - c = CallFor(n, '_build/obj/cxx-dbg/asdl/gen_cpp_test.o') - print(c) - print(c.implicit) - self.assertEqual( - [ '_gen/asdl/examples/demo_lib.asdl.h', - '_gen/asdl/examples/typed_demo.asdl.h', - '_gen/asdl/hnode.asdl.h', - '_gen/display/pretty.asdl.h', - ], - sorted(c.implicit)) - - def testShWrap(self): - # TODO: Rename to py_binary or py_tool - pass + def test_asdl_to_asdl(self): + n, ru = self._Rules() + + ru.asdl_library('asdl/hnode.asdl', + pretty_print_methods=False) # REQUIRED + ru.asdl_library('display/pretty.asdl') + + ru.asdl_library('asdl/examples/demo_lib.asdl') + + # 'use' in ASDL creates this dependency + ru.asdl_library('asdl/examples/typed_demo.asdl', + deps=['//asdl/examples/demo_lib.asdl']) + + actions = [call.rule for call in n.build_calls] + self.assertEqual(['asdl-cpp', 'asdl-cpp', 'asdl-cpp', 'asdl-cpp'], + actions) + + ru.cc_binary('asdl/gen_cpp_test.cc', + deps=['//asdl/examples/typed_demo.asdl'], + matrix=MATRIX1) + + ru.WriteRules() + + actions = [call.rule for call in n.build_calls] + print(actions) + self.assertEqual( + [ + 'asdl-cpp', + 'asdl-cpp', + 'asdl-cpp', + 'asdl-cpp', + 'compile_one', + 'compile_one', # compile demo_lib + 'compile_one', # compile typed_demo + 'compile_one', # compile gen_cpp_test + 'link', + ], + actions) + + c = CallFor(n, + '_build/obj/cxx-dbg/_gen/asdl/examples/typed_demo.asdl.o') + print(c) + + # typed_demo depends on demo_lib, so compiling typed_demo.asdl.c depends on + # the header demo_lib.asdl.h + self.assertEqual([ + '_gen/asdl/examples/demo_lib.asdl.h', '_gen/asdl/hnode.asdl.h', + '_gen/display/pretty.asdl.h' + ], sorted(c.implicit)) + + c = CallFor(n, '_build/obj/cxx-dbg/asdl/gen_cpp_test.o') + print(c) + print(c.implicit) + self.assertEqual([ + '_gen/asdl/examples/demo_lib.asdl.h', + '_gen/asdl/examples/typed_demo.asdl.h', + '_gen/asdl/hnode.asdl.h', + '_gen/display/pretty.asdl.h', + ], sorted(c.implicit)) + + def testShWrap(self): + # TODO: Rename to py_binary or py_tool + pass if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/build/ninja_main.py b/build/ninja_main.py index 8f89da692c..7a5447c0a8 100755 --- a/build/ninja_main.py +++ b/build/ninja_main.py @@ -31,75 +31,74 @@ from vendor import ninja_syntax - # The file Ninja runs by default. BUILD_NINJA = 'build.ninja' def TarballManifest(cc_h_files): - names = [] - - # Code we know about - names.extend(cc_h_files) - - names.extend([ - # Text - 'LICENSE.txt', - 'README-native.txt', - 'INSTALL.txt', - 'configure', - 'install', - 'doc/osh.1', - - # Build Scripts - 'build/common.sh', - 'build/native.sh', - - # These 2 are used by build/ninja-rules-cpp.sh - 'build/py2.sh', - 'build/dev-shell.sh', - - 'build/ninja-rules-cpp.sh', - 'mycpp/common.sh', - - # Generated - '_build/oils.sh', - - # These are in build/py.sh, not Ninja. Should probably put them in Ninja. - #'_gen/frontend/help_meta.h', - '_gen/frontend/match.re2c.h', - '_gen/frontend/id_kind.asdl_c.h', - '_gen/frontend/types.asdl_c.h', + names = [] + + # Code we know about + names.extend(cc_h_files) + + names.extend([ + # Text + 'LICENSE.txt', + 'README-native.txt', + 'INSTALL.txt', + 'configure', + 'install', + 'doc/osh.1', + + # Build Scripts + 'build/common.sh', + 'build/native.sh', + + # These 2 are used by build/ninja-rules-cpp.sh + 'build/py2.sh', + 'build/dev-shell.sh', + 'build/ninja-rules-cpp.sh', + 'mycpp/common.sh', + + # Generated + '_build/oils.sh', + + # These are in build/py.sh, not Ninja. Should probably put them in Ninja. + #'_gen/frontend/help_meta.h', + '_gen/frontend/match.re2c.h', + '_gen/frontend/id_kind.asdl_c.h', + '_gen/frontend/types.asdl_c.h', ]) - # For configure - names.extend(glob('build/detect-*.c')) + # For configure + names.extend(glob('build/detect-*.c')) - # TODO: crawl headers - # We can now use the headers=[] attribute - names.extend(glob('mycpp/*.h')) - names.extend(glob('cpp/*.h')) + # TODO: crawl headers + # We can now use the headers=[] attribute + names.extend(glob('mycpp/*.h')) + names.extend(glob('cpp/*.h')) - # ONLY the headers - names.extend(glob('prebuilt/*/*.h')) + # ONLY the headers + names.extend(glob('prebuilt/*/*.h')) - names.sort() # Pass them to tar sorted + names.sort() # Pass them to tar sorted - # Check for dupes here - unique = sorted(set(names)) - if names != unique: - dupes = [n for n in names if names.count(n) > 1] - raise AssertionError("Tarball manifest shouldn't have duplicates: %s" % dupes) + # Check for dupes here + unique = sorted(set(names)) + if names != unique: + dupes = [n for n in names if names.count(n) > 1] + raise AssertionError("Tarball manifest shouldn't have duplicates: %s" % + dupes) - for name in names: - print(name) + for name in names: + print(name) def ShellFunctions(cc_sources, f, argv0): - """ - Generate a shell script that invokes the same function that build.ninja does - """ - print('''\ + """ + Generate a shell script that invokes the same function that build.ninja does + """ + print('''\ #!/bin/sh # # _build/oils.sh - generated by %s @@ -134,12 +133,13 @@ def ShellFunctions(cc_sources, f, argv0): local compiler=${1:-cxx} # default is system compiler local variant=${2:-opt} # default is optimized build local skip_rebuild=${3:-} # if the output exists, skip build' -''' % (argv0), file=f) +''' % (argv0), + file=f) - out_dir = '_bin/$compiler-$variant-sh' - print(' local out_dir=%s' % out_dir, file=f) + out_dir = '_bin/$compiler-$variant-sh' + print(' local out_dir=%s' % out_dir, file=f) - print('''\ + print('''\ local out=$out_dir/oils-for-unix if test -n "$skip_rebuild" && test -f "$out"; then @@ -153,71 +153,73 @@ def ShellFunctions(cc_sources, f, argv0): echo "$0: Building oils-for-unix: $out" echo "$0: PWD = $PWD" echo -''', file=f) - - objects = [] - - in_out = [] - for src in sorted(cc_sources): - # e.g. _build/obj/cxx-dbg-sh/posix.o - prefix, _ = os.path.splitext(src) - obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix - in_out.append((src, obj)) - - bin_dir = '_bin/$compiler-$variant-sh' - obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out)) - - all_dirs = [bin_dir] + obj_dirs - # Double quote - all_dirs = ['"%s"' % d for d in all_dirs] - - print(' mkdir -p \\', file=f) - print(' %s' % ' \\\n '.join(all_dirs), file=f) - print('', file=f) - - do_fork = '' - - for i, (src, obj) in enumerate(in_out): - obj_quoted = '"%s"' % obj - objects.append(obj_quoted) - - # Only fork one translation unit that we know to be slow - if 'oils_for_unix.mycpp.cc' in src: - # There should only be one forked translation unit - # It can be turned off with OILS_PARALLEL_BUILD= _build/oils - assert do_fork == '' - do_fork = '_do_fork=$OILS_PARALLEL_BUILD' - else: - do_fork = '' +''', + file=f) - if do_fork: - print(' # Potentially fork this translation unit with &', file=f) - print(' %s _compile_one "$compiler" "$variant" "" \\' % do_fork, file=f) - print(' %s %s' % (src, obj_quoted), file=f) - print('', file=f) + objects = [] + + in_out = [] + for src in sorted(cc_sources): + # e.g. _build/obj/cxx-dbg-sh/posix.o + prefix, _ = os.path.splitext(src) + obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix + in_out.append((src, obj)) - print(' # wait for the translation unit before linking', file=f) - print(' echo WAIT', file=f) - # time -p shows any excess parallelism on 2 cores - # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all - # other translation units combined! + bin_dir = '_bin/$compiler-$variant-sh' + obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out)) - # Timing isn't POSIX - #print(' time -p wait', file=f) - print(' wait', file=f) - print('', file=f) + all_dirs = [bin_dir] + obj_dirs + # Double quote + all_dirs = ['"%s"' % d for d in all_dirs] - print(' echo "LINK $out"', file=f) - # note: can't have spaces in filenames - print(' link "$compiler" "$variant" "" "$out" \\', file=f) - # put each object on its own line, and indent by 4 - print(' %s' % (' \\\n '.join(objects)), file=f) - print('', file=f) + print(' mkdir -p \\', file=f) + print(' %s' % ' \\\n '.join(all_dirs), file=f) + print('', file=f) + + do_fork = '' + + for i, (src, obj) in enumerate(in_out): + obj_quoted = '"%s"' % obj + objects.append(obj_quoted) + + # Only fork one translation unit that we know to be slow + if 'oils_for_unix.mycpp.cc' in src: + # There should only be one forked translation unit + # It can be turned off with OILS_PARALLEL_BUILD= _build/oils + assert do_fork == '' + do_fork = '_do_fork=$OILS_PARALLEL_BUILD' + else: + do_fork = '' + + if do_fork: + print(' # Potentially fork this translation unit with &', file=f) + print(' %s _compile_one "$compiler" "$variant" "" \\' % do_fork, + file=f) + print(' %s %s' % (src, obj_quoted), file=f) + print('', file=f) + + print(' # wait for the translation unit before linking', file=f) + print(' echo WAIT', file=f) + # time -p shows any excess parallelism on 2 cores + # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all + # other translation units combined! + + # Timing isn't POSIX + #print(' time -p wait', file=f) + print(' wait', file=f) + print('', file=f) - # Strip opt binary - # TODO: provide a way for the user to get symbols? + print(' echo "LINK $out"', file=f) + # note: can't have spaces in filenames + print(' link "$compiler" "$variant" "" "$out" \\', file=f) + # put each object on its own line, and indent by 4 + print(' %s' % (' \\\n '.join(objects)), file=f) + print('', file=f) - print('''\ + # Strip opt binary + # TODO: provide a way for the user to get symbols? + + print('''\ local out_name=oils-for-unix if test "$variant" = opt; then strip -o "$out.stripped" "$out" @@ -235,201 +237,208 @@ def ShellFunctions(cc_sources, f, argv0): } main "$@" -''', file=f) +''', + file=f) def Preprocessed(n, cc_sources): - # See how much input we're feeding to the compiler. Test C++ template - # explosion, e.g. - # - # Limit to {dbg,opt} so we don't generate useless rules. Invoked by - # metrics/source-code.sh - - pre_matrix = [ - ('cxx', 'dbg'), - ('cxx', 'opt'), - ('clang', 'dbg'), - ('clang', 'opt'), - ] - for compiler, variant in pre_matrix: - preprocessed = [] - for src in cc_sources: - # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc - pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src) - preprocessed.append(pre) - - # Summary file - n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant), - 'line_count', - preprocessed) - n.newline() + # See how much input we're feeding to the compiler. Test C++ template + # explosion, e.g. + # + # Limit to {dbg,opt} so we don't generate useless rules. Invoked by + # metrics/source-code.sh + + pre_matrix = [ + ('cxx', 'dbg'), + ('cxx', 'opt'), + ('clang', 'dbg'), + ('clang', 'opt'), + ] + for compiler, variant in pre_matrix: + preprocessed = [] + for src in cc_sources: + # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc + pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src) + preprocessed.append(pre) + + # Summary file + n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant), + 'line_count', preprocessed) + n.newline() def InitSteps(n): - """Wrappers for build/ninja-rules-*.sh - - Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here. - """ - - # - # Compiling and linking - # - - # Preprocess one translation unit - n.rule('preprocess', - # compile_one detects the _build/preprocessed path - command='build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out', - description='PP $compiler $variant $more_cxx_flags $in $out') - n.newline() - - n.rule('line_count', - command='build/ninja-rules-cpp.sh line_count $out $in', - description='line_count $out $in') - n.newline() - - # Compile one translation unit - n.rule('compile_one', - command='build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d', - depfile='$out.d', - # no prefix since the compiler is the first arg - description='$compiler $variant $more_cxx_flags $in $out') - n.newline() - - # Link objects together - n.rule('link', - command='build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in', - description='LINK $compiler $variant $more_link_flags $out $in') - n.newline() - - # 1 input and 2 outputs - n.rule('strip', - command='build/ninja-rules-cpp.sh strip_ $in $out', - description='STRIP $in $out') - n.newline() - - # cc_binary can have symliks - n.rule('symlink', - command='build/ninja-rules-cpp.sh symlink $dir $target $new', - description='SYMLINK $dir $target $new') - n.newline() - - # - # Code generators - # - - n.rule('write-shwrap', - # $in must start with main program - command='build/ninja-rules-py.sh write-shwrap $template $out $in', - description='make-pystub $out $in') - n.newline() - - n.rule('gen-oils-for-unix', - command='build/ninja-rules-py.sh gen-oils-for-unix $main_name $out_prefix $preamble $in', - description='gen-oils-for-unix $main_name $out_prefix $preamble $in') - n.newline() + """Wrappers for build/ninja-rules-*.sh + + Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here. + """ + # + # Compiling and linking + # + + # Preprocess one translation unit + n.rule( + 'preprocess', + # compile_one detects the _build/preprocessed path + command= + 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out', + description='PP $compiler $variant $more_cxx_flags $in $out') + n.newline() + n.rule('line_count', + command='build/ninja-rules-cpp.sh line_count $out $in', + description='line_count $out $in') + n.newline() -def main(argv): - try: - action = argv[1] - except IndexError: - action = 'ninja' + # Compile one translation unit + n.rule( + 'compile_one', + command= + 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d', + depfile='$out.d', + # no prefix since the compiler is the first arg + description='$compiler $variant $more_cxx_flags $in $out') + n.newline() - if action == 'ninja': - f = open(BUILD_NINJA, 'w') - else: - f = cStringIO.StringIO() # thrown away + # Link objects together + n.rule( + 'link', + command= + 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in', + description='LINK $compiler $variant $more_link_flags $out $in') + n.newline() + + # 1 input and 2 outputs + n.rule('strip', + command='build/ninja-rules-cpp.sh strip_ $in $out', + description='STRIP $in $out') + n.newline() + + # cc_binary can have symliks + n.rule('symlink', + command='build/ninja-rules-cpp.sh symlink $dir $target $new', + description='SYMLINK $dir $target $new') + n.newline() + + # + # Code generators + # + + n.rule( + 'write-shwrap', + # $in must start with main program + command='build/ninja-rules-py.sh write-shwrap $template $out $in', + description='make-pystub $out $in') + n.newline() + + n.rule( + 'gen-oils-for-unix', + command= + 'build/ninja-rules-py.sh gen-oils-for-unix $main_name $out_prefix $preamble $in', + description='gen-oils-for-unix $main_name $out_prefix $preamble $in') + n.newline() - n = ninja_syntax.Writer(f) - ru = ninja_lib.Rules(n) - ru.comment('InitSteps()') - InitSteps(n) +def main(argv): + try: + action = argv[1] + except IndexError: + action = 'ninja' - # - # Create the graph. - # + if action == 'ninja': + f = open(BUILD_NINJA, 'w') + else: + f = cStringIO.StringIO() # thrown away - asdl_subgraph.NinjaGraph(ru) - ru.comment('') + n = ninja_syntax.Writer(f) + ru = ninja_lib.Rules(n) - bin_subgraph.NinjaGraph(ru) - ru.comment('') + ru.comment('InitSteps()') + InitSteps(n) - core_subgraph.NinjaGraph(ru) - ru.comment('') + # + # Create the graph. + # - cpp_subgraph.NinjaGraph(ru) - ru.comment('') + asdl_subgraph.NinjaGraph(ru) + ru.comment('') - data_lang_subgraph.NinjaGraph(ru) - ru.comment('') + bin_subgraph.NinjaGraph(ru) + ru.comment('') - display_subgraph.NinjaGraph(ru) - ru.comment('') + core_subgraph.NinjaGraph(ru) + ru.comment('') - frontend_subgraph.NinjaGraph(ru) - ru.comment('') + cpp_subgraph.NinjaGraph(ru) + ru.comment('') - mycpp_subgraph.NinjaGraph(ru) - ru.comment('') + data_lang_subgraph.NinjaGraph(ru) + ru.comment('') - ysh_subgraph.NinjaGraph(ru) - ru.comment('') + display_subgraph.NinjaGraph(ru) + ru.comment('') - osh_subgraph.NinjaGraph(ru) - ru.comment('') + frontend_subgraph.NinjaGraph(ru) + ru.comment('') - pea_subgraph.NinjaGraph(ru) - ru.comment('') + mycpp_subgraph.NinjaGraph(ru) + ru.comment('') - prebuilt_subgraph.NinjaGraph(ru) - ru.comment('') + ysh_subgraph.NinjaGraph(ru) + ru.comment('') - yaks_subgraph.NinjaGraph(ru) - ru.comment('') + osh_subgraph.NinjaGraph(ru) + ru.comment('') + pea_subgraph.NinjaGraph(ru) + ru.comment('') - # Materialize all the cc_binary() rules - ru.WriteRules() + prebuilt_subgraph.NinjaGraph(ru) + ru.comment('') - # Collect sources for metrics, tarball, shell script - cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc') + yaks_subgraph.NinjaGraph(ru) + ru.comment('') - if 0: - from pprint import pprint - pprint(cc_sources) + # Materialize all the cc_binary() rules + ru.WriteRules() - # TODO: could thin these out, not generate for unit tests, etc. - Preprocessed(n, cc_sources) + # Collect sources for metrics, tarball, shell script + cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc') - ru.WritePhony() + if 0: + from pprint import pprint + pprint(cc_sources) - n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh']) + # TODO: could thin these out, not generate for unit tests, etc. + Preprocessed(n, cc_sources) - if action == 'ninja': - log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA, - n.num_build_targets()) + ru.WritePhony() - elif action == 'shell': - out = '_build/oils.sh' - with open(out, 'w') as f: - ShellFunctions(cc_sources, f, argv[0]) - log(' (%s) -> %s', argv[0], out) + n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh']) - elif action == 'tarball-manifest': - h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc') - TarballManifest(cc_sources + h) + if action == 'ninja': + log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA, + n.num_build_targets()) - else: - raise RuntimeError('Invalid action %r' % action) + elif action == 'shell': + out = '_build/oils.sh' + with open(out, 'w') as f: + ShellFunctions(cc_sources, f, argv[0]) + log(' (%s) -> %s', argv[0], out) + + elif action == 'tarball-manifest': + h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc') + TarballManifest(cc_sources + h) + + else: + raise RuntimeError('Invalid action %r' % action) if __name__ == '__main__': - try: - main(sys.argv) - except RuntimeError as e: - print('FATAL: %s' % e, file=sys.stderr) - sys.exit(1) + try: + main(sys.argv) + except RuntimeError as e: + print('FATAL: %s' % e, file=sys.stderr) + sys.exit(1) # vim: sw=2 diff --git a/demo/url-search-params.ysh b/demo/url-search-params.ysh index 7477cb746c..91c14bac45 100755 --- a/demo/url-search-params.ysh +++ b/demo/url-search-params.ysh @@ -31,8 +31,6 @@ # # - need Vim syntax highlighting! # - e.g. multiline '' strings aren't higlighted -# - need pp [x] for debugging -# - need assert [x] for testing # - task files need completion # # - Eggex can use multiline /// syntax, though you can use \ for line continuation @@ -46,8 +44,8 @@ # # - ERROR messages for URL parsing should bubble up to the user! # - USER code should be able to point out to location info for bad escapes -# like %f or %0z -# - I guess we just need an idiom for this? A "class"? +# like %f or %0z +# - I guess we just need an idiom for this? source $LIB_OSH/task-five.sh #source $LIB_YSH/yblocks.ysh From cd15fc6c793622775c76177ca7dc0482436d93ce Mon Sep 17 00:00:00 2001 From: Christian Bourgeois Date: Sun, 28 Jul 2024 05:29:41 +0200 Subject: [PATCH 071/506] [osh] Implement set -o noclobber (#2008) - More conservative error message tweak, which retains the code snippet - Add extra message if EEXIST and noclobber is on - Improve spec tests. We don't need to use $TMP, since the tests are now started in that dir --------- Co-authored-by: Christian Bourgeois Co-authored-by: Andy C --- core/process.py | 25 ++++++++++----- core/process_test.py | 3 +- core/shell.py | 3 +- core/test_lib.py | 3 +- spec/redirect.test.sh | 2 +- spec/sh-options.test.sh | 69 ++++++++++++++++++++++++++++++++--------- 6 files changed, 79 insertions(+), 26 deletions(-) diff --git a/core/process.py b/core/process.py index e967543ca6..0d35f87ff5 100644 --- a/core/process.py +++ b/core/process.py @@ -9,7 +9,7 @@ """ from __future__ import print_function -from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC +from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC, EEXIST import fcntl as fcntl_ from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP, @@ -54,6 +54,7 @@ WNOHANG, O_APPEND, O_CREAT, + O_EXCL, O_NONBLOCK, O_NOCTTY, O_RDONLY, @@ -179,9 +180,10 @@ def __init__( errfmt, # type: ui.ErrorFormatter job_control, # type: JobControl job_list, # type: JobList - mem, #type: state.Mem + mem, # type: state.Mem tracer, # type: Optional[dev.Tracer] waiter, # type: Optional[Waiter] + exec_opts, # type: optview.Exec ): # type: (...) -> None """ @@ -197,6 +199,7 @@ def __init__( self.mem = mem self.tracer = tracer self.waiter = waiter + self.exec_opts = exec_opts def Open(self, path): # type: (str) -> mylib.LineReader @@ -377,16 +380,17 @@ def _ApplyRedirect(self, r): if case(redirect_arg_e.Path): arg = cast(redirect_arg.Path, UP_arg) - + # noclobber flag is OR'd with other flags when allowed + noclobber_mode = O_EXCL if self.exec_opts.noclobber() else 0 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &> # NOTE: This is different than >| because it respects noclobber, but # that option is almost never used. See test/wild.sh. - mode = O_CREAT | O_WRONLY | O_TRUNC + mode = O_CREAT | O_WRONLY | O_TRUNC | noclobber_mode elif r.op_id == Id.Redir_Clobber: # >| mode = O_CREAT | O_WRONLY | O_TRUNC elif r.op_id in (Id.Redir_DGreat, Id.Redir_AndDGreat): # >> &>> - mode = O_CREAT | O_WRONLY | O_APPEND + mode = O_CREAT | O_WRONLY | O_APPEND | noclobber_mode elif r.op_id == Id.Redir_Less: # < mode = O_RDONLY elif r.op_id == Id.Redir_LessGreat: # <> @@ -398,9 +402,14 @@ def _ApplyRedirect(self, r): try: open_fd = posix.open(arg.filename, mode, 0o666) except (IOError, OSError) as e: - self.errfmt.Print_("Can't open %r: %s" % - (arg.filename, pyutil.strerror(e)), - blame_loc=r.op_loc) + if e.errno == EEXIST and self.exec_opts.noclobber(): + extra = ' (noclobber)' + else: + extra = '' + self.errfmt.Print_( + "Can't open %r: %s%s" % + (arg.filename, pyutil.strerror(e), extra), + blame_loc=r.op_loc) raise # redirect failed new_fd = self._PushDup(open_fd, r.loc) diff --git a/core/process_test.py b/core/process_test.py index 7c75d591cf..c8b6377161 100755 --- a/core/process_test.py +++ b/core/process_test.py @@ -71,7 +71,8 @@ def setUp(self): self.tracer) errfmt = ui.ErrorFormatter() self.fd_state = process.FdState(errfmt, self.job_control, - self.job_list, None, self.tracer, None) + self.job_list, None, self.tracer, None, + exec_opts) self.ext_prog = process.ExternalProgram('', self.fd_state, errfmt, util.NullDebugFile()) diff --git a/core/shell.py b/core/shell.py index 2c5f0b578b..818416c4a6 100644 --- a/core/shell.py +++ b/core/shell.py @@ -418,7 +418,8 @@ def Main( job_control = process.JobControl() job_list = process.JobList() - fd_state = process.FdState(errfmt, job_control, job_list, mem, None, None) + fd_state = process.FdState(errfmt, job_control, job_list, mem, None, None, + exec_opts) my_pid = posix.getpid() diff --git a/core/test_lib.py b/core/test_lib.py index f7a686ec65..67e6379cde 100644 --- a/core/test_lib.py +++ b/core/test_lib.py @@ -213,7 +213,8 @@ def InitCommandEvaluator(parse_ctx=None, errfmt = ui.ErrorFormatter() job_control = process.JobControl() job_list = process.JobList() - fd_state = process.FdState(errfmt, job_control, job_list, None, None, None) + fd_state = process.FdState(errfmt, job_control, job_list, None, None, None, + exec_opts) aliases = {} if aliases is None else aliases procs = state.Procs(mem) methods = {} diff --git a/spec/redirect.test.sh b/spec/redirect.test.sh index fa334e81a5..b3b303dc3d 100644 --- a/spec/redirect.test.sh +++ b/spec/redirect.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 2 +## oils_failures_allowed: 1 ## compare_shells: bash dash mksh #### >& and <& are the same diff --git a/spec/sh-options.test.sh b/spec/sh-options.test.sh index d7f2ce53b9..5d0d666a54 100644 --- a/spec/sh-options.test.sh +++ b/spec/sh-options.test.sh @@ -1,7 +1,7 @@ # Test set flags, sh flags. ## compare_shells: bash dash mksh -## oils_failures_allowed: 3 +## oils_failures_allowed: 2 ## tags: interactive #### $- with -c @@ -315,24 +315,65 @@ failglob #### noclobber off set -o errexit -echo foo > $TMP/can-clobber + +echo foo > can-clobber +echo status=$? set +C -echo foo > $TMP/can-clobber + +echo foo > can-clobber +echo status=$? set +o noclobber -echo foo > $TMP/can-clobber -cat $TMP/can-clobber -## stdout: foo + +echo foo > can-clobber +echo status=$? +cat can-clobber + +## STDOUT: +status=0 +status=0 +status=0 +foo +## END #### noclobber on -# Not implemented yet. -rm $TMP/no-clobber + +rm -f no-clobber set -C -echo foo > $TMP/no-clobber -echo $? -echo foo > $TMP/no-clobber -echo $? -## stdout-json: "0\n1\n" -## OK dash stdout-json: "0\n2\n" + +echo foo > no-clobber +echo create=$? + +echo overwrite > no-clobber +echo overwrite=$? + +echo force >| no-clobber +echo force=$? + +cat no-clobber + +## STDOUT: +create=0 +overwrite=1 +force=0 +force +## END +## OK dash STDOUT: +create=0 +overwrite=2 +force=0 +force +## END + +#### noclobber on <> +set -C +echo foo >| $TMP/no-clobber +exec 3<> $TMP/no-clobber +read -n 1 <&3 +echo -n . >&3 +exec 3>&- +cat $TMP/no-clobber +## stdout-json: "f.o\n" +## N-I dash stdout-json: ".oo\n" #### SHELLOPTS is updated when options are changed echo $SHELLOPTS | grep -q xtrace From 6b3305a2b265c2ca95274d34cc57102693df00c2 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 00:00:07 -0400 Subject: [PATCH 072/506] [soil] Switch back to Dreamhost Mythic Beasts SSH was rejecting our concurrent connections --- soil/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/soil/common.sh b/soil/common.sh index 1dc85b71ae..c85d39b4c8 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -20,12 +20,12 @@ dump-env() { env | grep -v '^encrypted_' | sort } -if false; then +if true; then readonly SOIL_USER='travis_admin' readonly SOIL_HOST='travis-ci.oilshell.org' readonly SOIL_HOST_DIR=~/travis-ci.oilshell.org # used on server readonly SOIL_REMOTE_DIR=travis-ci.oilshell.org # used on client -elif true; then +elif false; then readonly SOIL_USER='oils' readonly SOIL_HOST='mb.oils.pub' # Extra level From a2f0ae0f1ac0acff89f8ef2016dd9cfea0681765 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 02:10:51 -0400 Subject: [PATCH 073/506] [osh] Implement first part of set -o errtrace Traps can be inherited in subshells. Makes a test in spec/builtin-trap-err pass. Based on PR #2014 --- Co-authored-by: Christian Bourgeois --- builtin/trap_osh.py | 12 ++++++++---- core/executor.py | 26 ++++++++++++-------------- core/process.py | 13 +++++-------- core/process_test.py | 15 ++++++++++----- doc/ref/chap-option.md | 8 +++++--- doc/ref/toc-osh.md | 2 +- frontend/option_def.py | 1 + spec/builtin-trap-err.test.sh | 2 +- 8 files changed, 43 insertions(+), 36 deletions(-) diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index 10e18ba6a0..5e7f94f401 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -47,13 +47,17 @@ def __init__(self, signal_safe): self.hooks = {} # type: Dict[str, command_t] self.traps = {} # type: Dict[int, command_t] - def ClearForSubProgram(self): - # type: () -> None + def ClearForSubProgram(self, inherit_errtrace): + # type: (bool) -> None """SubProgramThunk uses this because traps aren't inherited.""" - # bash clears DEBUG hook in subshell, command sub, etc. See - # spec/builtin-trap-bash. + # bash clears hooks like DEBUG in subshells. + # The ERR can be preserved if set -o errtrace + hook_err = self.hooks.get('ERR') self.hooks.clear() + if hook_err is not None and inherit_errtrace: + self.hooks['ERR'] = hook_err + self.traps.clear() def GetHook(self, hook_name): diff --git a/core/executor.py b/core/executor.py index b4dd04543b..2c31415278 100644 --- a/core/executor.py +++ b/core/executor.py @@ -157,8 +157,8 @@ def CheckCircularDeps(self): # type: () -> None assert self.cmd_ev is not None - def _MakeProcess(self, node, inherit_errexit=True): - # type: (command_t, bool) -> process.Process + def _MakeProcess(self, node, inherit_errexit, inherit_errtrace): + # type: (command_t, bool, bool) -> process.Process """Assume we will run the node in another process. Return a process. @@ -184,11 +184,9 @@ def _MakeProcess(self, node, inherit_errexit=True): # interleaved. # - We could turn the `exit` builtin into a error.FatalRuntime exception # and get this check for "free". - thunk = process.SubProgramThunk(self.cmd_ev, - node, - self.trap_state, - self.multi_trace, - inherit_errexit=inherit_errexit) + thunk = process.SubProgramThunk(self.cmd_ev, node, self.trap_state, + self.multi_trace, inherit_errexit, + inherit_errtrace) p = process.Process(thunk, self.job_control, self.job_list, self.tracer) return p @@ -395,7 +393,7 @@ def RunBackgroundJob(self, node): pi = process.Pipeline(self.exec_opts.sigpipe_status_ok(), self.job_control, self.job_list, self.tracer) for child in node.children: - p = self._MakeProcess(child) + p = self._MakeProcess(child, True, self.exec_opts.errtrace()) p.Init_ParentPipeline(pi) pi.Add(p) @@ -411,7 +409,7 @@ def RunBackgroundJob(self, node): # have to register SIGCHLD. But then that introduces race conditions. # If we haven't called Register yet, then we won't know who to notify. - p = self._MakeProcess(node) + p = self._MakeProcess(node, True, self.exec_opts.errtrace()) if self.job_control.Enabled(): p.AddStateChange( process.SetPgid(process.OWN_LEADER, self.tracer)) @@ -439,7 +437,7 @@ def RunPipeline(self, node, status_out): # TODO: determine these locations at parse time? pipe_locs.append(loc.Command(child)) - p = self._MakeProcess(child) + p = self._MakeProcess(child, True, self.exec_opts.errtrace()) p.Init_ParentPipeline(pi) pi.Add(p) @@ -458,7 +456,7 @@ def RunPipeline(self, node, status_out): def RunSubshell(self, node): # type: (command_t) -> int - p = self._MakeProcess(node) + p = self._MakeProcess(node, True, self.exec_opts.errtrace()) if self.job_control.Enabled(): p.AddStateChange(process.SetPgid(process.OWN_LEADER, self.tracer)) @@ -500,8 +498,8 @@ def RunCommandSub(self, cs_part): # MUTATE redir node so it's like $( None + def __init__(self, cmd_ev, node, trap_state, multi_trace, inherit_errexit, + inherit_errtrace): + # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool, bool) -> None self.cmd_ev = cmd_ev self.node = node self.trap_state = trap_state self.multi_trace = multi_trace self.inherit_errexit = inherit_errexit # for bash errexit compatibility + self.inherit_errtrace = inherit_errtrace # for bash errtrace compatibility def UserString(self): # type: () -> str @@ -839,7 +836,7 @@ def Run(self): from osh import cmd_eval # signal handlers aren't inherited - self.trap_state.ClearForSubProgram() + self.trap_state.ClearForSubProgram(self.inherit_errtrace) # NOTE: may NOT return due to exec(). if not self.inherit_errexit: diff --git a/core/process_test.py b/core/process_test.py index c8b6377161..40fec10619 100755 --- a/core/process_test.py +++ b/core/process_test.py @@ -182,9 +182,12 @@ def testPipeline2(self): node2 = _CommandNode('head', self.arena) node3 = _CommandNode('sort --reverse', self.arena) - thunk1 = process.SubProgramThunk(cmd_ev, node1, self.trap_state, None) - thunk2 = process.SubProgramThunk(cmd_ev, node2, self.trap_state, None) - thunk3 = process.SubProgramThunk(cmd_ev, node3, self.trap_state, None) + thunk1 = process.SubProgramThunk(cmd_ev, node1, self.trap_state, None, + True, False) + thunk2 = process.SubProgramThunk(cmd_ev, node2, self.trap_state, None, + True, False) + thunk3 = process.SubProgramThunk(cmd_ev, node3, self.trap_state, None, + True, False) p = process.Pipeline(False, self.job_control, self.job_list, self.tracer) @@ -222,8 +225,10 @@ def makeTestPipeline(self, jc): node1 = _CommandNode('/bin/echo testpipeline', self.arena) node2 = _CommandNode('cat', self.arena) - thunk1 = process.SubProgramThunk(cmd_ev, node1, self.trap_state, None) - thunk2 = process.SubProgramThunk(cmd_ev, node2, self.trap_state, None) + thunk1 = process.SubProgramThunk(cmd_ev, node1, self.trap_state, None, + True, False) + thunk2 = process.SubProgramThunk(cmd_ev, node2, self.trap_state, None, + True, False) pi.Add(Process(thunk1, jc, self.job_list, self.tracer)) pi.Add(Process(thunk2, jc, self.job_list, self.tracer)) diff --git a/doc/ref/chap-option.md b/doc/ref/chap-option.md index 59c913ac1d..f8365cba8a 100644 --- a/doc/ref/chap-option.md +++ b/doc/ref/chap-option.md @@ -82,6 +82,11 @@ called `-rf`. $ echo * myfile +## Other Option + + noclobber -C # Redirects can't overwrite files + errtrace -E # Enable ERR trap is both shell functions and subshells + ## Debugging These options are from POSIX shell: @@ -98,9 +103,6 @@ These options are from bash. emacs vi -## Other Option - - noclobber # Redirects don't overwrite files ## Compat diff --git a/doc/ref/toc-osh.md b/doc/ref/toc-osh.md index 94a1297931..e17469bc63 100644 --- a/doc/ref/toc-osh.md +++ b/doc/ref/toc-osh.md @@ -184,9 +184,9 @@ X [Unsupported] enable [Errors] nounset -u errexit -e inherit_errexit pipefail [Globbing] noglob -f nullglob failglob X dotglob dashglob (true) + [Other Option] noclobber -C errtrace -E [Debugging] xtrace X verbose X extdebug [Interactive] emacs vi - [Other POSIX] X noclobber [Compat] eval_unsafe_arith ignore_flags_not_impl ``` diff --git a/frontend/option_def.py b/frontend/option_def.py index 0ff5ade888..7e20fab07e 100644 --- a/frontend/option_def.py +++ b/frontend/option_def.py @@ -69,6 +69,7 @@ def DoneWithImplementedOptions(self): ('v', 'verbose'), # like xtrace, but prints unevaluated commands ('f', 'noglob'), ('C', 'noclobber'), + ('E', 'errtrace'), # A no-op for modernish. (None, 'posix'), diff --git a/spec/builtin-trap-err.test.sh b/spec/builtin-trap-err.test.sh index c1c8ebf243..b735a95b1d 100644 --- a/spec/builtin-trap-err.test.sh +++ b/spec/builtin-trap-err.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 4 +## oils_failures_allowed: 3 ## compare_shells: bash mksh ash # Notes on bash semantics: From d8572e6616641955b7adb8c0cb44f28485baec90 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 02:13:44 -0400 Subject: [PATCH 074/506] [osh] Second part of set -o errtrace If this option is on, trap ERR runs inside functions. --- osh/cmd_eval.py | 2 +- spec/builtin-trap-err.test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 84976f7a20..9a11340a51 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -2102,7 +2102,7 @@ def _MaybeRunErrTrap(self): return # bash rule - affected by set -o errtrace - if self.mem.InsideFunction(): + if not self.exec_opts.errtrace() and self.mem.InsideFunction(): return # NOTE: Don't set option_i._running_trap, because that's for diff --git a/spec/builtin-trap-err.test.sh b/spec/builtin-trap-err.test.sh index b735a95b1d..8233d93573 100644 --- a/spec/builtin-trap-err.test.sh +++ b/spec/builtin-trap-err.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 3 +## oils_failures_allowed: 2 ## compare_shells: bash mksh ash # Notes on bash semantics: From a1b1a7bf9004cb000a0422d82d6d353d19c32650 Mon Sep 17 00:00:00 2001 From: Ellen <38250543+ellen364@users.noreply.github.com> Date: Sun, 28 Jul 2024 15:37:53 +0100 Subject: [PATCH 075/506] [builtins] Add Dict->erase() method (#2029) --- builtin/method_dict.py | 18 ++++++++++++++++++ core/shell.py | 2 +- doc/ref/chap-type-method.md | 18 ++++++++++++++++++ mycpp/cppgen_pass.py | 2 +- spec/ysh-methods.test.sh | 13 +++++++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/builtin/method_dict.py b/builtin/method_dict.py index 556ad6f099..8b7193685d 100644 --- a/builtin/method_dict.py +++ b/builtin/method_dict.py @@ -6,6 +6,7 @@ from core import vm from frontend import typed_args +from mycpp import mylib from mycpp.mylib import log from typing import List @@ -43,3 +44,20 @@ def Call(self, rd): values = dictionary.values() # type: List[value_t] return value.List(values) + + +class Erase(vm._Callable): + + def __init__(self): + # type: () -> None + pass + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + + dictionary = rd.PosDict() + key = rd.PosStr() + rd.Done() + + mylib.dict_erase(dictionary, key) + return value.Null diff --git a/core/shell.py b/core/shell.py index 818416c4a6..fb78d9f0ad 100644 --- a/core/shell.py +++ b/core/shell.py @@ -746,7 +746,7 @@ def Main( } methods[value_e.Dict] = { 'get': None, # doesn't raise an error - 'erase': None, # ensures it doesn't exist + 'erase': method_dict.Erase(), 'keys': method_dict.Keys(), 'values': method_dict.Values(), diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 23e256b7a4..b4d9378c5f 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -343,6 +343,24 @@ Similar to `keys()`, but returns the values of the dictionary. ### erase() +Ensures that the given key does not exist in the dictionary. + + var book = { + title: "The Histories", + author: "Herodotus", + } + = book + # => (Dict) {title: "The Histories", author: "Herodotus"} + + call book->erase("author") + = book + # => (Dict) {title: "The Histories"} + + # repeating the erase call does not cause an error + call book->erase("author") + = book + # => (Dict) {title: "The Histories"} + ### inc() ### accum() diff --git a/mycpp/cppgen_pass.py b/mycpp/cppgen_pass.py index 2c0015e48d..830bb10023 100644 --- a/mycpp/cppgen_pass.py +++ b/mycpp/cppgen_pass.py @@ -2223,7 +2223,7 @@ def visit_del_stmt(self, o: 'mypy.nodes.DelStmt') -> T: else: # del mydict[mykey] raises KeyError, which we don't want raise AssertionError( - 'Use mylib.maybe_remove(d, key) instead of del d[key]') + 'Use mylib.dict_erase(d, key) instead of del d[key]') self.def_write(';\n') diff --git a/spec/ysh-methods.test.sh b/spec/ysh-methods.test.sh index 1ca5ab2c21..6a899895e8 100644 --- a/spec/ysh-methods.test.sh +++ b/spec/ysh-methods.test.sh @@ -393,6 +393,19 @@ pp line (en2fr => values()) (List) ["bonjour","ami","chat"] ## END +#### Dict -> erase() +var book = {title: "The Histories", author: "Herodotus"} +call book->erase("author") +pp line (book) +# confirm method is idempotent +call book->erase("author") +pp line (book) +## status: 0 +## STDOUT: +(Dict) {"title":"The Histories"} +(Dict) {"title":"The Histories"} +## END + #### Separation of -> attr and () calling const check = "abc" => startsWith pp line (check("a")) From 52e8fe568d23e8d51e47d749f3c4ddfd027cdd9c Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 02:36:58 -0400 Subject: [PATCH 076/506] [builtin] Implement _io->captureStdout() --- builtin/method_io.py | 25 ++++++++++--- core/error.py | 19 +++++----- core/executor.py | 75 +++++++++++++++++++++---------------- core/shell.py | 2 +- core/vm.py | 6 ++- doc/ref/chap-type-method.md | 11 +++++- doc/ref/toc-ysh.md | 2 +- test/spec.sh | 4 ++ 8 files changed, 93 insertions(+), 51 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index 9eb70051ad..7857c26e3a 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -4,11 +4,12 @@ from _devbuild.gen.value_asdl import value, value_t from core import error +from core import num from core import vm from mycpp.mylib import log from osh import prompt -from typing import cast, TYPE_CHECKING +from typing import Dict, cast, TYPE_CHECKING if TYPE_CHECKING: from frontend import typed_args @@ -28,13 +29,27 @@ def Call(self, rd): class CaptureStdout(vm._Callable): - def __init__(self): - # type: () -> None - pass + def __init__(self, shell_ex): + # type: (vm._Executor) -> None + self.shell_ex = shell_ex def Call(self, rd): # type: (typed_args.Reader) -> value_t - return value.Null + + io = rd.PosIO() + cmd = rd.PosCommand() + rd.Done() # no more args + + status, stdout_str = self.shell_ex.CaptureStdout(cmd) + if status != 0: + properties = { + 'status': num.ToBig(status) + } # type: Dict[str, value_t] + raise error.Structured( + 4, 'Captured command failed with status %d' % status, + rd.LeftParenToken(), properties) + + return value.Str(stdout_str) class PromptVal(vm._Callable): diff --git a/core/error.py b/core/error.py index 411b94e958..a1affda08d 100644 --- a/core/error.py +++ b/core/error.py @@ -4,6 +4,7 @@ from _devbuild.gen.syntax_asdl import loc_e, loc_t, loc from _devbuild.gen.value_asdl import (value, value_t, value_str) from core import num +from mycpp.mylib import NewDict from typing import Dict, Union, NoReturn, TYPE_CHECKING @@ -173,19 +174,19 @@ def __init__(self, status, msg, location, properties=None): def ToDict(self): # type: () -> value.Dict - if self.properties is None: - self.properties = {} + d = NewDict() # type: Dict[str, value_t] - # Override status and message. - # The _error Dict order is a bit quirky -- the optional properties come - # before these required fields. But we always want the required fields - # to take precedence, so it makes sense. + # The _error Dict order is odd -- the optional properties come BEFORE + # required fields. We always want the required fields to be present so + # it makes sense. + if self.properties is not None: + d.update(self.properties) # _error.code is better than _error.status - self.properties['code'] = num.ToBig(self.ExitStatus()) - self.properties['message'] = value.Str(self.msg) + d['code'] = num.ToBig(self.ExitStatus()) + d['message'] = value.Str(self.msg) - return value.Dict(self.properties) + return value.Dict(d) class AssertionErr(Expr): diff --git a/core/executor.py b/core/executor.py index 2c31415278..9a70194883 100644 --- a/core/executor.py +++ b/core/executor.py @@ -30,7 +30,7 @@ import posix_ as posix -from typing import cast, Dict, List, Optional, TYPE_CHECKING +from typing import cast, Dict, List, Tuple, Optional, TYPE_CHECKING if TYPE_CHECKING: from _devbuild.gen.runtime_asdl import (cmd_value, CommandStatus, StatusArray) @@ -462,11 +462,49 @@ def RunSubshell(self, node): return p.RunProcess(self.waiter, trace.ForkWait) + def CaptureStdout(self, node): + # type: (command_t) -> Tuple[int, str] + + p = self._MakeProcess(node, self.exec_opts.inherit_errexit(), + self.exec_opts.errtrace()) + # Shell quirk: Command subs remain part of the shell's process group, so we + # don't use p.AddStateChange(process.SetPgid(...)) + + r, w = posix.pipe() + p.AddStateChange(process.StdoutToPipe(r, w)) + + p.StartProcess(trace.CommandSub) + #log('Command sub started %d', pid) + + chunks = [] # type: List[str] + posix.close(w) # not going to write + while True: + n, err_num = pyos.Read(r, 4096, chunks) + + if n < 0: + if err_num == EINTR: + pass # retry + else: + # Like the top level IOError handler + e_die_status( + 2, + 'Oils I/O error (read): %s' % posix.strerror(err_num)) + + elif n == 0: # EOF + break + posix.close(r) + + status = p.Wait(self.waiter) + stdout_str = ''.join(chunks).rstrip('\n') + + return status, stdout_str + def RunCommandSub(self, cs_part): # type: (CommandSub) -> str if not self.exec_opts._allow_command_sub(): - # _allow_command_sub is used in two places. Only one of them turns off _allow_process_sub + # _allow_command_sub is used in two places. Only one of them turns + # off _allow_process_sub if not self.exec_opts._allow_process_sub(): why = "status wouldn't be checked (strict_errexit)" else: @@ -498,36 +536,7 @@ def RunCommandSub(self, cs_part): # MUTATE redir node so it's like $( $@") # Why rstrip()? # https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char - return ''.join(chunks).rstrip('\n') + return stdout_str def RunProcessSub(self, cs_part): # type: (CommandSub) -> str diff --git a/core/shell.py b/core/shell.py index fb78d9f0ad..f5df48b27b 100644 --- a/core/shell.py +++ b/core/shell.py @@ -788,7 +788,7 @@ def Main( 'eval': method_io.Eval(), # identical to command sub - 'captureStdout': method_io.CaptureStdout(), + 'captureStdout': method_io.CaptureStdout(shell_ex), 'promptVal': method_io.PromptVal(), 'time': method_io.Time(), 'strftime': method_io.Strftime(), diff --git a/core/vm.py b/core/vm.py index 9380ce3f57..2a8fb4be8c 100644 --- a/core/vm.py +++ b/core/vm.py @@ -10,7 +10,7 @@ from core import pyos from mycpp.mylib import log -from typing import List, Any, TYPE_CHECKING +from typing import List, Tuple, Any, TYPE_CHECKING if TYPE_CHECKING: from _devbuild.gen.runtime_asdl import cmd_value, RedirValue from _devbuild.gen.syntax_asdl import (command, command_t, CommandSub) @@ -198,6 +198,10 @@ def RunSubshell(self, node): # type: (command_t) -> int return 0 + def CaptureStdout(self, node): + # type: (command_t) -> Tuple[int, str] + return 0, '' + def RunCommandSub(self, cs_part): # type: (CommandSub) -> str return '' diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index b4d9378c5f..045b3e0573 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -498,7 +498,16 @@ Like the `eval` builtin, but useful in pure functions. ### captureStdout() -Like `$()`, but useful in pure functions. +Capture stdout of a command a string. + + var c = ^(echo hi) + var stdout_str = _io->captureStdout(c) # => "hi" + +It's like `$()`, but useful in pure functions. Trailing newlines `\n` are +removed. + +If the command fails, `captureStdout()` raises an error, which can be caught +with `try`. ### promptVal() diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 0048ec7405..b6ac5372c5 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -60,7 +60,7 @@ error handling, and more. X [Func] name() location() toJson() X [Proc] name() location() toJson() X [Module] name() filename() - [IO] X eval() X captureStdout() + [IO] X eval() captureStdout() promptVal() X time() X strftime() X glob() diff --git a/test/spec.sh b/test/spec.sh index 93cf8318ec..8bfcfb551d 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -824,6 +824,10 @@ ysh-methods() { run-file ysh-methods "$@" } +ysh-method-io() { + run-file ysh-method-io "$@" +} + ysh-func() { run-file ysh-func "$@" } From 05e21fa72103b73974c0f2554e229de0292b39c3 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 12:05:50 -0400 Subject: [PATCH 077/506] [builtin] First pass of _io->eval(myblock) It's consistent with eval (myblock) Although now I see that errors are not caught, which may be bad for tracking errors. Though this also happens in cd /tmp (; ; myblock) --- builtin/method_io.py | 27 +++++++++++++++++++---- core/shell.py | 2 +- doc/ref/chap-type-method.md | 18 ++++++++++++++- doc/ref/toc-ysh.md | 4 ++-- osh/cmd_eval.py | 12 +++++++++- spec/ysh-builtin-eval.test.sh | 41 +++++++++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 9 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index 7857c26e3a..b7cb56dedf 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -12,18 +12,34 @@ from typing import Dict, cast, TYPE_CHECKING if TYPE_CHECKING: from frontend import typed_args + from osh import cmd_eval _ = log class Eval(vm._Callable): + """ + These are similar: - def __init__(self): - # type: () -> None - pass + var c = ^(echo hi) + + eval (c) + call _io->eval(c) + + The CALLER must handle errors. + """ + def __init__(self, cmd_ev): + # type: (cmd_eval.CommandEvaluator) -> None + self.cmd_ev = cmd_ev def Call(self, rd): # type: (typed_args.Reader) -> value_t + io = rd.PosIO() + cmd = rd.PosCommand() + rd.Done() # no more args + + # errors can arise from false' and 'exit' + unused_status = self.cmd_ev.EvalCommand(cmd) return value.Null @@ -42,11 +58,14 @@ def Call(self, rd): status, stdout_str = self.shell_ex.CaptureStdout(cmd) if status != 0: + # Note that $() raises error.ErrExit with the status. + # But I think that results in a more confusing error message, so we + # "wrap" the errors. properties = { 'status': num.ToBig(status) } # type: Dict[str, value_t] raise error.Structured( - 4, 'Captured command failed with status %d' % status, + 4, 'captureStdout(): command failed with status %d' % status, rd.LeftParenToken(), properties) return value.Str(stdout_str) diff --git a/core/shell.py b/core/shell.py index f5df48b27b..5f229ef321 100644 --- a/core/shell.py +++ b/core/shell.py @@ -785,7 +785,7 @@ def Main( methods[value_e.IO] = { # io->eval(myblock) is the functional version of eval (myblock) # Should we also have expr->eval() instead of evalExpr? - 'eval': method_io.Eval(), + 'eval': method_io.Eval(cmd_ev), # identical to command sub 'captureStdout': method_io.CaptureStdout(shell_ex), diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 045b3e0573..948c58fab5 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -494,7 +494,19 @@ A module is a file with YSH code. ### eval() -Like the `eval` builtin, but useful in pure functions. +Evaluate a command, and return `null`. + + var c = ^(echo hi) + call _io->eval(c) + +It's like like the `eval` builtin, and meant to be used in pure functions. + + ### captureStdout() @@ -509,6 +521,10 @@ removed. If the command fails, `captureStdout()` raises an error, which can be caught with `try`. + try { + var s = _io->captureStdout(c) + } + ### promptVal() An API the wraps the `$PS1` language. For example, to simulate `PS1='\w\$ '`: diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index b6ac5372c5..70ca8ee1ac 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -48,7 +48,7 @@ error handling, and more. search() leftMatch() [List] List/append() pop() extend() indexOf() X insert() X remove() reverse() - [Dict] keys() values() X get() X erase() + [Dict] keys() values() X get() erase() X inc() X accum() [Range] [Eggex] @@ -60,7 +60,7 @@ error handling, and more. X [Func] name() location() toJson() X [Proc] name() location() toJson() X [Module] name() filename() - [IO] X eval() captureStdout() + [IO] eval() captureStdout() promptVal() X time() X strftime() X glob() diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 9a11340a51..234dd62dfa 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -2013,7 +2013,17 @@ def EvalCommand(self, block): # type: (command_t) -> int """For builtins to evaluate command args. - e.g. cd /tmp (x) + Many exceptions are raised. + + Examples: + + cd /tmp (; ; mycmd) + + And: + eval (mycmd) + call _io->eval(mycmd) + + (Should those be more like eval 'mystring'?) """ status = 0 try: diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 6db6c0bccf..abc06dde1c 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -97,3 +97,44 @@ p { ## STDOUT: TODO ## END + + +#### eval 'mystring' vs. eval (myblock) + +eval 'echo plain' +echo plain=$? +var b = ^(echo plain) +eval (b) +echo plain=$? + +echo + +# This calls main_loop.Batch(), which catches +# - error.Parse +# - error.ErrExit +# - error.FatalRuntime - glob errors, etc.? + +try { + eval 'echo one; false; echo two' +} +pp line (_error) + +# This calls CommandEvaluator.EvalCommand(), as blocks do + +var b = ^(echo one; false; echo two) +try { + eval (b) +} +pp line (_error) + +## STDOUT: +plain +plain=0 +plain +plain=0 + +one +(Dict) {"code":1} +one +(Dict) {"code":1} +## END From 964f41050a4feddc3b57553d8b103c5549d918ec Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 14:20:47 -0400 Subject: [PATCH 078/506] [builtin/pp] Show code quotation with pp (x) too Not just pp [x]. This is so it works the same way in OSH and YSH. Arguably, we don't need pp [x] at all then. But: 1. It's easier to type 2. In theory, we could have # line wrap pp [x + \ important] And then printing the whole unevaluated expression would become possible. --- builtin/io_ysh.py | 27 ++++++++++++++++++--------- doc/ref/chap-builtin-cmd.md | 8 ++++---- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 482a90ae12..453b8fd10b 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -7,7 +7,7 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value from _devbuild.gen.syntax_asdl import command_e, BraceGroup, loc -from _devbuild.gen.value_asdl import value, value_e +from _devbuild.gen.value_asdl import value, value_e, value_t from asdl import format as fmt from core import error from core.error import e_usage @@ -65,20 +65,29 @@ def _PrettyPrint(self, cmd_val): val = rd.PosValue() rd.Done() + blame_tok = rd.LeftParenToken() + + # It might be nice to add a string too, like + # pp 'my annotation' (actual) + # But the var name should meaningful in most cases + UP_val = val + result = None # type: value_t with tagswitch(val) as case: if case(value_e.Expr): # Destructured assert [true === f()] val = cast(value.Expr, UP_val) - blame_tok = rd.LeftParenToken() - result = self.expr_ev.EvalExpr(val.e, blame_tok) - # Show it with location - excerpt, prefix = ui.CodeExcerptAndPrefix(blame_tok) - self.stdout_.write(excerpt) - ui.PrettyPrintValue(prefix, result, self.stdout_) + # In this case, we could get the unevaluated code string and + # print it. Although quoting the line seems enough. + result = self.expr_ev.EvalExpr(val.e, blame_tok) else: - # IOError caught by caller - ui.PrettyPrintValue('', val, self.stdout_) + result = val + + # Show it with location + excerpt, prefix = ui.CodeExcerptAndPrefix(blame_tok) + self.stdout_.write(excerpt) + ui.PrettyPrintValue(prefix, result, self.stdout_) + return 0 def Run(self, cmd_val): diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index bcc088b502..92e318da64 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -46,13 +46,13 @@ Similar names: [append][] The most common use is to pretty print expressions: $ var x = 42 - $ pp [x + 5] # pass unevaluated expression + $ pp (x + 5) myfile.ysh:1: (Int) 47 # print value with code location -You can also print a value, with no code location: +You can also pass an unevaluated expression: - $ pp (x + 5) - (Int) 47 + $ pp [x + 5] + myfile.ysh:1: (Int) 47 # evaluate first The `pp` builtin can also print low-level interpreter state. Some of of these are implementation details, subject to change. From 8fde31b9fbcc046cadcd794ed165a720567a3323 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 14:34:56 -0400 Subject: [PATCH 079/506] [spec/ysh-builtin-meta] Fix spec test I pasted the exact output for now. --- spec/ysh-builtin-meta.test.sh | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index 3b5f66728a..cb71e26208 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -240,15 +240,27 @@ pp (u'one \t two \n') | cat pp (repeat([123], 40)) | cat ## STDOUT: -(Str) 'foo' -(Str) b'isn\'t this sq' -(Str) '"dq $myvar"' -(Str) b'\\ backslash \\\\' -(Str) b'one \t two \n' -(List) -[ - 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, - 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, - 123, 123, 123, 123, 123, 123, 123, 123, 123, 123 -] + pp ('foo') | cat + ^ +[ stdin ]:5: (Str) 'foo' + pp ("isn't this sq") | cat + ^ +[ stdin ]:7: (Str) b'isn\'t this sq' + pp ('"dq $myvar"') | cat + ^ +[ stdin ]:9: (Str) '"dq $myvar"' + pp (r'\ backslash \\') | cat + ^ +[ stdin ]:11: (Str) b'\\ backslash \\\\' + pp (u'one \t two \n') | cat + ^ +[ stdin ]:13: (Str) b'one \t two \n' + pp (repeat([123], 40)) | cat + ^ +[ stdin ]:15: (List) + [ + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123 + ] ## END From 0175cb24db16db8390ba5a6fd66cbf53e8aef644 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 15:39:29 -0400 Subject: [PATCH 080/506] [builtin/pp] Distinguish between pp (x) and pp value (x) - The former quotes code - The latter is like the = operator, which is for interactive use Except you can pipe it like: pp value (x) | less -r --- builtin/io_ysh.py | 84 ++++++++++++++----------- doc/ref/chap-builtin-cmd.md | 23 ++++++- doc/ref/toc-ysh.md | 3 +- spec/ysh-builtin-meta.test.sh | 112 ++++++++++++++++++---------------- spec/ysh-printing.test.sh | 32 +++++----- 5 files changed, 146 insertions(+), 108 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 453b8fd10b..b3bf903fd3 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -84,6 +84,7 @@ def _PrettyPrint(self, cmd_val): result = val # Show it with location + self.stdout_.write('\n') excerpt, prefix = ui.CodeExcerptAndPrefix(blame_tok) self.stdout_.write(excerpt) ui.PrettyPrintValue(prefix, result, self.stdout_) @@ -98,39 +99,24 @@ def Run(self, cmd_val): action, action_loc = arg_r.Peek2() - # pp (x) prints in the same way that '= x' does - # TODO: We also need pp [x], which shows the expression + # Special cases + # pp (x) quotes its code location + # pp [x] also evaluates if action is None: return self._PrettyPrint(cmd_val) arg_r.Next() - # Actions that print unstable formats start with '.' - if action == 'cell': - argv, locs = arg_r.Rest2() - - status = 0 - for i, name in enumerate(argv): - if name.startswith(':'): - name = name[1:] + if action == 'value': + # pp value (x) prints in the same way that '= x' does + rd = typed_args.ReaderForProc(cmd_val) + val = rd.PosValue() + rd.Done() - if not match.IsValidVarName(name): - raise error.Usage('got invalid variable name %r' % name, - locs[i]) + ui.PrettyPrintValue('', val, self.stdout_) + return 0 - cell = self.mem.GetCell(name) - if cell is None: - self.errfmt.Print_("Couldn't find a variable named %r" % - name, - blame_loc=locs[i]) - status = 1 - else: - self.stdout_.write('%s = ' % name) - pretty_f = fmt.DetectConsoleOutput(self.stdout_) - fmt.PrintTree(cell.PrettyTree(), pretty_f) - self.stdout_.write('\n') - - elif action == 'asdl': + if action == 'asdl': # TODO: could be pp asdl (x, y, z) rd = typed_args.ReaderForProc(cmd_val) val = rd.PosValue() @@ -150,10 +136,12 @@ def Run(self, cmd_val): fmt.PrintTree(tree, pretty_f) self.stdout_.write('\n') - status = 0 + return 0 + + if action == 'line': + # TODO: could be pp _test - elif action == 'line': - # Print format for unit tests + # Print format for spec tests # TODO: could be pp line (x, y, z) rd = typed_args.ReaderForProc(cmd_val) @@ -166,11 +154,39 @@ def Run(self, cmd_val): j8.PrintLine(val, self.stdout_) + return 0 + + if action == 'cell': + # should this be pp .cell, and pp .asdl? + # or pp _cell pp _asdl? + # pp _test is possible too + argv, locs = arg_r.Rest2() + status = 0 + for i, name in enumerate(argv): + if name.startswith(':'): + name = name[1:] + + if not match.IsValidVarName(name): + raise error.Usage('got invalid variable name %r' % name, + locs[i]) + + cell = self.mem.GetCell(name) + if cell is None: + self.errfmt.Print_("Couldn't find a variable named %r" % + name, + blame_loc=locs[i]) + status = 1 + else: + self.stdout_.write('%s = ' % name) + pretty_f = fmt.DetectConsoleOutput(self.stdout_) + fmt.PrintTree(cell.PrettyTree(), pretty_f) + self.stdout_.write('\n') + return status elif action == 'gc-stats': print('TODO') - status = 0 + return 0 elif action == 'proc': names, locs = arg_r.Rest2() @@ -208,12 +224,10 @@ def Run(self, cmd_val): j8.EncodeString(doc, buf, unquoted_ok=True) print(buf.getvalue()) - status = 0 - - else: - e_usage('got invalid action %r' % action, action_loc) + return 0 - return status + e_usage('got invalid action %r' % action, action_loc) + #return status class Write(_Builtin): diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index 92e318da64..5be83bf978 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -54,9 +54,29 @@ You can also pass an unevaluated expression: $ pp [x + 5] myfile.ysh:1: (Int) 47 # evaluate first +The `value` command is a synonym for the interactive `=` operator: + + $ pp value (x) + (Int) 42 + + $ = x + (Int) 42 + +Print proc names and doc comments: + + $ pp proc # subject to change + The `pp` builtin can also print low-level interpreter state. Some of of these are implementation details, subject to change. + + Examples: var x = :| one two | @@ -66,9 +86,6 @@ Examples: pp line (x) # single-line stable format, for spec tests - pp proc # print all procs and their doc comments - - ## Handle Errors ### error diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 70ca8ee1ac..eca45f721a 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -105,7 +105,8 @@ X [Wok] _field() ```chapter-links-builtin-cmd_42 [Memory] cmd/append Add elements to end of array - pp asdl cell X gc-stats line proc + pp value proc line + asdl cell X gc-stats [Handle Errors] error error 'failed' (status=2) try Run with errexit, set _error failed Test if _error.code !== 0 diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index cb71e26208..409b7b4d2c 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -135,29 +135,6 @@ pp asdl (d) | fgrep -o 'cycle ...' cycle ... ## END -#### pp line supports BashArray, BashAssoc - -declare -a array=(a b c) -pp line (array) - -array[5]=z -pp line (array) - -declare -A assoc=([k]=v [k2]=v2) -pp line (assoc) - -# I think assoc arrays can never null / unset - -assoc['k3']= -pp line (assoc) - -## STDOUT: -{"type":"BashArray","data":{"0":"a","1":"b","2":"c"}} -{"type":"BashArray","data":{"0":"a","1":"b","2":"c","5":"z"}} -{"type":"BashAssoc","data":{"k":"v","k2":"v2"}} -{"type":"BashAssoc","data":{"k":"v","k2":"v2","k3":""}} -## END - #### pp gc-stats @@ -218,49 +195,78 @@ proc_name doc_comment f "doc ' comment with \" quotes" ## END +#### pp (x) and pp [x] quote code -#### pp (x) is like = keyword +pp (42) + +shopt --set ysh:upgrade + +pp [42] + +## STDOUT: + + pp (42) + ^ +[ stdin ]:1: (Int) 42 + + pp [42] + ^ +[ stdin ]:5: (Int) 42 +## END + +#### pp line supports BashArray, BashAssoc + +declare -a array=(a b c) +pp line (array) + +array[5]=z +pp line (array) + +declare -A assoc=([k]=v [k2]=v2) +pp line (assoc) + +# I think assoc arrays can never null / unset + +assoc['k3']= +pp line (assoc) + +## STDOUT: +{"type":"BashArray","data":{"0":"a","1":"b","2":"c"}} +{"type":"BashArray","data":{"0":"a","1":"b","2":"c","5":"z"}} +{"type":"BashAssoc","data":{"k":"v","k2":"v2"}} +{"type":"BashAssoc","data":{"k":"v","k2":"v2","k3":""}} +## END + +#### pp value (x) is like = keyword shopt --set ysh:upgrade source $LIB_YSH/list.ysh # It can be piped! -pp ('foo') | cat +pp value ('foo') | cat -pp ("isn't this sq") | cat +pp value ("isn't this sq") | cat -pp ('"dq $myvar"') | cat +pp value ('"dq $myvar"') | cat -pp (r'\ backslash \\') | cat +pp value (r'\ backslash \\') | cat -pp (u'one \t two \n') | cat +pp value (u'one \t two \n') | cat # Without a terminal, default width is 80 -pp (repeat([123], 40)) | cat +pp value (repeat([123], 40)) | cat ## STDOUT: - pp ('foo') | cat - ^ -[ stdin ]:5: (Str) 'foo' - pp ("isn't this sq") | cat - ^ -[ stdin ]:7: (Str) b'isn\'t this sq' - pp ('"dq $myvar"') | cat - ^ -[ stdin ]:9: (Str) '"dq $myvar"' - pp (r'\ backslash \\') | cat - ^ -[ stdin ]:11: (Str) b'\\ backslash \\\\' - pp (u'one \t two \n') | cat - ^ -[ stdin ]:13: (Str) b'one \t two \n' - pp (repeat([123], 40)) | cat - ^ -[ stdin ]:15: (List) - [ - 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, - 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, - 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123 - ] +(Str) 'foo' +(Str) b'isn\'t this sq' +(Str) '"dq $myvar"' +(Str) b'\\ backslash \\\\' +(Str) b'one \t two \n' +(List) +[ + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123 +] ## END diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index 82a0e1a640..c9ac7bf350 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -37,11 +37,11 @@ #### Range var x = 1..100 -pp (x) +pp value (x) # TODO: show type here, like (Range 1 .. 100) -pp ({k: x}) +pp value ({k: x}) echo @@ -68,9 +68,9 @@ remove-addr() { sed 's/0x[0-9a-f]\+/0x---/' } -pp (pat) | remove-addr +pp value (pat) | remove-addr -pp ({k: pat}) | remove-addr +pp value ({k: pat}) | remove-addr # TODO: change this @@ -95,12 +95,12 @@ array_1[5]=5 var empty = _a2sp(empty) var array_1 = _a2sp(array_1) -pp (empty) -pp (array_1) +pp value (empty) +pp value (array_1) echo -pp ({k: empty}) -pp ({k: array_1}) +pp value ({k: empty}) +pp value ({k: array_1}) echo pp line (empty) @@ -128,12 +128,12 @@ pp line ({k: array_1}) declare -a empty=() declare -a array_1=(hello) -pp (empty) -pp (array_1) +pp value (empty) +pp value (array_1) echo -pp ({k: empty}) -pp ({k: array_1}) +pp value ({k: empty}) +pp value ({k: array_1}) echo pp line (empty) @@ -178,12 +178,12 @@ do eiusmod.) declare -A empty declare -A assoc=(['k']=$'foo \x01\u03bc') -pp (empty) -pp (assoc) +pp value (empty) +pp value (assoc) echo -pp ({k:empty}) -pp ({k:assoc}) +pp value ({k:empty}) +pp value ({k:assoc}) echo pp line (empty) From 589bd5ff9ffebc34bb842d3a18ef68dadba60024 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 16:03:45 -0400 Subject: [PATCH 081/506] [builtin/pp] Rename actions to show that some are private These are public: pp (x) pp value (x) These are private: pp asdl_ (x) pp cell_ x # the name of a cell, not a value pp test_ (x) # TODO: replace pp line May be subsumed by Proc reflection: pp proc --- builtin/io_ysh.py | 21 ++++++--------------- data_lang/json-survey.sh | 4 ++-- demo/xtrace1.sh | 2 +- doc/ref/chap-builtin-cmd.md | 20 ++++++-------------- doc/ref/toc-ysh.md | 4 ++-- spec/ysh-builtin-meta.test.sh | 28 ++++++++++++++-------------- spec/ysh-json.test.sh | 4 ++-- spec/ysh-reserved.test.sh | 2 +- 8 files changed, 34 insertions(+), 51 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index b3bf903fd3..f590a97a87 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -116,8 +116,8 @@ def Run(self, cmd_val): ui.PrettyPrintValue('', val, self.stdout_) return 0 - if action == 'asdl': - # TODO: could be pp asdl (x, y, z) + if action == 'asdl_': + # TODO: could be pp asdl_ (x, y, z) rd = typed_args.ReaderForProc(cmd_val) val = rd.PosValue() rd.Done() @@ -138,11 +138,7 @@ def Run(self, cmd_val): return 0 - if action == 'line': - # TODO: could be pp _test - - # Print format for spec tests - + if action == 'line': # Print format for spec tests # TODO: could be pp line (x, y, z) rd = typed_args.ReaderForProc(cmd_val) val = rd.PosValue() @@ -156,16 +152,11 @@ def Run(self, cmd_val): return 0 - if action == 'cell': - # should this be pp .cell, and pp .asdl? - # or pp _cell pp _asdl? - # pp _test is possible too + if action == 'cell_': # Format may change argv, locs = arg_r.Rest2() status = 0 for i, name in enumerate(argv): - if name.startswith(':'): - name = name[1:] if not match.IsValidVarName(name): raise error.Usage('got invalid variable name %r' % name, @@ -184,11 +175,11 @@ def Run(self, cmd_val): self.stdout_.write('\n') return status - elif action == 'gc-stats': + if action == 'gc-stats_': print('TODO') return 0 - elif action == 'proc': + if action == 'proc': names, locs = arg_r.Rest2() if len(names): for i, name in enumerate(names): diff --git a/data_lang/json-survey.sh b/data_lang/json-survey.sh index eb2faa1e0b..2721c8f2d3 100755 --- a/data_lang/json-survey.sh +++ b/data_lang/json-survey.sh @@ -243,12 +243,12 @@ multiple-refs() { echo # Same with Oils - bin/osh -c 'var mylist = [1,2,3]; var val = [mylist, mylist]; = val; json write (val); pp asdl (val)' + bin/osh -c 'var mylist = [1,2,3]; var val = [mylist, mylist]; = val; json write (val); pp asdl_ (val)' echo } oils-cycles() { - bin/ysh -c 'var d = {}; setvar d.key = d; = d; pp line (d); pp asdl (d); json write (d)' + bin/ysh -c 'var d = {}; setvar d.key = d; = d; pp line (d); pp asdl_ (d); json write (d)' } surrogate-pair() { diff --git a/demo/xtrace1.sh b/demo/xtrace1.sh index b2ac83e749..b97631cde1 100755 --- a/demo/xtrace1.sh +++ b/demo/xtrace1.sh @@ -64,7 +64,7 @@ posix() { # # Related debugging features of OSH: # -# - pp cell (ASDL), pp proc (QTT) +# - pp cell_ (ASDL), pp proc (QTT) # - osh -n (ASDL) # - Oil expressions: = keyword (ASDL) diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index 5be83bf978..540789ba61 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -66,25 +66,17 @@ Print proc names and doc comments: $ pp proc # subject to change -The `pp` builtin can also print low-level interpreter state. Some of of these -are implementation details, subject to change. - - +The `pp` builtin can also print low-level interpreter state. The trailing `_` +indicates that the exact format may change: Examples: - var x = :| one two | - pp cell x # dump the "guts" of a cell, which is a location for a value + $ var x = :| one two | + $ pp cell_ x # dump the "guts" of a cell, which is a location for a value - pp asdl (x) # dump the ASDL "guts" + $ pp asdl_ (x) # dump the ASDL "guts" - pp line (x) # single-line stable format, for spec tests + $ pp test_ (x) # single-line stable format, for spec tests ## Handle Errors diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index eca45f721a..ff8e5e7ac0 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -105,8 +105,8 @@ X [Wok] _field() ```chapter-links-builtin-cmd_42 [Memory] cmd/append Add elements to end of array - pp value proc line - asdl cell X gc-stats + pp value proc test_ + asdl_ cell_ X gc-stats_ [Handle Errors] error error 'failed' (status=2) try Run with errexit, set _error failed Test if _error.code !== 0 diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index 409b7b4d2c..e62b3a3ff3 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -91,7 +91,7 @@ Block ## END -#### pp asdl +#### pp asdl_ shopt -s ysh:upgrade @@ -99,11 +99,11 @@ fopen >out.txt { x=42 setvar y = {foo: x} - pp asdl (x) - pp asdl (y) + pp asdl_ (x) + pp asdl_ (y) # TODO, this might be nice? - # pp asdl (x, y) + # pp asdl_ (x, y) } # Two lines with value.Str @@ -119,7 +119,7 @@ grep -n -o value.Str out.txt 2:value.Str ## END -#### pp asdl can handle an object cycle +#### pp asdl_ can handle an object cycle shopt -s ysh:upgrade @@ -128,7 +128,7 @@ setvar d.cycle = d pp line (d) | fgrep -o '{"cycle":' -pp asdl (d) | fgrep -o 'cycle ...' +pp asdl_ (d) | fgrep -o 'cycle ...' ## STDOUT: {"cycle": @@ -136,24 +136,24 @@ cycle ... ## END -#### pp gc-stats +#### pp gc-stats_ -pp gc-stats +pp gc-stats_ ## STDOUT: ## END -#### pp cell +#### pp cell_ x=42 -pp cell x +pp cell_ x echo status=$? -pp -- cell :x +pp -- cell_ x echo status=$? -pp cell nonexistent +pp cell_ nonexistent echo status=$? ## STDOUT: x = (Cell exported:F readonly:F nameref:F val:(value.Str s:42)) @@ -163,10 +163,10 @@ status=0 status=1 ## END -#### pp cell on indexed array with hole +#### pp cell_ on indexed array with hole declare -a array array[3]=42 -pp cell array +pp cell_ array ## STDOUT: array = (Cell exported:F readonly:F nameref:F val:(value.BashArray strs:[_ _ _ 42])) ## END diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index 2bfee0b0d0..6b9c61768a 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -151,14 +151,14 @@ json write (_reply) #### json read with redirect echo '{"age": 42}' > $TMP/foo.txt json read (&x) < $TMP/foo.txt -pp cell :x +pp cell_ x ## STDOUT: x = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:42)])) ## END #### json read at end of pipeline (relies on lastpipe) echo '{"age": 43}' | json read (&y) -pp cell y +pp cell_ y ## STDOUT: y = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:43)])) ## END diff --git a/spec/ysh-reserved.test.sh b/spec/ysh-reserved.test.sh index cac10907ea..d7005eefa4 100644 --- a/spec/ysh-reserved.test.sh +++ b/spec/ysh-reserved.test.sh @@ -3,7 +3,7 @@ #### Standalone generator expression var x = (i+1 for i in 1:3) # This is NOT a list. TODO: This test is overspecified. -pp cell x | grep -o ' Date: Sun, 28 Jul 2024 16:24:17 -0400 Subject: [PATCH 082/506] [doc] Fix YSH tour --- doc/ysh-tour.md | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/doc/ysh-tour.md b/doc/ysh-tour.md index 4ea2eca63d..5a5bc173ed 100644 --- a/doc/ysh-tour.md +++ b/doc/ysh-tour.md @@ -1164,10 +1164,10 @@ Example: ### Structured: JSON8, TSV8 -You can write and read **tree-shaped** as [JSON][]: +You can write and read **tree-shaped** data as [JSON][]: var d = {key: 'value'} - json write (d) # dump variable d as JSON + json write (d) # dump variable d as JSON # => # { # "key": "value" @@ -1176,9 +1176,8 @@ You can write and read **tree-shaped** as [JSON][]: echo '["ale", 42]' > example.json json read (&d2) < example.json # parse JSON into var d2 - pp cell d2 # inspect the in-memory value - # => - # ['ale', 42] + pp (d2) # pretty print it + # => (List) ['ale', 42] [JSON][] will lose information when strings have binary data, but the slight [JSON8]($xref) upgrade won't: @@ -1192,12 +1191,6 @@ You can write and read **tree-shaped** as [JSON][]: [JSON]: $xref - - **Table-shaped** data can be read and written as [TSV8]($xref). (TODO: not yet implemented.) From 9d313ca4ec3e8e5d5f4c85812f52b5624e2c3c59 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 28 Jul 2024 16:42:52 -0400 Subject: [PATCH 083/506] [builtin/pp] Rename pp line -> pp test_ The public actions are: pp (x) pp value (x) Update doc/ref --- builtin/io_ysh.py | 4 +- data_lang/j8-errors.sh | 8 ++-- data_lang/json-survey.sh | 2 +- doc/error-catalog.md | 2 +- doc/pretty-printing.md | 2 +- doc/ref/chap-builtin-cmd.md | 16 +++++-- spec/ble-idioms.test.sh | 2 +- spec/ysh-assign.test.sh | 8 ++-- spec/ysh-augmented.test.sh | 8 ++-- spec/ysh-bugs.test.sh | 6 +-- spec/ysh-builtin-error.test.sh | 8 ++-- spec/ysh-builtin-eval.test.sh | 4 +- spec/ysh-builtin-meta.test.sh | 18 +++---- spec/ysh-expr-arith.test.sh | 26 +++++----- spec/ysh-expr-bool.test.sh | 26 +++++----- spec/ysh-expr-compare.test.sh | 8 ++-- spec/ysh-expr.test.sh | 48 +++++++++---------- spec/ysh-func.test.sh | 22 ++++----- spec/ysh-int-float.test.sh | 8 ++-- spec/ysh-json.test.sh | 86 +++++++++++++++++----------------- spec/ysh-list.test.sh | 4 +- spec/ysh-methods.test.sh | 26 +++++----- spec/ysh-printing.test.sh | 32 ++++++------- spec/ysh-proc.test.sh | 24 +++++----- spec/ysh-scope.test.sh | 24 +++++----- spec/ysh-slice-range.test.sh | 30 ++++++------ spec/ysh-stdlib-args.test.sh | 10 ++-- spec/ysh-stdlib.test.sh | 6 +-- spec/ysh-string.test.sh | 8 ++-- spec/ysh-unicode.test.sh | 20 ++++---- stdlib/ysh/yblocks-test.ysh | 4 +- stdlib/ysh/yblocks.ysh | 6 +-- test/ysh-parse-errors.sh | 20 ++++---- test/ysh-runtime-errors.sh | 12 ++--- 34 files changed, 272 insertions(+), 266 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index f590a97a87..9350c3612c 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -138,8 +138,8 @@ def Run(self, cmd_val): return 0 - if action == 'line': # Print format for spec tests - # TODO: could be pp line (x, y, z) + if action == 'test_': # Print format for spec tests + # TODO: could be pp test_ (x, y, z) rd = typed_args.ReaderForProc(cmd_val) val = rd.PosValue() rd.Done() diff --git a/data_lang/j8-errors.sh b/data_lang/j8-errors.sh index 9d852db249..73ea57b6a8 100755 --- a/data_lang/j8-errors.sh +++ b/data_lang/j8-errors.sh @@ -87,13 +87,13 @@ EOF # JSON _ysh-error-here-X 1 << 'EOF' echo $'"foo \x01 "' | json read -pp line (_reply) +pp test_ (_reply) EOF # J8 _ysh-error-here-X 1 << 'EOF' var invalid = b'\y01' echo $["u'foo" ++ invalid ++ "'"] | json8 read -pp line (_reply) +pp test_ (_reply) EOF } @@ -139,7 +139,7 @@ test-encode() { _error-case-X 1 'var L = []; call L->append(L); json write (L)' # This should fail! - # But not pp line (L) + # But not pp test_ (L) _error-case-X 1 'var L = []; call L->append(/d+/); j8 write (L)' } @@ -162,7 +162,7 @@ EOF var lines = @( echo '"unbalanced' ) -pp line (lines) +pp test_ (lines) EOF # error in word language diff --git a/data_lang/json-survey.sh b/data_lang/json-survey.sh index 2721c8f2d3..ca1da2da14 100755 --- a/data_lang/json-survey.sh +++ b/data_lang/json-survey.sh @@ -248,7 +248,7 @@ multiple-refs() { } oils-cycles() { - bin/ysh -c 'var d = {}; setvar d.key = d; = d; pp line (d); pp asdl_ (d); json write (d)' + bin/ysh -c 'var d = {}; setvar d.key = d; = d; pp test_ (d); pp asdl_ (d); json write (d)' } surrogate-pair() { diff --git a/doc/error-catalog.md b/doc/error-catalog.md index 002aea6005..d02cdeb723 100644 --- a/doc/error-catalog.md +++ b/doc/error-catalog.md @@ -319,7 +319,7 @@ test/ysh-runtime-errors.sh test-float-equality --> ``` - pp line (42.0 === x) + pp (42.0 === x) ^~~ [ -c flag ]:3: fatal: Equality isn't defined on Float values (OILS-ERR-202) ``` diff --git a/doc/pretty-printing.md b/doc/pretty-printing.md index 1715bc1dd2..2159e348e0 100644 --- a/doc/pretty-printing.md +++ b/doc/pretty-printing.md @@ -520,7 +520,7 @@ This is a global pass that computes a Dict[int, int] The graph is specified by single root node, e.g. the argument to - pp line (obj) + pp value (obj) Pass this dict into the second step. diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index 540789ba61..3513f5e2c4 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -43,13 +43,15 @@ Similar names: [append][] ### pp -The most common use is to pretty print expressions: +The `pp` builtin pretty prints values and interpreter state. + +Pretty printing expressions is the most common: $ var x = 42 $ pp (x + 5) myfile.ysh:1: (Int) 47 # print value with code location -You can also pass an unevaluated expression: +You can pass an unevaluated expression: $ pp [x + 5] myfile.ysh:1: (Int) 47 # evaluate first @@ -66,18 +68,22 @@ Print proc names and doc comments: $ pp proc # subject to change -The `pp` builtin can also print low-level interpreter state. The trailing `_` -indicates that the exact format may change: +You can also print low-level interpreter state. The trailing `_` indicates +that the exact format may change: Examples: $ var x = :| one two | - $ pp cell_ x # dump the "guts" of a cell, which is a location for a value $ pp asdl_ (x) # dump the ASDL "guts" $ pp test_ (x) # single-line stable format, for spec tests + # dump the ASDL representation of a "Cell", which is a location for a value + # (not the value itself) + $ pp cell_ x + + ## Handle Errors ### error diff --git a/spec/ble-idioms.test.sh b/spec/ble-idioms.test.sh index 5615044d9d..073ccfd288 100644 --- a/spec/ble-idioms.test.sh +++ b/spec/ble-idioms.test.sh @@ -274,7 +274,7 @@ echo "${a[@]}" case $SH in bash|zsh|mksh|ash) exit ;; esac -#pp line (a) +#pp test_ (a) a=( foo {25..27} bar ) diff --git a/spec/ysh-assign.test.sh b/spec/ysh-assign.test.sh index 59c0cded08..d96389c370 100644 --- a/spec/ysh-assign.test.sh +++ b/spec/ysh-assign.test.sh @@ -294,16 +294,16 @@ status=1 #### circular dict - TODO 2023-06 REGRESS var d = {name: 'foo'} -pp line (d) +pp test_ (d) setvar d['name'] = 123 -pp line (d) +pp test_ (d) setvar d['name'] = 'mystr' -pp line (d) +pp test_ (d) setvar d['name'] = d -pp line (d) +pp test_ (d) # This used to print ... diff --git a/spec/ysh-augmented.test.sh b/spec/ysh-augmented.test.sh index 21512bf10b..ba293fcaf2 100644 --- a/spec/ysh-augmented.test.sh +++ b/spec/ysh-augmented.test.sh @@ -45,17 +45,17 @@ x=11 #### Augmented assignment on string changes to Int Float var x = '42' -pp line (x) +pp test_ (x) setvar x += 4 * 1 -pp line (x) +pp test_ (x) setvar x += '9' -pp line (x) +pp test_ (x) setvar x = '42' setvar x /= 4 -pp line (x) +pp test_ (x) ## STDOUT: (Str) "42" diff --git a/spec/ysh-bugs.test.sh b/spec/ysh-bugs.test.sh index 5e51b985a3..7d3bbf2b75 100644 --- a/spec/ysh-bugs.test.sh +++ b/spec/ysh-bugs.test.sh @@ -122,7 +122,7 @@ type -a returned 1 var x = [] true && call x->append(42) false && call x->append(43) -pp line (x) +pp test_ (x) func amp() { true && return (42) @@ -132,8 +132,8 @@ func pipe() { false || return (42) } -pp line (amp()) -pp line (pipe()) +pp test_ (amp()) +pp test_ (pipe()) ## STDOUT: ## END diff --git a/spec/ysh-builtin-error.test.sh b/spec/ysh-builtin-error.test.sh index e5a5b6ef31..7c5031c389 100644 --- a/spec/ysh-builtin-error.test.sh +++ b/spec/ysh-builtin-error.test.sh @@ -74,18 +74,18 @@ message=divide by zero: 5 / 0 try { error 'bad' (code=99) } -pp line (_error) +pp test_ (_error) # Note: myData co try { error 'bad' (code=99, myData={spam:'eggs'}) } -pp line (_error) +pp test_ (_error) try { error 'bad' (code=99, message='cannot override') } -pp line (_error) +pp test_ (_error) ## STDOUT: (Dict) {"code":99,"message":"bad"} @@ -431,7 +431,7 @@ code 1 try { $SH -c ' - #pp line (42 === 42 === 43) + #pp test_ (42 === 42 === 43) assert [42 === 42 === 43] echo unreachable ' diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index abc06dde1c..bf944e1c8c 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -117,7 +117,7 @@ echo try { eval 'echo one; false; echo two' } -pp line (_error) +pp test_ (_error) # This calls CommandEvaluator.EvalCommand(), as blocks do @@ -125,7 +125,7 @@ var b = ^(echo one; false; echo two) try { eval (b) } -pp line (_error) +pp test_ (_error) ## STDOUT: plain diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index e62b3a3ff3..c2cd3fcc97 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -50,9 +50,9 @@ echo proc ty (w; t; n; block) { echo 'ty' - pp line (w) - pp line (t) - pp line (n) + pp test_ (w) + pp test_ (t) + pp test_ (n) echo $[type(block)] } @@ -126,7 +126,7 @@ shopt -s ysh:upgrade var d = {} setvar d.cycle = d -pp line (d) | fgrep -o '{"cycle":' +pp test_ (d) | fgrep -o '{"cycle":' pp asdl_ (d) | fgrep -o 'cycle ...' @@ -214,21 +214,21 @@ pp [42] [ stdin ]:5: (Int) 42 ## END -#### pp line supports BashArray, BashAssoc +#### pp test_ supports BashArray, BashAssoc declare -a array=(a b c) -pp line (array) +pp test_ (array) array[5]=z -pp line (array) +pp test_ (array) declare -A assoc=([k]=v [k2]=v2) -pp line (assoc) +pp test_ (assoc) # I think assoc arrays can never null / unset assoc['k3']= -pp line (assoc) +pp test_ (assoc) ## STDOUT: {"type":"BashArray","data":{"0":"a","1":"b","2":"c"}} diff --git a/spec/ysh-expr-arith.test.sh b/spec/ysh-expr-arith.test.sh index 5e414f09b2..c24b0d5d29 100644 --- a/spec/ysh-expr-arith.test.sh +++ b/spec/ysh-expr-arith.test.sh @@ -219,27 +219,27 @@ json write (~'3.5') $SH -c ' var x = 0.12345 -pp line (x) +pp test_ (x) ' echo float=$? $SH -c ' # Becomes infinity var x = 0.123456789e1234567 -pp line (x) +pp test_ (x) var x = -0.123456789e1234567 -pp line (x) +pp test_ (x) ' echo float=$? $SH -c ' # Becomes infinity var x = 0.123456789e-1234567 -pp line (x) +pp test_ (x) var x = -0.123456789e-1234567 -pp line (x) +pp test_ (x) ' echo float=$? @@ -259,52 +259,52 @@ float=0 # Decimal $SH -c ' var x = 1111 -pp line (x) +pp test_ (x) ' echo dec=$? $SH -c ' var x = 1111_2222_3333_4444_5555_6666 -pp line (x) +pp test_ (x) ' echo dec=$? # Binary $SH -c ' var x = 0b11 -pp line (x) +pp test_ (x) ' echo bin=$? $SH -c ' var x = 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111 -pp line (x) +pp test_ (x) ' echo bin=$? # Octal $SH -c ' var x = 0o77 -pp line (x) +pp test_ (x) ' echo oct=$? $SH -c ' var x = 0o1111_2222_3333_4444_5555_6666 -pp line (x) +pp test_ (x) ' echo oct=$? # Hex $SH -c ' var x = 0xff -pp line (x) +pp test_ (x) ' echo hex=$? $SH -c ' var x = 0xaaaa_bbbb_cccc_dddd_eeee_ffff -pp line (x) +pp test_ (x) ' echo hex=$? diff --git a/spec/ysh-expr-bool.test.sh b/spec/ysh-expr-bool.test.sh index 4cfc524165..b463171118 100644 --- a/spec/ysh-expr-bool.test.sh +++ b/spec/ysh-expr-bool.test.sh @@ -27,13 +27,13 @@ OK # consistent with if statement, ternary if, and, or -pp line (not "s") -pp line (not 3) -pp line (not 4.5) -pp line (not {}) -pp line (not []) -pp line (not false) -pp line (not true) +pp test_ (not "s") +pp test_ (not 3) +pp test_ (not 4.5) +pp test_ (not {}) +pp test_ (not []) +pp test_ (not false) +pp test_ (not true) ## STDOUT: (Bool) false @@ -110,11 +110,11 @@ echo $[0 and 42] echo $[0.0 or 0.5] echo $[0.0 and 0.5] -pp line (["a"] or []) -pp line (["a"] and []) +pp test_ (["a"] or []) +pp test_ (["a"] and []) -pp line ({"d": 1} or {}) -pp line ({"d": 1} and {}) +pp test_ ({"d": 1} or {}) +pp test_ ({"d": 1} and {}) echo $[0 or 0.0 or false or [] or {} or "OR"] echo $[1 and 1.0 and true and [5] and {"d":1} and "AND"] @@ -161,10 +161,10 @@ AND #### or BashArray, or BashAssoc declare -a array=(1 2 3) -pp line (array or 'yy') +pp test_ (array or 'yy') declare -A assoc=([k]=v) -pp line (assoc or 'zz') +pp test_ (assoc or 'zz') ## STDOUT: {"type":"BashArray","data":{"0":"1","1":"2","2":"3"}} diff --git a/spec/ysh-expr-compare.test.sh b/spec/ysh-expr-compare.test.sh index 2641fab016..a9294f2784 100644 --- a/spec/ysh-expr-compare.test.sh +++ b/spec/ysh-expr-compare.test.sh @@ -105,14 +105,14 @@ one $SH -c ' shopt -s ysh:upgrade -pp line (1.0 === 2.0) +pp test_ (1.0 === 2.0) echo ok ' echo status=$? $SH -c ' shopt -s ysh:upgrade -pp line (42 === 3.0) +pp test_ (42 === 3.0) echo ok ' echo status=$? @@ -126,9 +126,9 @@ status=3 #### floatsEqual() var x = 42.0 -pp line (floatsEqual(42.0, x)) +pp test_ (floatsEqual(42.0, x)) -pp line (floatsEqual(42.0, x + 1)) +pp test_ (floatsEqual(42.0, x + 1)) ## STDOUT: (Bool) true diff --git a/spec/ysh-expr.test.sh b/spec/ysh-expr.test.sh index d3d5b80379..e5f151476d 100644 --- a/spec/ysh-expr.test.sh +++ b/spec/ysh-expr.test.sh @@ -326,28 +326,28 @@ echo 'should not get here' ## END #### Float Division -pp line (5/2) -pp line (-5/2) -pp line (5/-2) -pp line (-5/-2) +pp test_ (5/2) +pp test_ (-5/2) +pp test_ (5/-2) +pp test_ (-5/-2) echo --- var x = 9 setvar x /= 2 -pp line (x) +pp test_ (x) var x = -9 setvar x /= 2 -pp line (x) +pp test_ (x) var x = 9 setvar x /= -2 -pp line (x) +pp test_ (x) var x = -9 setvar x /= -2 -pp line (x) +pp test_ (x) ## STDOUT: @@ -363,28 +363,28 @@ pp line (x) ## END #### Integer Division (rounds toward zero) -pp line (5//2) -pp line (-5//2) -pp line (5//-2) -pp line (-5//-2) +pp test_ (5//2) +pp test_ (-5//2) +pp test_ (5//-2) +pp test_ (-5//-2) echo --- var x = 9 setvar x //= 2 -pp line (x) +pp test_ (x) var x = -9 setvar x //= 2 -pp line (x) +pp test_ (x) var x = 9 setvar x //= -2 -pp line (x) +pp test_ (x) var x = -9 setvar x //= -2 -pp line (x) +pp test_ (x) ## STDOUT: (Int) 2 @@ -399,20 +399,20 @@ pp line (x) ## END #### % operator is remainder -pp line ( 5 % 3) -pp line (-5 % 3) +pp test_ ( 5 % 3) +pp test_ (-5 % 3) # negative divisor illegal (tested in test/ysh-runtime-errors.sh) -#pp line ( 5 % -3) -#pp line (-5 % -3) +#pp test_ ( 5 % -3) +#pp test_ (-5 % -3) var z = 10 setvar z %= 3 -pp line (z) +pp test_ (z) var z = -10 setvar z %= 3 -pp line (z) +pp test_ (z) ## STDOUT: (Int) 2 @@ -649,7 +649,7 @@ var e = ^[42 === x and true] echo $[evalExpr(e)] var mylist = ^[3, 4] -pp line (evalExpr(mylist)) +pp test_ (evalExpr(mylist)) ## STDOUT: type=Expr @@ -662,7 +662,7 @@ true #### No list comprehension in ^[] var mylist = ^[x for x in y] -pp line (evalExpr(mylist)) +pp test_ (evalExpr(mylist)) ## status: 2 ## STDOUT: diff --git a/spec/ysh-func.test.sh b/spec/ysh-func.test.sh index 1441c5e6c1..50bfc606ad 100644 --- a/spec/ysh-func.test.sh +++ b/spec/ysh-func.test.sh @@ -64,7 +64,7 @@ x=4 #### Named args with ...rest func f(; x=3, ...named) { echo x=$x - pp line (named) + pp test_ (named) } call f() @@ -104,8 +104,8 @@ call f(; ...args) #### Multiple spreads func f(...pos; ...named) { - pp line (pos) - pp line (named) + pp test_ (pos) + pp test_ (named) } var a = [1,2,3] @@ -229,9 +229,9 @@ func f(x) { setvar x = 'bar' } -pp line (x) -pp line (f(x)) -pp line (x) +pp test_ (x) +pp test_ (f(x)) +pp test_ (x) # reference var y = ['a', 'b', 'c'] @@ -240,9 +240,9 @@ func g(y) { setvar y[0] = 'z' } -pp line (y) -pp line (g(y)) -pp line (y) +pp test_ (y) +pp test_ (g(y)) +pp test_ (y) ## STDOUT: (Str) "foo" (Null) null @@ -349,7 +349,7 @@ hit: 8 #### Varadic arguments, no other args func f(...args) { -pp line (args) +pp test_ (args) } call f() @@ -365,7 +365,7 @@ call f(1, 2, 3) #### Varadic arguments, other args func f(a, b, ...args) { -pp line ([a, b, args]) +pp test_ ([a, b, args]) } call f(1, 2) diff --git a/spec/ysh-int-float.test.sh b/spec/ysh-int-float.test.sh index 6c08c62572..42a757377f 100644 --- a/spec/ysh-int-float.test.sh +++ b/spec/ysh-int-float.test.sh @@ -136,7 +136,7 @@ nan is not nan #### pretty print INFINITY, -INFINITY, NAN = [INFINITY, -INFINITY, NAN] -pp line ([INFINITY, -INFINITY, NAN]) +pp test_ ([INFINITY, -INFINITY, NAN]) ## STDOUT: (List) [INFINITY, -INFINITY, NAN] @@ -149,13 +149,13 @@ pp line ([INFINITY, -INFINITY, NAN]) shopt --set ysh:upgrade -pp line (1/3) | read --all +pp test_ (1/3) | read --all if (_reply ~ / '0.' '3'+ / ) { echo one-third } -pp line (2/3) | read --all -#pp line (_reply) +pp test_ (2/3) | read --all +#pp test_ (_reply) if (_reply ~ / '0.' '6'+ / ) { echo two-thirds } diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index 6b9c61768a..2b7ba48699 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -166,7 +166,7 @@ y = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int #### invalid JSON echo '{' | json read (&y) echo pipeline status = $? -pp line (y) +pp test_ (y) ## status: 1 ## STDOUT: pipeline status = 1 @@ -236,7 +236,7 @@ setvar L[0] = L shopt -s ysh:upgrade fopen >tmp.txt { - pp line (L) + pp test_ (L) } fgrep -n -o '[ -->' tmp.txt @@ -255,7 +255,7 @@ setvar d.k = d shopt -s ysh:upgrade fopen >tmp.txt { - pp line (d) + pp test_ (d) } fgrep -n -o '{ -->' tmp.txt @@ -288,7 +288,7 @@ json read </dev/null <<'EOF' @@ -505,36 +505,36 @@ b'\u{1}\yff\u{1f}' #### json8 read echo '{ }' | json8 read -pp line (_reply) +pp test_ (_reply) echo '[ ]' | json8 read -pp line (_reply) +pp test_ (_reply) echo '[42]' | json8 read -pp line (_reply) +pp test_ (_reply) echo '[true, false]' | json8 read -pp line (_reply) +pp test_ (_reply) echo '{"k": "v"}' | json8 read -pp line (_reply) +pp test_ (_reply) echo '{"k": null}' | json8 read -pp line (_reply) +pp test_ (_reply) echo '{"k": 1, "k2": 2}' | json8 read -pp line (_reply) +pp test_ (_reply) echo "{u'k': {b'k2': null}}" | json8 read -pp line (_reply) +pp test_ (_reply) echo '{"k": {"k2": "v2"}, "k3": "backslash \\ \" \n line 2 \u03bc "}' | json8 read -pp line (_reply) +pp test_ (_reply) json8 read (&x) <<'EOF' {u'k': {u'k2': u'v2'}, u'k3': u'backslash \\ \" \n line 2 \u{3bc} '} EOF -pp line (x) +pp test_ (x) ## STDOUT: (Dict) {} @@ -581,7 +581,7 @@ var d = { json write (d) | json read -pp line (_reply) +pp test_ (_reply) ## STDOUT: (Dict) {"short":"-v","long":"--verbose","type":null,"default":"","help":"Enable verbose logging"} @@ -624,7 +624,7 @@ shopt -s ysh:upgrade for j in '"\ud83e"' '"\udd26"' { var s = fromJson(j) write -- "$j" - pp line (s) + pp test_ (s) write -n 'json '; json write (s) @@ -688,11 +688,11 @@ var m1 = '[42,1.5,null,true,"hi"]' # JSON8 message var m2 = '[42,1.5,null,true,"hi",' ++ "u''" ++ ']' -pp line (fromJson8(m1)) -pp line (fromJson(m1)) +pp test_ (fromJson8(m1)) +pp test_ (fromJson(m1)) -pp line (fromJson8(m2)) -pp line (fromJson(m2)) # fails +pp test_ (fromJson8(m2)) +pp test_ (fromJson(m2)) # fails ## status: 4 ## STDOUT: @@ -827,12 +827,12 @@ echo -n u'''\yfd''' | od -A n -t x1 json8 read (&b) <<'EOF' b'\yfe' EOF -pp line (b) +pp test_ (b) json8 read (&u) <<'EOF' u'\yfe' EOF -pp line (u) # undefined +pp test_ (u) # undefined ## status: 1 ## STDOUT: @@ -902,13 +902,13 @@ source $LIB_YSH/list.ysh # Create inf var big = repeat('12345678', 100) ++ '.0' -#pp line (s) +#pp test_ (s) var inf = fromJson(big) var neg_inf = fromJson('-' ++ big) # Can be printed -pp line (inf) -pp line (neg_inf) +pp test_ (inf) +pp test_ (neg_inf) echo -- # Can't be serialized @@ -941,7 +941,7 @@ null #### NaN is encoded as null, like JavaScript -pp line (NAN) +pp test_ (NAN) json write (NAN) @@ -995,10 +995,10 @@ status=1 #### '' means the same thing as u'' echo "''" | json8 read -pp line (_reply) +pp test_ (_reply) echo "'\u{3bc}'" | json8 read -pp line (_reply) +pp test_ (_reply) echo "'\yff'" | json8 read echo status=$? @@ -1015,7 +1015,7 @@ json=$(( 1 << 33 )) echo $json echo $json | json read -pp line (_reply) +pp test_ (_reply) ## STDOUT: 8589934592 @@ -1027,13 +1027,13 @@ pp line (_reply) $SH <<'EOF' json read <<< '123456789123456789123456789' echo status=$? -pp line (_reply) +pp test_ (_reply) EOF $SH <<'EOF' json read <<< '-123456789123456789123456789' echo status=$? -pp line (_reply) +pp test_ (_reply) EOF echo ok @@ -1120,12 +1120,12 @@ status=1 #### Data after internal NUL (issue #2026) $SH <<'EOF' -pp line (fromJson(b'123\y00abc')) +pp test_ (fromJson(b'123\y00abc')) EOF echo status=$? $SH <<'EOF' -pp line (fromJson(b'123\y01abc')) +pp test_ (fromJson(b'123\y01abc')) EOF echo status=$? @@ -1153,13 +1153,13 @@ status=1 $SH <<'EOF' json read <<< '123456789123456789123456789.12345e67890' echo status=$? -pp line (_reply) +pp test_ (_reply) EOF $SH <<'EOF' json read <<< '-123456789123456789123456789.12345e67890' echo status=$? -pp line (_reply) +pp test_ (_reply) EOF ## STDOUT: @@ -1190,7 +1190,7 @@ msg=$(pairs 50) #echo $msg echo "$msg" | json read -pp line (_reply) +pp test_ (_reply) echo len=$[len(_reply)] ## STDOUT: @@ -1202,10 +1202,10 @@ len=1 #### Too many opening [[[ - blocking stack python2 -c 'print("[" * 10000)' | json read -pp line (_reply) +pp test_ (_reply) python2 -c 'print("{" * 10000)' | json read -pp line (_reply) +pp test_ (_reply) ## STDOUT: ## END diff --git a/spec/ysh-list.test.sh b/spec/ysh-list.test.sh index 50482e6115..1c6b88466e 100644 --- a/spec/ysh-list.test.sh +++ b/spec/ysh-list.test.sh @@ -26,12 +26,12 @@ dq 1 #### Can print type of List with pp var b = :|true| # this is a string -pp line (b) +pp test_ (b) # = b var empty = :|| -pp line (empty) +pp test_ (empty) # = empty diff --git a/spec/ysh-methods.test.sh b/spec/ysh-methods.test.sh index 6a899895e8..2a94c8928b 100644 --- a/spec/ysh-methods.test.sh +++ b/spec/ysh-methods.test.sh @@ -376,7 +376,7 @@ var en2fr = {} setvar en2fr["hello"] = "bonjour" setvar en2fr["friend"] = "ami" setvar en2fr["cat"] = "chat" -pp line (en2fr => keys()) +pp test_ (en2fr => keys()) ## status: 0 ## STDOUT: (List) ["hello","friend","cat"] @@ -387,7 +387,7 @@ var en2fr = {} setvar en2fr["hello"] = "bonjour" setvar en2fr["friend"] = "ami" setvar en2fr["cat"] = "chat" -pp line (en2fr => values()) +pp test_ (en2fr => values()) ## status: 0 ## STDOUT: (List) ["bonjour","ami","chat"] @@ -396,10 +396,10 @@ pp line (en2fr => values()) #### Dict -> erase() var book = {title: "The Histories", author: "Herodotus"} call book->erase("author") -pp line (book) +pp test_ (book) # confirm method is idempotent call book->erase("author") -pp line (book) +pp test_ (book) ## status: 0 ## STDOUT: (Dict) {"title":"The Histories"} @@ -408,7 +408,7 @@ pp line (book) #### Separation of -> attr and () calling const check = "abc" => startsWith -pp line (check("a")) +pp test_ (check("a")) ## status: 0 ## STDOUT: (Bool) true @@ -417,15 +417,15 @@ pp line (check("a")) #### Bound methods, receiver value/reference semantics var is_a_ref = { "foo": "bar" } const f = is_a_ref => keys -pp line (f()) +pp test_ (f()) setvar is_a_ref["baz"] = 42 -pp line (f()) +pp test_ (f()) var is_a_val = "abc" const g = is_a_val => startsWith -pp line (g("a")) +pp test_ (g("a")) setvar is_a_val = "xyz" -pp line (g("a")) +pp test_ (g("a")) ## status: 0 ## STDOUT: (List) ["foo"] @@ -479,10 +479,10 @@ call a->reverse() call b->reverse() call c->reverse() -pp line (empty) -pp line (a) -pp line (b) -pp line (c) +pp test_ (empty) +pp test_ (a) +pp test_ (b) +pp test_ (c) ## STDOUT: (List) [] diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index c9ac7bf350..52330e4acd 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -49,8 +49,8 @@ remove-addr() { sed 's/0x[0-9a-f]\+/0x---/' } -pp line (x) | remove-addr -pp line ({k: x}) | remove-addr +pp test_ (x) | remove-addr +pp test_ ({k: x}) | remove-addr ## STDOUT: (Range 1 .. 100) @@ -76,8 +76,8 @@ pp value ({k: pat}) | remove-addr echo -pp line (pat) | remove-addr -pp line ({k: pat}) | remove-addr +pp test_ (pat) | remove-addr +pp test_ ({k: pat}) | remove-addr ## STDOUT: @@ -103,12 +103,12 @@ pp value ({k: empty}) pp value ({k: array_1}) echo -pp line (empty) -pp line (array_1) +pp test_ (empty) +pp test_ (array_1) echo -pp line ({k: empty}) -pp line ({k: array_1}) +pp test_ ({k: empty}) +pp test_ ({k: array_1}) ## STDOUT: (SparseArray) @@ -136,12 +136,12 @@ pp value ({k: empty}) pp value ({k: array_1}) echo -pp line (empty) -pp line (array_1) +pp test_ (empty) +pp test_ (array_1) echo -pp line ({k: empty}) -pp line ({k: array_1}) +pp test_ ({k: empty}) +pp test_ ({k: array_1}) ## STDOUT: (BashArray) @@ -186,12 +186,12 @@ pp value ({k:empty}) pp value ({k:assoc}) echo -pp line (empty) -pp line (assoc) +pp test_ (empty) +pp test_ (assoc) echo -pp line ({k:empty}) -pp line ({k:assoc}) +pp test_ ({k:empty}) +pp test_ ({k:assoc}) ## STDOUT: (BashAssoc) diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index f8af213e9d..de1bc0dbfa 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -134,10 +134,10 @@ shopt --set ysh:upgrade # TODO: duplicate param names aren't allowed proc p (a; mylist, mydict; opt Int = 42) { - pp line (a) - pp line (mylist) - pp line (mydict) - #pp line (opt) + pp test_ (a) + pp test_ (mylist) + pp test_ (mydict) + #pp test_ (opt) } p WORD ([1,2,3], {name: 'bob'}) @@ -331,11 +331,11 @@ expression shopt --set ysh:upgrade proc p2 (...words; ...typed; ...named; block) { - pp line (words) - pp line (typed) - pp line (named) - #pp line (block) - # To avoid - could change pp line + pp test_ (words) + pp test_ (typed) + pp test_ (named) + #pp test_ (block) + # To avoid - could change pp test_ echo $[type(block)] } @@ -448,9 +448,9 @@ argv.py global @ARGV shopt -s ysh:upgrade typed proc p (w; t; n; block) { - pp line (w) - pp line (t) - pp line (n) + pp test_ (w) + pp test_ (t) + pp test_ (n) echo $[type(block)] } diff --git a/spec/ysh-scope.test.sh b/spec/ysh-scope.test.sh index a30f616d5a..b9e3ce6cda 100644 --- a/spec/ysh-scope.test.sh +++ b/spec/ysh-scope.test.sh @@ -338,16 +338,16 @@ proc mutate { setglobal g['key2'] = 'mutated' echo 'local that is ignored' - pp line (g) + pp test_ (g) } echo 'BEFORE mutate global' -pp line (g) +pp test_ (g) mutate echo 'AFTER mutate global' -pp line (g) +pp test_ (g) ## STDOUT: BEFORE mutate global @@ -369,16 +369,16 @@ proc mutate { echo 'local that is ignored' setglobal a[0] = 42 - pp line (a) + pp test_ (a) } echo 'BEFORE mutate global' -pp line (a) +pp test_ (a) mutate echo 'AFTER mutate global' -pp line (a) +pp test_ (a) ## STDOUT: BEFORE mutate global @@ -406,8 +406,8 @@ proc mutate { mutate -pp line (mylist) -pp line (mydict) +pp test_ (mylist) +pp test_ (mydict) ## STDOUT: (List) [5] @@ -445,14 +445,14 @@ proc mutate2 { mutate1 -pp line (mylist) -pp line (mydict) +pp test_ (mylist) +pp test_ (mydict) echo mutate2 -pp line (mylist) -pp line (mydict) +pp test_ (mylist) +pp test_ (mydict) ## STDOUT: (List) [0,11] diff --git a/spec/ysh-slice-range.test.sh b/spec/ysh-slice-range.test.sh index 156c80d2e6..523e1669c8 100644 --- a/spec/ysh-slice-range.test.sh +++ b/spec/ysh-slice-range.test.sh @@ -28,16 +28,16 @@ #### subscript and slice :| 1 2 3 4 | var myarray = :|1 2 3 4| -pp line (myarray[1]) -pp line (myarray[1:3]) +pp test_ (myarray[1]) +pp test_ (myarray[1:3]) echo 'implicit' -pp line (myarray[:2]) -pp line (myarray[2:]) +pp test_ (myarray[:2]) +pp test_ (myarray[2:]) echo 'out of bounds' -pp line (myarray[:5]) -pp line (myarray[-5:]) +pp test_ (myarray[:5]) +pp test_ (myarray[-5:]) # Stride not supported #= myarray[1:4:2] @@ -112,12 +112,12 @@ compare 'a[4:5]' #### subscript and slice of List var mylist = [1,2,3,4] -pp line (mylist[1]) -pp line (mylist[1:3]) +pp test_ (mylist[1]) +pp test_ (mylist[1:3]) echo 'implicit' -pp line (mylist[:2]) -pp line (mylist[2:]) +pp test_ (mylist[:2]) +pp test_ (mylist[2:]) ## STDOUT: (Int) 2 (List) [2,3] @@ -128,12 +128,12 @@ implicit #### expressions and negative indices var myarray = :|1 2 3 4 5| -pp line (myarray[-1]) -pp line (myarray[-4:-2]) +pp test_ (myarray[-1]) +pp test_ (myarray[-4:-2]) echo 'implicit' -pp line (myarray[:-2]) -pp line (myarray[-2:]) +pp test_ (myarray[:-2]) +pp test_ (myarray[-2:]) ## STDOUT: (Str) "5" (List) ["2","3"] @@ -153,7 +153,7 @@ echo $val #### Copy with a[:] var a = [1,2,3] var b = a[:] -pp line (b) +pp test_ (b) ## STDOUT: (List) [1,2,3] ## END diff --git a/spec/ysh-stdlib-args.test.sh b/spec/ysh-stdlib-args.test.sh index 67d7bded23..57b3e90350 100644 --- a/spec/ysh-stdlib-args.test.sh +++ b/spec/ysh-stdlib-args.test.sh @@ -25,7 +25,7 @@ parser (&spec) { var args = parseArgs(spec, :| mysrc -P 12 mydest a b c |) echo "Verbose $[args.verbose]" -pp line (args) +pp test_ (args) ## STDOUT: Verbose false (Dict) {"src":"mysrc","max-procs":12,"dest":"mydest","files":["a","b","c"],"verbose":false,"invert":true} @@ -48,7 +48,7 @@ var argv = ['-v', 'src/path', 'dst/path', 'x', 'y', 'z'] var args = parseArgs(spec, argv) -pp line (args) +pp test_ (args) if (args.verbose) { echo "$[args.src] -> $[args.dst]" @@ -83,7 +83,7 @@ for args in (argsCases) { var args_str = join(args, ' ') echo "---------- $args_str ----------" echo "\$ bin/ysh example.sh $args_str" - pp line (parseArgs(spec, args)) + pp test_ (parseArgs(spec, args)) echo } @@ -175,7 +175,7 @@ for args in (argsCases) { var args_str = args->join(" ") echo "---------- $args_str ----------" echo "\$ bin/ysh example.sh $args_str" - pp line (parseArgs(spec, args)) + pp test_ (parseArgs(spec, args)) echo echo "\$ python3 example.py $args_str" @@ -257,7 +257,7 @@ parser (&spec) { var args = parseArgs(spec, []) -pp line (args) +pp test_ (args) ## STDOUT: (Dict) {"sanitize":false,"verbose":false,"max-procs":null} ## END diff --git a/spec/ysh-stdlib.test.sh b/spec/ysh-stdlib.test.sh index db6c9585a7..3971f9c6d3 100644 --- a/spec/ysh-stdlib.test.sh +++ b/spec/ysh-stdlib.test.sh @@ -194,7 +194,7 @@ negative try { $SH -c ' source $LIB_YSH/list.ysh - pp line (repeat(null, 3)) + pp test_ (repeat(null, 3)) echo bad' } echo code=$[_error.code] @@ -202,7 +202,7 @@ echo code=$[_error.code] try { $SH -c ' source $LIB_YSH/list.ysh - pp line (repeat({}, 3)) + pp test_ (repeat({}, 3)) echo bad' } echo code=$[_error.code] @@ -210,7 +210,7 @@ echo code=$[_error.code] try { $SH -c ' source $LIB_YSH/list.ysh - pp line (repeat(42, 3)) + pp test_ (repeat(42, 3)) echo bad' } echo code=$[_error.code] diff --git a/spec/ysh-string.test.sh b/spec/ysh-string.test.sh index 3ba91134df..33b8c85d6f 100644 --- a/spec/ysh-string.test.sh +++ b/spec/ysh-string.test.sh @@ -5,14 +5,14 @@ # everything except \b \f \n var nl = \n -pp line (nl) +pp test_ (nl) var tab = \t -pp line (tab) +pp test_ (tab) -pp line (\r) +pp test_ (\r) -pp line (\" ++ \' ++ \\) +pp test_ (\" ++ \' ++ \\) echo backslash $[\\] echo "backslash $[\\]" diff --git a/spec/ysh-unicode.test.sh b/spec/ysh-unicode.test.sh index f31a1debd5..20a05a0011 100644 --- a/spec/ysh-unicode.test.sh +++ b/spec/ysh-unicode.test.sh @@ -79,10 +79,10 @@ echo status too_big=$? # python2 -c 'import sys; c = sys.argv[1].decode("utf-8"); print len(c)' "$too_big" var max = u'\u{10ffff}' -pp line (max) +pp test_ (max) var too_big = u'\u{110000}' -pp line (too_big) # should not get here +pp test_ (too_big) # should not get here # These are errors too var max = b'\u{10ffff}' @@ -111,11 +111,11 @@ EOF echo "var x = u'"$max"'; = x" | $SH echo status=$? -#pp line (_reply) +#pp test_ (_reply) echo "var x = u'"$too_big"'; = x" | $SH echo status=$? -#pp line (_reply) +#pp test_ (_reply) ## STDOUT: ## END @@ -138,24 +138,24 @@ EOF echo '"'$max'"' | json read echo status=$? -#pp line (_reply) +#pp test_ (_reply) # Need to propagate the reason here echo '"'$too_big'"' | json read echo status=$? -#pp line (_reply) +#pp test_ (_reply) # J8 string echo "u'"$max"'" | json8 read echo status=$? -#pp line (_reply) +#pp test_ (_reply) echo "u'"$too_big"'" | json8 read echo status=$? -#pp line (_reply) +#pp test_ (_reply) ## STDOUT: status=0 @@ -164,7 +164,7 @@ status=0 status=1 ## END -#### Max code point: json, json8, = keyword, pp line +#### Max code point: json, json8, = keyword, pp test_ var max = u'\u{10ffff}' @@ -172,7 +172,7 @@ json write (max) json8 write (max) = max -pp line (max) +pp test_ (max) #echo "var x = u'"$max"'; = x" | $SH diff --git a/stdlib/ysh/yblocks-test.ysh b/stdlib/ysh/yblocks-test.ysh index 71f6304717..a1e6989628 100755 --- a/stdlib/ysh/yblocks-test.ysh +++ b/stdlib/ysh/yblocks-test.ysh @@ -8,7 +8,7 @@ source $LIB_OSH/task-five.sh proc _check (; val) { # TODO: assert if (not val) { - pp line (val) + pp test_ (val) error "Failed:" } } @@ -31,7 +31,7 @@ proc test-yb-capture { yb-capture (&r) { echo hi } - #pp line (r) + #pp test_ (r) _check (0 === r.status) _check (u'hi\n' === r.stdout) diff --git a/stdlib/ysh/yblocks.ysh b/stdlib/ysh/yblocks.ysh index ded0b8149e..c2b9b2f689 100755 --- a/stdlib/ysh/yblocks.ysh +++ b/stdlib/ysh/yblocks.ysh @@ -23,7 +23,7 @@ proc yb-capture(; out; ; block) { var result = {status: _pipeline_status[0], stdout} #echo 'result-1' - #pp line (result) + #pp test_ (result) call out->setValue(result) } @@ -35,11 +35,11 @@ proc yb-capture-2(; out; ; block) { try { eval (block) 2>&1 | read --all (&stderr) } - #pp line (_pipeline_status) + #pp test_ (_pipeline_status) var result = {status: _pipeline_status[0], stderr} #echo 'result-2' - #pp line (result) + #pp test_ (result) call out->setValue(result) } diff --git a/test/ysh-parse-errors.sh b/test/ysh-parse-errors.sh index a5928c3c88..c973c0b2a6 100755 --- a/test/ysh-parse-errors.sh +++ b/test/ysh-parse-errors.sh @@ -1250,29 +1250,29 @@ test-bug-1118() { } test-bug-1850() { - _ysh-should-parse 'pp line (42); pp line (43)' - #_osh-should-parse 'pp line (42); pp line (43)' + _ysh-should-parse 'pp test_ (42); pp line (43)' + #_osh-should-parse 'pp test_ (42); pp line (43)' # Extra word is bad - _ysh-parse-error 'pp line (42) extra' + _ysh-parse-error 'pp test_ (42) extra' # Bug -- newline or block should come after arg list - _ysh-parse-error 'pp line (42), echo' + _ysh-parse-error 'pp test_ (42), echo' # This properly checks a similar error. It's in a word. - _ysh-parse-error 'pp line @(echo), echo' + _ysh-parse-error 'pp test_ @(echo), echo' # Common cases - _ysh-should-parse 'pp line (42)' - _ysh-should-parse 'pp line (42) ' - _ysh-should-parse 'pp line (42);' - _ysh-should-parse 'pp line (42) { echo hi }' + _ysh-should-parse 'pp test_ (42)' + _ysh-should-parse 'pp test_ (42) ' + _ysh-should-parse 'pp test_ (42);' + _ysh-should-parse 'pp test_ (42) { echo hi }' # Original bug # Accidental comma instead of ; # Wow this is parsed horribly - (42) replaced (43) - _ysh-parse-error 'pp line (42), pp line (43)' + _ysh-parse-error 'pp test_ (42), pp line (43)' } test-bug-1850-more() { diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 35f92135ee..376bd8acc4 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -734,9 +734,9 @@ test-equality() { test-float-equality() { _ysh-expr-error ' var x = 1 -pp line (42.0 === x)' +pp test_ (42.0 === x)' - _ysh-expr-error 'pp line (2.0 === 1.0)' + _ysh-expr-error 'pp test_ (2.0 === 1.0)' } test-place() { @@ -906,21 +906,21 @@ test-setglobal() { _ysh-should-run ' var a = [0] setglobal a[1-1] = 42 -pp line (a) +pp test_ (a) ' _ysh-expr-error ' var a = [0] setglobal a[a.bad] = 42 -pp line (a) +pp test_ (a) ' _ysh-should-run ' var d = {e:{f:0}} setglobal d.e.f = 42 -pp line (d) +pp test_ (d) setglobal d.e.f += 1 -pp line (d) +pp test_ (d) ' } From d6a15be7a07cb1373ecba32599b0ae97c3b23cc9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 29 Jul 2024 01:42:30 -0400 Subject: [PATCH 084/506] [osh] Make behavior of var x = $(echo command sub) more consistent In YSH, we failed as expected. But in OSH, we didn't have all the errexit options on. We haven't fully specified the behavior of 'var' in OSH. There are two options: - we could leave it out, just like we leave out proc and func now - we could enable var (and setvar setglobal), proc, and func - this is the strategy of making ALL of YSH available in OSH, without every typing bin/ysh, or shopt --set ysh:all There is a similar issue with set -o nounset - it is not set in a var statement running under OSH. I think the most principled thing to do is to turn on ysh:all inside var. And likewise in proc and func. I may try that next. --- core/state.py | 18 ++++++++++++++++++ spec/osh-bugs.test.sh | 31 +++++++++++++++++++++++++++++++ test/spec.sh | 4 ++++ 3 files changed, 53 insertions(+) create mode 100644 spec/osh-bugs.test.sh diff --git a/core/state.py b/core/state.py index 33fd16ac40..c04623f931 100644 --- a/core/state.py +++ b/core/state.py @@ -286,7 +286,21 @@ class ctx_YshExpr(object): def __init__(self, mutable_opts): # type: (MutableOpts) -> None + + # Similar to $LIB_OSH/bash-strict.sh + + # TODO: consider errexit:all group, or even ysh:all + # It would be nice if this were more efficient mutable_opts.Push(option_i.command_sub_errexit, True) + mutable_opts.Push(option_i.errexit, True) + mutable_opts.Push(option_i.pipefail, True) + mutable_opts.Push(option_i.inherit_errexit, True) + mutable_opts.Push(option_i.strict_errexit, True) + + # What about nounset? This has a similar pitfall -- it's not running + # like YSH. + # e.g. var x = $(echo $zz) + self.mutable_opts = mutable_opts def __enter__(self): @@ -296,6 +310,10 @@ def __enter__(self): def __exit__(self, type, value, traceback): # type: (Any, Any, Any) -> None self.mutable_opts.Pop(option_i.command_sub_errexit) + self.mutable_opts.Pop(option_i.errexit) + self.mutable_opts.Pop(option_i.pipefail) + self.mutable_opts.Pop(option_i.inherit_errexit) + self.mutable_opts.Pop(option_i.strict_errexit) class ctx_ErrExit(object): diff --git a/spec/osh-bugs.test.sh b/spec/osh-bugs.test.sh new file mode 100644 index 0000000000..45dd5d76d8 --- /dev/null +++ b/spec/osh-bugs.test.sh @@ -0,0 +1,31 @@ +# For OSH only functionality + +#### var x = $(echo bad; false) in OSH + +#shopt -s verbose_errexit + +# This turns on command_sub_errexit and fails +var x = $(echo bad; false) +echo 'unreachable' + +pp test_ (x) + +## status: 1 +## STDOUT: +## END + + +#### var x = $(echo one; false; echo two) in OSH + +#shopt -s verbose_errexit + +# I don't understand why this doesn't fail +var x = $(echo one; false; echo two) +echo 'unreachable' + +pp test_ (x) + +## status: 1 +## STDOUT: +## END + diff --git a/test/spec.sh b/test/spec.sh index 8bfcfb551d..c264e96378 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -87,6 +87,10 @@ bugs() { run-file bugs "$@" } +osh-bugs() { + run-file osh-bugs "$@" +} + TODO-deprecate() { run-file TODO-deprecate "$@" } From f3a1da9c91ef16b1676bc626e9693298334f5960 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 29 Jul 2024 12:00:59 -0400 Subject: [PATCH 085/506] [doc/ref] Update stdlib docs. Also convert yblocks-test.sh to use assert. Looks nice! --- doc/ref/chap-stdlib.md | 118 +++++++++++++++++++++++++++++------ doc/ref/toc-ysh.md | 42 ++++++------- stdlib/osh/no-quotes-test.sh | 8 +++ stdlib/ysh/yblocks-test.ysh | 34 ++++------ 4 files changed, 140 insertions(+), 62 deletions(-) diff --git a/doc/ref/chap-stdlib.md b/doc/ref/chap-stdlib.md index b5e37d7af8..e97da0fef3 100644 --- a/doc/ref/chap-stdlib.md +++ b/doc/ref/chap-stdlib.md @@ -22,6 +22,81 @@ for OSH and YSH.
+## two + +These functions are in `two.sh` + + source $OSH_LIB/two.sh + +### log + +Write a message to stderr: + + log "hi $x" + log '---' + +### die + +Write an error message with the script name, and exit with status 1. + + die 'Expected a number' + +## no-quotes + +### nq-assert + +Use the syntax of the [test][] builtin to assert a condition is true. + + nq-assert 99 = "$status" + nq-assert "$status" -lt 2 + + +[test]: chap-builtin-cmd.html#test + +### nq-run + +Run a command and "return" its status with nameref variables. + + test-foo() { + local status + + nq-run status \ + false + nq-assert 1 = "$status" + } + +### nq-capture + +Run a command and return its status and stdout. + +### nq-capture-2 + +Run a command and return its status and stderr. + +### nq-redir + +Run a command and return its status and a file with its stdout, so you can diff +it. + +### nq-redir-2 + +Run a command and return its status and a file with its stderr, so you can diff +it. + +## task-five + +### task-five + +Dispatch to shell functions, and provide BYO test enumeration. + +OSH: + + task-five "$@" + +YSH: + + task-five @ARGV + ## math ### abs() @@ -32,7 +107,7 @@ Compute the absolute (positive) value of a number (float or int). = abs(0) # => 0 = abs(1) # => 1 -Note, you will need to `source --builtin math.ysh` to use this function. +Note, you will need to `source $LIB_YSH/math.ysh` to use this function. ### max() @@ -48,7 +123,7 @@ For example: = max(1, 2) # => 2 = max([1, 2, 3]) # => 3 -Note, you will need to `source --builtin math.ysh` to use this function. +Note, you will need to `source $LIB_YSH/math.ysh` to use this function. ### min() @@ -64,7 +139,7 @@ For example: = min(2, 3) # => 2 = max([1, 2, 3]) # => 1 -Note, you will need to `source --builtin math.ysh` to use this function. +Note, you will need to `source $LIB_YSH/math.ysh` to use this function. ### round() @@ -80,7 +155,7 @@ Returns 0 for an empty list. = sum([0]) # => 0 = sum([1, 2, 3]) # => 6 -Note, you will need to `source --builtin list.ysh` to use this function. +Note, you will need to `source $LIB_YSH/list.ysh` to use this function. ## list @@ -97,7 +172,7 @@ If the list is empty, return true. = any([false, true]) # => false = any(["foo", true, true]) # => true -Note, you will need to `source --builtin list.ysh` to use this function. +Note, you will need to `source $LIB_YSH/list.ysh` to use this function. ### any() @@ -111,7 +186,7 @@ If the list is empty, return false. = any([false, false]) # => false = any([false, "foo", false]) # => true -Note, you will need to `source --builtin list.ysh` to use this function. +Note, you will need to `source $LIB_YSH/list.ysh` to use this function. ### repeat() @@ -125,33 +200,38 @@ Negative repetitions are equivalent to zero: = repeat('foo', -5) # => '' = repeat(['foo', 'bar'], -5) # => [] -## two +## yblocks -These functions are in `two.sh` +Helpers to assert the status and output of commands. - source $OSH_LIB/two.sh +### yb-capture -### log - -Write a message to stderr: +Capture the status and stdout of a command block: - log "hi $x" - log '---' + yb-capture (&r) { + echo hi + } + assert [0 === r.status] + assert [u'hi\n' === r.stdout] -### die +### yb-capture-2 -Write an error message with the script name, and exit with status 1. +Capture the status and stderr of a command block: - die 'Expected a number' + yb-capture-2 (&r) { + echo hi >& 2 + } + assert [0 === r.status] + assert [u'hi\n' === r.stderr] -## Args Parser +## args YSH includes a command-line argument parsing utility called `parseArgs`. This is intended to be used for command-line interfaces to YSH programs. To use it, first import `args.ysh`: - source --builtin args.ysh + source $LIB_YSH/args.ysh Then, create an argument parser **spec**ification: diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index ff8e5e7ac0..382d87d3ab 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -125,7 +125,6 @@ X [Wok] _field() write Like echo, with --, --sep, --end fork forkwait Replace & and (), and takes a block fopen Open multiple streams, takes a block - X dbg Only thing that can be used in funcs [Hay Config] hay haynode For DSLs and config files [Completion] compadjust compexport [Data Formats] json read write @@ -136,32 +135,30 @@ X [Wok] _field() Standard Librarystdlib

+```chapter-links-stdlib + [math] abs() max() min() X round() + sum() + [list] all() any() repeat() + [yblocks] yb-capture yb-capture-2 + [args] parser flag arg rest + parseArgs() +``` + +Design for streams and tables (awk/xargs/dplyr): + ```chapter-links-stdlib_42 - [math] abs() - max() min() - X round() - sum() - [list] all() any() - repeat() - [args] parser Parse command line arguments - flag - arg - rest - parseArgs() - [yblocks] yb-capture - yb-capture-2 X [Lines] slurp-by combine adjacent lines into cells -X [Awk] each-line --j8 --max-jobs (Str, Template, Block) - xargs - each-row --max-jobs (Str, Template, Block) - xargs +X [Awk] each-line --j8 --max-jobs (Str, Template, Block) + each-row --max-jobs (Str, Template, Block) each-word xargs-like splitting, similar to IFS too split-by (str=\n, ifs=':', pattern=/s+/) - if-split-by + if-split-by only lines that match chop alias for split-by (pattern=/s+/) must-match (/ /) - if-match -X [Table Create] table --by-row --by-col (&place); construct/parse a table + if-match only lines that match +X [Table Create] table construct/parse --by-row --by-col (&place) table/cols cols name age - cols name:Str age:Int types type Str Int attr attr units - secs @@ -169,16 +166,17 @@ X [Table Create] table --by-row --by-col (&place); construct/p table cat concatenate TSV8 table align to ssv8 table tabify to tsv8 - table header (cols = :|name age|, types = :|Str Int|, units = :|- secs|) + table header cols = :|name age|, types = :|Str Int|, ... table slice e.g. slice (1, -1) slice (5, 7) table to-tsv lose type info, and error on \t in cells X [Table Ops] where subset of rows; dplyr filter() pick subset of columns ('select' taken by shell) - mutate transmute [average = count / sum] - drop the ones that are used? + mutate [average = count / sum] + transmuate drop columns that are used rename (bytes='bytes', path='filename') group-by add a column with a group ID [ext] sort-by sort by columns; dplyr arrange() [ext] - summary count, sum, histogram, any, all, reduce(), etc. [ext] + summary count/sum, histogram, any/all, reduce, ... ``` ## Builtins @@ -47,4 +80,11 @@ missing documentation for shell! ### [trap]($help) +## Appendix: Non-Shell Tools +- `xargs` and `xargs -P` +- `find -exec` +- `make -j` + - doesn't do anything smart with output +- `ninja` + - buffers output too diff --git a/frontend/reader.py b/frontend/reader.py index be37491e11..fcdbad85f2 100644 --- a/frontend/reader.py +++ b/frontend/reader.py @@ -88,9 +88,9 @@ class FileLineReader(_Reader): def __init__(self, f, arena): # type: (mylib.LineReader, Arena) -> None """ - Args: - lines: List of (line_id, line) pairs - """ + Args: + lines: List of (line_id, line) pairs + """ _Reader.__init__(self, arena) self.f = f self.last_line_hint = False diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 8c569547e4..37c83ecc51 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1959,11 +1959,15 @@ def ExecuteAndCatch(self, node, cmd_flags=0): if cmd_flags & Optimize: node = self._RemoveSubshells(node) #if self.exec_opts.no_fork_last(): + + # Bug: analysis happens too early: + # + # sh -c 'trap "echo trap" EXIT; date' + #if not self.trap_state.ThisProcessHasTraps(): + self._NoForkLast(node) # turn the last ones into exec - # wow: this makes a difference in job control test - # yeah there is a PID difference of two - # we have to restore nofork + # TODO: this makes a difference in job control test #self._NoForkSentence(node) if 0: diff --git a/test/syscall.sh b/test/syscall.sh index 0fed47e9b4..fb2926876f 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -16,7 +16,10 @@ YSH=${YSH:-ysh} # Compare bash 4 vs. bash 5 #readonly -a SHELLS=(dash bash-4.4 bash $OSH) -readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash yash $OSH) +#readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash yash $OSH) + +# Remove yash since functions are over-optimized - by-code.wrapped +readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash $OSH) readonly BASE_DIR='_tmp/syscall' # What we'll publish readonly RAW_DIR='_tmp/syscall-raw' # Raw data @@ -56,6 +59,11 @@ run-case() { local num=$1 local code_str=$2 + local func_wrap=${3:-} + + if test -n "$func_wrap"; then + code_str="wrapper() { $code_str; }; wrapper" + fi for sh in "${SHELLS[@]}"; do local out_prefix=$RAW_DIR/${sh}__${num} @@ -102,9 +110,12 @@ echo hi # external command date -# Oil sentence +# OSH calls this "sentence" date ; +# trap - bash has special logic for this +trap 'echo mytrap' EXIT; date + # external then builtin date; echo hi @@ -196,6 +207,14 @@ date | read x # osh does 5 when others do 3. ( echo a; echo b ) | ( wc -l ) + +echo hi & wait + +date & wait + +echo hi | wc -l & wait + +date | wc -l & wait EOF # Discarded because they're identical @@ -316,6 +335,7 @@ readonly MAX_CASES=100 by-code() { ### Run cases that vary by code snippet + local func_wrap=${1:-} if ! strace true; then echo "Aborting because we couldn't run strace" @@ -329,7 +349,13 @@ by-code() { write-sourced - local suite='by-code' + local suite + if test -n "$func_wrap"; then + suite='by-code-wrapped' + else + suite='by-code' + fi + local cases=$BASE_DIR/cases.${suite}.txt number-cases > $cases @@ -339,7 +365,7 @@ by-code() { echo "$num $code_str" echo - run-case $num "$code_str" + run-case $num "$code_str" "$func_wrap" done # omit total line @@ -399,6 +425,10 @@ soil-run() { # Note: Only $BASE_DIR/*.txt is included in the release/$VERSION/other.wwz by-code + + # wrapped + by-code T + by-input echo 'OK' From d798afa54c9d67f4fb70fb088ad1428dadde6590 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 6 Aug 2024 21:33:32 -0400 Subject: [PATCH 112/506] [core] fork() optimization is disabled when the process has traps This is issue #1853. - In ExecuteAndCatch(), if cmd_flags & Optimize, we find the last command.Simple in the process and mark it. - When running command.Simple, we look at that flag and decide whether to pass DO_FORK to the executor. --- builtin/meta_osh.py | 7 +++---- core/executor.py | 2 +- frontend/syntax.asdl | 4 ++-- frontend/syntax_abbrev.py | 3 ++- osh/cmd_eval.py | 17 ++++++++++++----- osh/cmd_parse.py | 5 +++-- test/syscall.sh | 7 +++++++ 7 files changed, 30 insertions(+), 15 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index 29bf756320..4a5a1db0cd 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -294,12 +294,11 @@ def Run(self, cmd_val): cmd_val.pos_args, cmd_val.named_args, cmd_val.block_arg) - # If we respected do_fork here instead of passing True, the case - # 'command date | wc -l' would take 2 processes instead of 3. But no other - # shell does that, and this rare case isn't worth the bookkeeping. - # See test/syscall cmd_st = CommandStatus.CreateNull(alloc_lists=True) + # If we respected do_fork here instead of passing DO_FORK + # unconditionally, the case 'command date | wc -l' would take 2 + # processes instead of 3. See test/syscall run_flags = executor.DO_FORK | executor.NO_CALL_PROCS if arg.p: run_flags |= executor.USE_DEFAULT_PATH diff --git a/core/executor.py b/core/executor.py index 2e948af615..ca3e94f2c9 100644 --- a/core/executor.py +++ b/core/executor.py @@ -536,7 +536,7 @@ def RunCommandSub(self, cs_part): # Blame < because __cat has no location blame_tok = redir_node.redirects[0].op simple = command.Simple(blame_tok, [], [cat_word], None, None, - True) + False) # MUTATE redir node so it's like $( hnode_t p_node = runtime.NewRecord('C') - if (obj.more_env or obj.typed_args or obj.block or obj.do_fork == False): + if (obj.more_env or obj.typed_args or obj.block or + obj.is_last_cmd == True): return None # we have other fields to display; don't abbreviate p_node.abbrev = True diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 37c83ecc51..760eac5978 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -811,8 +811,15 @@ def _DoSimple(self, node, cmd_st): # shells aren't consistent. # self.mem.SetLastArgument('') - run_flags = executor.DO_FORK if node.do_fork else 0 - # NOTE: RunSimpleCommand never returns when do_fork=False! + if self.trap_state.ThisProcessHasTraps(): + run_flags = executor.DO_FORK + else: + if node.is_last_cmd: + run_flags = 0 + else: + run_flags = executor.DO_FORK + + # NOTE: RunSimpleCommand may never return if len(node.more_env): # I think this guard is necessary? is_other_special = False # TODO: There are other special builtins too! if cmd_val.tag() == cmd_value_e.Assign or is_other_special: @@ -1870,7 +1877,7 @@ def _NoForkLast(self, node): with tagswitch(node) as case: if case(command_e.Simple): node = cast(command.Simple, UP_node) - node.do_fork = False + node.is_last_cmd = True if 0: log('Simple optimized') @@ -1886,7 +1893,7 @@ def _NoForkLast(self, node): self._NoForkLast(node.child) elif case(command_e.CommandList): - # Subshells start with CommandList, even if there's only one. + # Subshells often have a CommandList child node = cast(command.CommandList, UP_node) self._NoForkLast(node.children[-1]) @@ -1907,7 +1914,7 @@ def _NoForkSentence(self, node): with tagswitch(node) as case: if case(command_e.Simple): node = cast(command.Simple, UP_node) - node.do_fork = False + node.is_last_cmd = False if 0: log('Simple optimized') diff --git a/osh/cmd_parse.py b/osh/cmd_parse.py index a755caf6f0..463049650d 100644 --- a/osh/cmd_parse.py +++ b/osh/cmd_parse.py @@ -379,8 +379,9 @@ def _MakeSimpleCommand( more_env = [] # type: List[EnvPair] _AppendMoreEnv(preparsed_list, more_env) - # do_fork by default - return command.Simple(blame_tok, more_env, words3, typed_args, block, True) + # is_last_cmd is False by default + return command.Simple(blame_tok, more_env, words3, typed_args, block, + False) class VarChecker(object): diff --git a/test/syscall.sh b/test/syscall.sh index fb2926876f..005daf630b 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -215,6 +215,13 @@ date & wait echo hi | wc -l & wait date | wc -l & wait + +trap 'echo mytrap' EXIT; date & wait + +trap 'echo mytrap' EXIT; date | wc -l & wait + +# trap in SubProgramThunk +{ trap 'echo mytrap' EXIT; date; } & wait EOF # Discarded because they're identical From f234919e66de96a07f58a8dfe08346ce8823f235 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 6 Aug 2024 22:24:19 -0400 Subject: [PATCH 113/506] [core] Optimize forks by moving decision into executor We pass cmd_val.is_last_cmd to builtins, and they pass back executor.IS_LAST_CMD This makes this case more efficient: command date | wc -l Prior to this change, we "lost" the info that 'date' is the last command. It's now identical to this: date | wc -l On test/syscall, we are now more optimal. --- builtin/error_ysh.py | 10 +++++----- builtin/meta_osh.py | 25 ++++++++++++++----------- builtin/process_osh.py | 4 ++-- core/executor.py | 11 ++++++++--- core/runtime.asdl | 1 + osh/cmd_eval.py | 15 ++++++--------- osh/word_eval.py | 24 ++++++++++++------------ 7 files changed, 48 insertions(+), 42 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index eac5ea9c1a..5caf6270a0 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -213,13 +213,13 @@ def Run(self, cmd_val): e_usage('expected a command to run', loc.Missing) argv, locs = arg_r.Rest2() - cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.typed_args, - cmd_val.pos_args, cmd_val.named_args, - cmd_val.block_arg) + cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd, + cmd_val.typed_args, cmd_val.pos_args, + cmd_val.named_args, cmd_val.block_arg) cmd_st = CommandStatus.CreateNull(alloc_lists=True) - status = self.shell_ex.RunSimpleCommand(cmd_val2, cmd_st, - executor.DO_FORK) + run_flags = executor.IS_LAST_CMD if cmd_val.is_last_cmd else 0 + status = self.shell_ex.RunSimpleCommand(cmd_val2, cmd_st, run_flags) if status not in (0, 1): e_die_status(status, diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index 4a5a1db0cd..a31c6bf1f7 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -290,16 +290,18 @@ def Run(self, cmd_val): return status - cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.typed_args, - cmd_val.pos_args, cmd_val.named_args, - cmd_val.block_arg) + cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd, + cmd_val.typed_args, cmd_val.pos_args, + cmd_val.named_args, cmd_val.block_arg) cmd_st = CommandStatus.CreateNull(alloc_lists=True) # If we respected do_fork here instead of passing DO_FORK # unconditionally, the case 'command date | wc -l' would take 2 # processes instead of 3. See test/syscall - run_flags = executor.DO_FORK | executor.NO_CALL_PROCS + run_flags = executor.NO_CALL_PROCS + if cmd_val.is_last_cmd: + run_flags |= executor.IS_LAST_CMD if arg.p: run_flags |= executor.USE_DEFAULT_PATH @@ -309,8 +311,9 @@ def Run(self, cmd_val): def _ShiftArgv(cmd_val): # type: (cmd_value.Argv) -> cmd_value.Argv return cmd_value.Argv(cmd_val.argv[1:], cmd_val.arg_locs[1:], - cmd_val.typed_args, cmd_val.pos_args, - cmd_val.named_args, cmd_val.block_arg) + cmd_val.is_last_cmd, cmd_val.typed_args, + cmd_val.pos_args, cmd_val.named_args, + cmd_val.block_arg) class Builtin(vm._Builtin): @@ -370,13 +373,13 @@ def Run(self, cmd_val): self.errfmt.PrintMessage('runproc: no proc named %r' % name) return 1 - cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.typed_args, - cmd_val.pos_args, cmd_val.named_args, - cmd_val.block_arg) + cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd, + cmd_val.typed_args, cmd_val.pos_args, + cmd_val.named_args, cmd_val.block_arg) cmd_st = CommandStatus.CreateNull(alloc_lists=True) - return self.shell_ex.RunSimpleCommand(cmd_val2, cmd_st, - executor.DO_FORK) + run_flags = executor.IS_LAST_CMD if cmd_val.is_last_cmd else 0 + return self.shell_ex.RunSimpleCommand(cmd_val2, cmd_st, run_flags) def _ResolveName( diff --git a/builtin/process_osh.py b/builtin/process_osh.py index bc8976c465..1901e78d3c 100644 --- a/builtin/process_osh.py +++ b/builtin/process_osh.py @@ -212,8 +212,8 @@ def Run(self, cmd_val): e_die_status(127, 'exec: %r not found' % cmd, cmd_val.arg_locs[1]) # shift off 'exec', and remove typed args because they don't apply - c2 = cmd_value.Argv(cmd_val.argv[i:], cmd_val.arg_locs[i:], None, None, - None, None) + c2 = cmd_value.Argv(cmd_val.argv[i:], cmd_val.arg_locs[i:], + cmd_val.is_last_cmd, None, None, None, None) self.ext_prog.Exec(argv0_path, c2, environ) # NEVER RETURNS # makes mypy and C++ compiler happy diff --git a/core/executor.py b/core/executor.py index ca3e94f2c9..6c20d17c84 100644 --- a/core/executor.py +++ b/core/executor.py @@ -88,8 +88,8 @@ def MaybeWaitOnProcessSubs(self, waiter, status_array): status_array.locs = locs -# Big flgas for RunSimpleCommand -DO_FORK = 1 << 1 +# Big flags for RunSimpleCommand +IS_LAST_CMD = 1 << 1 NO_CALL_PROCS = 1 << 2 # command ls suppresses function lookup USE_DEFAULT_PATH = 1 << 3 # for command -p ls changes the path @@ -340,8 +340,13 @@ def RunSimpleCommand(self, cmd_val, cmd_st, run_flags): self.errfmt.Print_('%r not found (OILS-ERR-100)' % arg0, arg0_loc) return 127 + if self.trap_state.ThisProcessHasTraps(): + do_fork = True + else: + do_fork = not cmd_val.is_last_cmd + # Normal case: ls / - if run_flags & DO_FORK: + if do_fork: thunk = process.ExternalThunk(self.ext_prog, argv0_path, cmd_val, environ) p = process.Process(thunk, self.job_control, self.job_list, diff --git a/core/runtime.asdl b/core/runtime.asdl index c2af5f8aac..e0ed366004 100644 --- a/core/runtime.asdl +++ b/core/runtime.asdl @@ -22,6 +22,7 @@ module runtime # note: could import 'builtin' from synthetic option_asdl cmd_value = Argv(List[str] argv, List[CompoundWord] arg_locs, + bool is_last_cmd, ArgList? typed_args, # Evaluated args, similar to typed_args.py Reader List[value]? pos_args, Dict[str, value]? named_args, diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 760eac5978..516293e01d 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -120,7 +120,8 @@ def MakeBuiltinArgv(argv1): argv = [''] # dummy for argv[0] argv.extend(argv1) missing = None # type: CompoundWord - return cmd_value.Argv(argv, [missing] * len(argv), None, None, None, None) + return cmd_value.Argv(argv, [missing] * len(argv), False, None, None, None, + None) class Deps(object): @@ -786,7 +787,9 @@ def _DoSimple(self, node, cmd_st): # - $() and <() can have failures. This can happen in DBracket, # DParen, etc. too # - Tracing: this can start processes for proc sub and here docs! - cmd_val = self.word_ev.EvalWordSequence2(words, allow_assign=True) + cmd_val = self.word_ev.EvalWordSequence2(words, + node.is_last_cmd, + allow_assign=True) UP_cmd_val = cmd_val if UP_cmd_val.tag() == cmd_value_e.Argv: @@ -811,13 +814,7 @@ def _DoSimple(self, node, cmd_st): # shells aren't consistent. # self.mem.SetLastArgument('') - if self.trap_state.ThisProcessHasTraps(): - run_flags = executor.DO_FORK - else: - if node.is_last_cmd: - run_flags = 0 - else: - run_flags = executor.DO_FORK + run_flags = executor.IS_LAST_CMD if node.is_last_cmd else 0 # NOTE: RunSimpleCommand may never return if len(node.more_env): # I think this guard is necessary? diff --git a/osh/word_eval.py b/osh/word_eval.py index 76a4de7e30..e8ef229a03 100644 --- a/osh/word_eval.py +++ b/osh/word_eval.py @@ -2162,8 +2162,8 @@ def _DetectAssignBuiltin(self, val0, words, meta_offset): return self._DetectAssignBuiltinStr(val0.s, words, meta_offset) return None - def SimpleEvalWordSequence2(self, words, allow_assign): - # type: (List[CompoundWord], bool) -> cmd_value_t + def SimpleEvalWordSequence2(self, words, is_last_cmd, allow_assign): + # type: (List[CompoundWord], bool, bool) -> cmd_value_t """Simple word evaluation for YSH.""" strs = [] # type: List[str] locs = [] # type: List[CompoundWord] @@ -2222,10 +2222,10 @@ def SimpleEvalWordSequence2(self, words, allow_assign): strs.append(''.join(tmp)) # no split or glob locs.append(w) - return cmd_value.Argv(strs, locs, None, None, None, None) + return cmd_value.Argv(strs, locs, is_last_cmd, None, None, None, None) - def EvalWordSequence2(self, words, allow_assign=False): - # type: (List[CompoundWord], bool) -> cmd_value_t + def EvalWordSequence2(self, words, is_last_cmd, allow_assign=False): + # type: (List[CompoundWord], bool, bool) -> cmd_value_t """Turns a list of Words into a list of strings. Unlike the EvalWord*() methods, it does globbing. @@ -2234,7 +2234,8 @@ def EvalWordSequence2(self, words, allow_assign=False): allow_assign: True for command.Simple, False for BashArray a=(1 2 3) """ if self.exec_opts.simple_word_eval(): - return self.SimpleEvalWordSequence2(words, allow_assign) + return self.SimpleEvalWordSequence2(words, is_last_cmd, + allow_assign) # Parse time: # 1. brace expansion. TODO: Do at parse time. @@ -2325,7 +2326,7 @@ def EvalWordSequence2(self, words, allow_assign=False): # A non-assignment command. # NOTE: Can't look up builtins here like we did for assignment, because # functions can override builtins. - return cmd_value.Argv(strs, locs, None, None, None, None) + return cmd_value.Argv(strs, locs, is_last_cmd, None, None, None, None) def EvalWordSequence(self, words): # type: (List[CompoundWord]) -> List[str] @@ -2333,11 +2334,10 @@ def EvalWordSequence(self, words): They don't allow assignment builtins. """ - UP_cmd_val = self.EvalWordSequence2(words) - - assert UP_cmd_val.tag() == cmd_value_e.Argv - cmd_val = cast(cmd_value.Argv, UP_cmd_val) - return cmd_val.argv + # is_last_cmd is irrelevant + cmd_val = self.EvalWordSequence2(words, False) + assert cmd_val.tag() == cmd_value_e.Argv + return cast(cmd_value.Argv, cmd_val).argv class NormalWordEvaluator(AbstractWordEvaluator): From 0abb57cade56b188b91dcd064a2e1ba8534c33bc Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 6 Aug 2024 22:42:33 -0400 Subject: [PATCH 114/506] [test/unit] Fix tests --- core/process_test.py | 4 ++-- core/test_lib.py | 4 ++-- cpp/obj_layout_test.cc | 2 ++ frontend/args_test.py | 3 ++- osh/word_eval_test.py | 6 ++++-- test/syscall.sh | 7 ++++--- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/core/process_test.py b/core/process_test.py index 40fec10619..9425593d34 100755 --- a/core/process_test.py +++ b/core/process_test.py @@ -77,8 +77,8 @@ def setUp(self): util.NullDebugFile()) def _ExtProc(self, argv): - arg_vec = cmd_value.Argv(argv, [loc.Missing] * len(argv), None, None, - None, None) + arg_vec = cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None, + None, None, None) argv0_path = None for path_entry in ['/bin', '/usr/bin']: full_path = os.path.join(path_entry, argv[0]) diff --git a/core/test_lib.py b/core/test_lib.py index 67e6379cde..b2d6f32bb0 100644 --- a/core/test_lib.py +++ b/core/test_lib.py @@ -53,8 +53,8 @@ def MakeBuiltinArgv(argv): - return cmd_value.Argv(argv, [loc.Missing] * len(argv), None, None, None, - None) + return cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None, None, + None, None) def FakeTok(id_, val): diff --git a/cpp/obj_layout_test.cc b/cpp/obj_layout_test.cc index bd71c63eb3..5aa3f7cdd5 100644 --- a/cpp/obj_layout_test.cc +++ b/cpp/obj_layout_test.cc @@ -22,6 +22,8 @@ TEST sizeof_syntax() { // Reordered to be 16 bytes log("sizeof(runtime_asdl::Cell) = %d", sizeof(runtime_asdl::Cell)); + // 56 bytes - I think we should try to remove 4 pointers + log("sizeof(runtime_asdl::cmd_value::Argv) = %d", sizeof(runtime_asdl::cmd_value::Argv)); // 24 bytes: std::vector log("sizeof(List) = %d", sizeof(List)); diff --git a/frontend/args_test.py b/frontend/args_test.py index f7b2d5c46b..9b5b73f054 100755 --- a/frontend/args_test.py +++ b/frontend/args_test.py @@ -20,7 +20,8 @@ def _MakeBuiltinArgv(argv): argv = [''] + argv # add dummy since arg_vec includes argv[0] # no location info missing = loc.Missing # type: loc_t - return cmd_value.Argv(argv, [missing] * len(argv), None, None, None, None) + return cmd_value.Argv(argv, [missing] * len(argv), False, None, None, None, + None) def _MakeReader(argv): diff --git a/osh/word_eval_test.py b/osh/word_eval_test.py index edfedb5f23..e070717794 100755 --- a/osh/word_eval_test.py +++ b/osh/word_eval_test.py @@ -96,7 +96,9 @@ def testEvalWordSequence_Errors(self): node = assertParseSimpleCommand(self, case) ev = InitEvaluator() try: - argv = ev.EvalWordSequence2(node.words, allow_assign=True) + argv = ev.EvalWordSequence2(node.words, + False, + allow_assign=True) except error.FatalRuntime: pass else: @@ -143,7 +145,7 @@ def testEvalWordSequence(self): print('\t%s' % case) node = assertParseSimpleCommand(self, case) ev = InitEvaluator() - argv = ev.EvalWordSequence2(node.words, allow_assign=True) + argv = ev.EvalWordSequence2(node.words, False, allow_assign=True) print() print('\tcmd_value:') diff --git a/test/syscall.sh b/test/syscall.sh index 005daf630b..c663bcc5e3 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -16,10 +16,11 @@ YSH=${YSH:-ysh} # Compare bash 4 vs. bash 5 #readonly -a SHELLS=(dash bash-4.4 bash $OSH) -#readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash yash $OSH) +readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash yash $OSH) -# Remove yash since functions are over-optimized - by-code.wrapped -readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash $OSH) +# yash does something fundamentally different in by-code.wrapped - it +# understands functions +# SHELLS+=(yash) readonly BASE_DIR='_tmp/syscall' # What we'll publish readonly RAW_DIR='_tmp/syscall-raw' # Raw data From c78b9936d8143cc1f6cfb376b7479573b5bf713a Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 6 Aug 2024 23:15:17 -0400 Subject: [PATCH 115/506] [ysh refactor] Move optional ProcArgs out of cmd_value.Argv This logically separates YSH typed args, and it means that cmd_value.Argv fits in the 48-byte pool. benchmarks2/gc-cachegrind shows a slight speedup. --- builtin/error_ysh.py | 3 +- builtin/io_osh.py | 2 +- builtin/json_ysh.py | 4 +-- builtin/meta_osh.py | 12 +++----- builtin/process_osh.py | 2 +- builtin/pure_osh.py | 2 +- builtin/read_osh.py | 8 +++--- core/executor.py | 4 +-- core/process_test.py | 3 +- core/runtime.asdl | 17 ++++++++---- core/test_lib.py | 3 +- cpp/obj_layout_test.cc | 2 +- frontend/args_test.py | 3 +- frontend/flag_util.py | 15 +++++----- frontend/typed_args.py | 41 +++++++++++++++++---------- osh/cmd_eval.py | 11 ++++---- osh/word_eval.py | 4 +-- test/syscall.sh | 5 ++-- ysh/func_proc.py | 63 +++++++++++++++++++++++------------------- 19 files changed, 110 insertions(+), 94 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index 5caf6270a0..e5802892da 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -214,8 +214,7 @@ def Run(self, cmd_val): argv, locs = arg_r.Rest2() cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd, - cmd_val.typed_args, cmd_val.pos_args, - cmd_val.named_args, cmd_val.block_arg) + cmd_val.proc_args) cmd_st = CommandStatus.CreateNull(alloc_lists=True) run_flags = executor.IS_LAST_CMD if cmd_val.is_last_cmd else 0 diff --git a/builtin/io_osh.py b/builtin/io_osh.py index 2bd05aca30..514d1039ef 100644 --- a/builtin/io_osh.py +++ b/builtin/io_osh.py @@ -58,7 +58,7 @@ def Run(self, cmd_val): argv = cmd_val.argv[1:] if self.exec_opts.simple_echo(): - typed_args.DoesNotAccept(cmd_val.typed_args) # Disallow echo (42) + typed_args.DoesNotAccept(cmd_val.proc_args) # Disallow echo (42) arg = self._SimpleFlag() # Avoid parsing -e -n else: attrs, arg_r = flag_util.ParseLikeEcho('echo', cmd_val) diff --git a/builtin/json_ysh.py b/builtin/json_ysh.py index 62d87d898f..0fe8b279e7 100644 --- a/builtin/json_ysh.py +++ b/builtin/json_ysh.py @@ -96,12 +96,12 @@ def Run(self, cmd_val): attrs = flag_util.Parse('json_read', arg_r) #arg_jr = arg_types.json_read(attrs.attrs) - if cmd_val.typed_args: # json read (&x) + if cmd_val.proc_args: # json read (&x) rd = typed_args.ReaderForProc(cmd_val) place = rd.PosPlace() rd.Done() - blame_loc = cmd_val.typed_args.left # type: loc_t + blame_loc = cmd_val.proc_args.typed_args.left # type: loc_t else: # json read var_name = '_reply' diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index a31c6bf1f7..14619d3fb4 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -62,7 +62,7 @@ def __init__( def Run(self, cmd_val): # type: (cmd_value.Argv) -> int - if cmd_val.typed_args: # eval (mycmd) + if cmd_val.proc_args: # eval (mycmd) rd = typed_args.ReaderForProc(cmd_val) cmd = rd.PosCommand() rd.Done() @@ -291,8 +291,7 @@ def Run(self, cmd_val): return status cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd, - cmd_val.typed_args, cmd_val.pos_args, - cmd_val.named_args, cmd_val.block_arg) + cmd_val.proc_args) cmd_st = CommandStatus.CreateNull(alloc_lists=True) @@ -311,9 +310,7 @@ def Run(self, cmd_val): def _ShiftArgv(cmd_val): # type: (cmd_value.Argv) -> cmd_value.Argv return cmd_value.Argv(cmd_val.argv[1:], cmd_val.arg_locs[1:], - cmd_val.is_last_cmd, cmd_val.typed_args, - cmd_val.pos_args, cmd_val.named_args, - cmd_val.block_arg) + cmd_val.is_last_cmd, cmd_val.proc_args) class Builtin(vm._Builtin): @@ -374,8 +371,7 @@ def Run(self, cmd_val): return 1 cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd, - cmd_val.typed_args, cmd_val.pos_args, - cmd_val.named_args, cmd_val.block_arg) + cmd_val.proc_args) cmd_st = CommandStatus.CreateNull(alloc_lists=True) run_flags = executor.IS_LAST_CMD if cmd_val.is_last_cmd else 0 diff --git a/builtin/process_osh.py b/builtin/process_osh.py index 1901e78d3c..470be69af5 100644 --- a/builtin/process_osh.py +++ b/builtin/process_osh.py @@ -213,7 +213,7 @@ def Run(self, cmd_val): # shift off 'exec', and remove typed args because they don't apply c2 = cmd_value.Argv(cmd_val.argv[i:], cmd_val.arg_locs[i:], - cmd_val.is_last_cmd, None, None, None, None) + cmd_val.is_last_cmd, None) self.ext_prog.Exec(argv0_path, c2, environ) # NEVER RETURNS # makes mypy and C++ compiler happy diff --git a/builtin/pure_osh.py b/builtin/pure_osh.py index bb4442495c..ff881e2239 100644 --- a/builtin/pure_osh.py +++ b/builtin/pure_osh.py @@ -47,7 +47,7 @@ def Run(self, cmd_val): # type: (cmd_value.Argv) -> int # These ignore regular args, but shouldn't accept typed args. - typed_args.DoesNotAccept(cmd_val.typed_args) + typed_args.DoesNotAccept(cmd_val.proc_args) return self.status diff --git a/builtin/read_osh.py b/builtin/read_osh.py index 7946909e63..c70e1c27ed 100644 --- a/builtin/read_osh.py +++ b/builtin/read_osh.py @@ -349,12 +349,12 @@ def _ReadYsh(self, arg, arg_r, cmd_val): """ place = None # type: value.Place - if cmd_val.typed_args: # read --flag (&x) + if cmd_val.proc_args: # read --flag (&x) rd = typed_args.ReaderForProc(cmd_val) place = rd.PosPlace() rd.Done() - blame_loc = cmd_val.typed_args.left # type: loc_t + blame_loc = cmd_val.proc_args.typed_args.left # type: loc_t else: # read --flag var_name = '_reply' @@ -398,10 +398,10 @@ def _Run(self, cmd_val): if arg.raw_line or arg.all or mops.BigTruncate(arg.num_bytes) != -1: return self._ReadYsh(arg, arg_r, cmd_val) - if cmd_val.typed_args: + if cmd_val.proc_args: raise error.Usage( "doesn't accept typed args without --all, or --num-bytes", - cmd_val.typed_args.left) + cmd_val.proc_args.typed_args.left) if arg.t >= 0.0: if arg.t != 0.0: diff --git a/core/executor.py b/core/executor.py index 6c20d17c84..6eb79490c7 100644 --- a/core/executor.py +++ b/core/executor.py @@ -326,10 +326,10 @@ def RunSimpleCommand(self, cmd_val, cmd_st, run_flags): environ = self.mem.GetExported() # Include temporary variables - if cmd_val.typed_args: + if cmd_val.proc_args: e_die( '%r appears to be external. External commands don\'t accept typed args (OILS-ERR-200)' - % arg0, cmd_val.typed_args.left) + % arg0, cmd_val.proc_args.typed_args.left) # Resolve argv[0] BEFORE forking. if run_flags & USE_DEFAULT_PATH: diff --git a/core/process_test.py b/core/process_test.py index 9425593d34..781d47572d 100755 --- a/core/process_test.py +++ b/core/process_test.py @@ -77,8 +77,7 @@ def setUp(self): util.NullDebugFile()) def _ExtProc(self, argv): - arg_vec = cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None, - None, None, None) + arg_vec = cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None) argv0_path = None for path_entry in ['/bin', '/usr/bin']: full_path = os.path.join(path_entry, argv[0]) diff --git a/core/runtime.asdl b/core/runtime.asdl index e0ed366004..50d3f18ddb 100644 --- a/core/runtime.asdl +++ b/core/runtime.asdl @@ -19,15 +19,22 @@ module runtime # in 'local foo', rval is None. AssignArg = (str var_name, value? rval, bool plus_eq, CompoundWord blame_word) + ProcArgs = ( + # Unevaluated args + ArgList typed_args, + + # Evaluated args, similar to typed_args.py Reader + List[value]? pos_args, Dict[str, value]? named_args, + + # block_arg comes from either p (; ; myblock) or p { echo b } + value? block_arg + ) + # note: could import 'builtin' from synthetic option_asdl cmd_value = Argv(List[str] argv, List[CompoundWord] arg_locs, bool is_last_cmd, - ArgList? typed_args, - # Evaluated args, similar to typed_args.py Reader - List[value]? pos_args, Dict[str, value]? named_args, - # block_arg comes from either p (; ; myblock) or p { echo b } - value? block_arg) + ProcArgs? proc_args) | Assign(int builtin_id, List[str] argv, List[CompoundWord] arg_locs, diff --git a/core/test_lib.py b/core/test_lib.py index b2d6f32bb0..e9f1045f30 100644 --- a/core/test_lib.py +++ b/core/test_lib.py @@ -53,8 +53,7 @@ def MakeBuiltinArgv(argv): - return cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None, None, - None, None) + return cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None) def FakeTok(id_, val): diff --git a/cpp/obj_layout_test.cc b/cpp/obj_layout_test.cc index 5aa3f7cdd5..f17d55e7d4 100644 --- a/cpp/obj_layout_test.cc +++ b/cpp/obj_layout_test.cc @@ -22,7 +22,7 @@ TEST sizeof_syntax() { // Reordered to be 16 bytes log("sizeof(runtime_asdl::Cell) = %d", sizeof(runtime_asdl::Cell)); - // 56 bytes - I think we should try to remove 4 pointers + // now 32 bytes, down from 56 log("sizeof(runtime_asdl::cmd_value::Argv) = %d", sizeof(runtime_asdl::cmd_value::Argv)); // 24 bytes: std::vector diff --git a/frontend/args_test.py b/frontend/args_test.py index 9b5b73f054..bc00c01a45 100755 --- a/frontend/args_test.py +++ b/frontend/args_test.py @@ -20,8 +20,7 @@ def _MakeBuiltinArgv(argv): argv = [''] + argv # add dummy since arg_vec includes argv[0] # no location info missing = loc.Missing # type: loc_t - return cmd_value.Argv(argv, [missing] * len(argv), False, None, None, None, - None) + return cmd_value.Argv(argv, [missing] * len(argv), False, None) def _MakeReader(argv): diff --git a/frontend/flag_util.py b/frontend/flag_util.py index cafcc7eedb..65e251b066 100644 --- a/frontend/flag_util.py +++ b/frontend/flag_util.py @@ -3,8 +3,7 @@ """ from __future__ import print_function -from _devbuild.gen.runtime_asdl import cmd_value -from _devbuild.gen.syntax_asdl import ArgList +from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs from core.error import e_usage from frontend import args from frontend import flag_spec @@ -23,18 +22,18 @@ def LookupFlagSpec2(name): return flag_spec.FLAG_SPEC_AND_MORE[name] -def _DoesNotAccept(arg_list): - # type: (Optional[ArgList]) -> None +def _DoesNotAccept(proc_args): + # type: (Optional[ProcArgs]) -> None """ Copy from frontend/typed_args.py, to break dependency """ - if arg_list is not None: - e_usage('got unexpected typed args', arg_list.left) + if proc_args is not None: + e_usage('got unexpected typed args', proc_args.typed_args.left) def ParseCmdVal(spec_name, cmd_val, accept_typed_args=False): # type: (str, cmd_value.Argv, bool) -> Tuple[args._Attributes, args.Reader] if not accept_typed_args: - _DoesNotAccept(cmd_val.typed_args) + _DoesNotAccept(cmd_val.proc_args) arg_r = args.Reader(cmd_val.argv, locs=cmd_val.arg_locs) arg_r.Next() # move past the builtin name @@ -46,7 +45,7 @@ def ParseCmdVal(spec_name, cmd_val, accept_typed_args=False): def ParseLikeEcho(spec_name, cmd_val): # type: (str, cmd_value.Argv) -> Tuple[args._Attributes, args.Reader] - _DoesNotAccept(cmd_val.typed_args) + _DoesNotAccept(cmd_val.proc_args) arg_r = args.Reader(cmd_val.argv, locs=cmd_val.arg_locs) arg_r.Next() # move past the builtin name diff --git a/frontend/typed_args.py b/frontend/typed_args.py index fc06ffd3ea..6c0169d836 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 from __future__ import print_function -from _devbuild.gen.runtime_asdl import cmd_value +from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, LiteralBlock, command_t, expr_t, Token) from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch) @@ -17,10 +17,10 @@ _ = log -def DoesNotAccept(arg_list): - # type: (Optional[ArgList]) -> None - if arg_list is not None: - e_usage('got unexpected typed args', arg_list.left) +def DoesNotAccept(proc_args): + # type: (Optional[ProcArgs]) -> None + if proc_args is not None: + e_usage('got unexpected typed args', proc_args.typed_args.left) def OptionalBlock(cmd_val): @@ -28,7 +28,7 @@ def OptionalBlock(cmd_val): """Helper for shopt, etc.""" cmd = None # type: Optional[command_t] - if cmd_val.typed_args: + if cmd_val.proc_args: r = ReaderForProc(cmd_val) cmd = r.OptionalBlock() r.Done() @@ -40,7 +40,7 @@ def OptionalLiteralBlock(cmd_val): """Helper for Hay """ block = None # type: Optional[LiteralBlock] - if cmd_val.typed_args: + if cmd_val.proc_args: r = ReaderForProc(cmd_val) block = r.OptionalLiteralBlock() r.Done() @@ -50,14 +50,25 @@ def OptionalLiteralBlock(cmd_val): def ReaderForProc(cmd_val): # type: (cmd_value.Argv) -> Reader - # mycpp rewrite: doesn't understand 'or' pattern - pos_args = (cmd_val.pos_args if cmd_val.pos_args is not None else []) - named_args = (cmd_val.named_args if cmd_val.named_args is not None else {}) - - arg_list = (cmd_val.typed_args - if cmd_val.typed_args is not None else ArgList.CreateNull()) - - rd = Reader(pos_args, named_args, cmd_val.block_arg, arg_list) + proc_args = cmd_val.proc_args + + if proc_args: + # mycpp rewrite: doesn't understand 'or' pattern + pos_args = (proc_args.pos_args + if proc_args.pos_args is not None else []) + named_args = (proc_args.named_args + if proc_args.named_args is not None else {}) + + arg_list = (proc_args.typed_args if proc_args.typed_args is not None + else ArgList.CreateNull()) + block_arg = proc_args.block_arg + else: + pos_args = [] + named_args = {} + arg_list = ArgList.CreateNull() + block_arg = None + + rd = Reader(pos_args, named_args, block_arg, arg_list) # Fix location info bug with 'try' or try foo' -- it should get a typed arg rd.SetFallbackLocation(cmd_val.arg_locs[0]) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 516293e01d..3963ae2792 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -55,11 +55,12 @@ from _devbuild.gen.runtime_asdl import ( cmd_value, cmd_value_e, + CommandStatus, + flow_e, RedirValue, redirect_arg, - flow_e, + ProcArgs, scope_e, - CommandStatus, StatusArray, ) from _devbuild.gen.types_asdl import redir_arg_type_e @@ -120,8 +121,7 @@ def MakeBuiltinArgv(argv1): argv = [''] # dummy for argv[0] argv.extend(argv1) missing = None # type: CompoundWord - return cmd_value.Argv(argv, [missing] * len(argv), False, None, None, None, - None) + return cmd_value.Argv(argv, [missing] * len(argv), False, None) class Deps(object): @@ -801,8 +801,9 @@ def _DoSimple(self, node, cmd_st): self.mem.SetLastArgument('') if node.typed_args or node.block: # guard to avoid allocs + cmd_val.proc_args = ProcArgs(node.typed_args, None, None, None) func_proc.EvalTypedArgsToProc(self.expr_ev, self.mutable_opts, - node, cmd_val) + node, cmd_val.proc_args) else: if node.block: e_die("ShAssignment builtins don't accept blocks", diff --git a/osh/word_eval.py b/osh/word_eval.py index e8ef229a03..47f085315c 100644 --- a/osh/word_eval.py +++ b/osh/word_eval.py @@ -2222,7 +2222,7 @@ def SimpleEvalWordSequence2(self, words, is_last_cmd, allow_assign): strs.append(''.join(tmp)) # no split or glob locs.append(w) - return cmd_value.Argv(strs, locs, is_last_cmd, None, None, None, None) + return cmd_value.Argv(strs, locs, is_last_cmd, None) def EvalWordSequence2(self, words, is_last_cmd, allow_assign=False): # type: (List[CompoundWord], bool, bool) -> cmd_value_t @@ -2326,7 +2326,7 @@ def EvalWordSequence2(self, words, is_last_cmd, allow_assign=False): # A non-assignment command. # NOTE: Can't look up builtins here like we did for assignment, because # functions can override builtins. - return cmd_value.Argv(strs, locs, is_last_cmd, None, None, None, None) + return cmd_value.Argv(strs, locs, is_last_cmd, None) def EvalWordSequence(self, words): # type: (List[CompoundWord]) -> List[str] diff --git a/test/syscall.sh b/test/syscall.sh index c663bcc5e3..05f0a7df90 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -14,9 +14,10 @@ source build/dev-shell.sh OSH=${OSH:-osh} YSH=${YSH:-ysh} -# Compare bash 4 vs. bash 5 #readonly -a SHELLS=(dash bash-4.4 bash $OSH) -readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash yash $OSH) + +# Compare bash 4 vs. bash 5 +readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash $OSH) # yash does something fundamentally different in by-code.wrapped - it # understands functions diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 7a550c1bb0..27d35c1da8 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -5,7 +5,7 @@ from __future__ import print_function from _devbuild.gen.id_kind_asdl import Id -from _devbuild.gen.runtime_asdl import cmd_value +from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs from _devbuild.gen.syntax_asdl import (proc_sig, proc_sig_e, Param, ParamGroup, NamedArg, Func, loc, ArgList, expr, expr_e, expr_t) @@ -211,18 +211,18 @@ def EvalTypedArgsToProc( expr_ev, # type: expr_eval.ExprEvaluator mutable_opts, # type: state.MutableOpts node, # type: command.Simple - cmd_val, # type: cmd_value.Argv + proc_args, # type: ProcArgs ): # type: (...) -> None """Evaluate word, typed, named, and block args for a proc.""" - cmd_val.typed_args = node.typed_args + proc_args.typed_args = node.typed_args # We only got here if the call looks like # p (x) # p { echo hi } # p () { echo hi } # So allocate this unconditionally - cmd_val.pos_args = [] + proc_args.pos_args = [] ty = node.typed_args if ty: @@ -230,23 +230,24 @@ def EvalTypedArgsToProc( # Defer evaluation by wrapping in value.Expr for exp in ty.pos_args: - cmd_val.pos_args.append(value.Expr(exp)) + proc_args.pos_args.append(value.Expr(exp)) # TODO: ...spread is illegal n1 = ty.named_args if n1 is not None: - cmd_val.named_args = NewDict() + proc_args.named_args = NewDict() for named_arg in n1: name = lexer.TokenVal(named_arg.name) - cmd_val.named_args[name] = value.Expr(named_arg.value) + proc_args.named_args[name] = value.Expr(named_arg.value) # TODO: ...spread is illegal else: # json write (x) with state.ctx_YshExpr(mutable_opts): # What EvalExpr() does - _EvalPosArgs(expr_ev, ty.pos_args, cmd_val.pos_args) + _EvalPosArgs(expr_ev, ty.pos_args, proc_args.pos_args) if ty.named_args is not None: - cmd_val.named_args = _EvalNamedArgs(expr_ev, ty.named_args) + proc_args.named_args = _EvalNamedArgs( + expr_ev, ty.named_args) if ty.block_expr and node.block: e_die("Can't accept both block expression and block literal", @@ -255,22 +256,22 @@ def EvalTypedArgsToProc( # p ( ; ; block) is an expression to be evaluated if ty.block_expr: # fallback location is ( - cmd_val.block_arg = expr_ev.EvalExpr(ty.block_expr, ty.left) + proc_args.block_arg = expr_ev.EvalExpr(ty.block_expr, ty.left) # p { echo hi } is an unevaluated block if node.block: # TODO: conslidate value.Block (holds LiteralBlock) and value.Command - cmd_val.block_arg = value.Block(node.block) + proc_args.block_arg = value.Block(node.block) # Add location info so the cmd_val looks the same for both: # cd /tmp (; ; ^(echo hi)) # cd /tmp { echo hi } - if not cmd_val.typed_args: - cmd_val.typed_args = ArgList.CreateNull() + if not proc_args.typed_args: + proc_args.typed_args = ArgList.CreateNull() # Also add locations for error message: ls { echo invalid } - cmd_val.typed_args.left = node.block.brace_group.left - cmd_val.typed_args.right = node.block.brace_group.right + proc_args.typed_args.left = node.block.brace_group.left + proc_args.typed_args.right = node.block.brace_group.right def _BindWords( @@ -453,6 +454,8 @@ def _BindFuncArgs(func, rd, mem): def BindProcArgs(proc, cmd_val, mem): # type: (value.Proc, cmd_value.Argv, state.Mem) -> None + proc_args = cmd_val.proc_args + UP_sig = proc.sig if UP_sig.tag() != proc_sig_e.Closed: # proc is-closed () return @@ -482,15 +485,16 @@ def BindProcArgs(proc, cmd_val, mem): ### Handle typed positional args. This includes a block arg, if any. - if cmd_val.typed_args: # blame ( of call site - blame_loc = cmd_val.typed_args.left + if proc_args and proc_args.typed_args: # blame ( of call site + blame_loc = proc_args.typed_args.left + pos_args = proc_args.pos_args if proc_args else None if sig.positional: # or sig.block_param: _BindTyped(proc.name, sig.positional, proc.defaults.for_typed, - cmd_val.pos_args, mem, blame_loc) + pos_args, mem, blame_loc) else: - if cmd_val.pos_args is not None: - num_pos = len(cmd_val.pos_args) + if pos_args is not None: + num_pos = len(pos_args) if num_pos != 0: raise error.Expr( "Proc %r takes no typed args, but got %d" % @@ -498,17 +502,18 @@ def BindProcArgs(proc, cmd_val, mem): ### Handle typed named args - if cmd_val.typed_args: # blame ; of call site if possible - semi = cmd_val.typed_args.semi_tok + if proc_args and proc_args.typed_args: # blame ; of call site if possible + semi = proc_args.typed_args.semi_tok if semi is not None: blame_loc = semi + named_args = proc_args.named_args if proc_args else None if sig.named: - _BindNamed(proc.name, sig.named, proc.defaults.for_named, - cmd_val.named_args, mem, blame_loc) + _BindNamed(proc.name, sig.named, proc.defaults.for_named, named_args, + mem, blame_loc) else: - if cmd_val.named_args is not None: - num_named = len(cmd_val.named_args) + if named_args is not None: + num_named = len(named_args) if num_named != 0: raise error.Expr( "Proc %r takes no named args, but got %d" % @@ -516,15 +521,15 @@ def BindProcArgs(proc, cmd_val, mem): # Maybe blame second ; of call site. Because value_t doesn't generally # have location info, as opposed to expr_t. - if cmd_val.typed_args: - semi = cmd_val.typed_args.semi_tok2 + if proc_args and proc_args.typed_args: + semi = proc_args.typed_args.semi_tok2 if semi is not None: blame_loc = semi ### Handle block arg block_param = sig.block_param - block_arg = cmd_val.block_arg + block_arg = proc_args.block_arg if proc_args else None if block_param: if block_arg is None: From 22bd27d74653fab9b177b99a5cd92b07974a63e6 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 6 Aug 2024 23:49:38 -0400 Subject: [PATCH 116/506] [core] Optimize forks for command.Subshell, similar to command.Simple test/syscall: add more cases of non-optimal shell snippets. There are still a few cases where we can go further - but only in the $sh -c $code_str case, not when it's wrapped by a function. When wrapped by a function, we're already optimal. --- frontend/syntax.asdl | 2 +- osh/cmd_eval.py | 62 +++++++++++++------------------------------- osh/cmd_parse.py | 2 +- test/syscall.sh | 11 +++++++- 4 files changed, 30 insertions(+), 47 deletions(-) diff --git a/frontend/syntax.asdl b/frontend/syntax.asdl index 01bdf7d1cc..1784c74664 100644 --- a/frontend/syntax.asdl +++ b/frontend/syntax.asdl @@ -396,7 +396,7 @@ module syntax # A brace group is a compound command, with redirects. | BraceGroup %BraceGroup # Contains a single child, like CommandSub - | Subshell(Token left, command child, Token right) + | Subshell(Token left, command child, Token right, bool is_last_cmd) | DParen(Token left, arith_expr child, Token right) | DBracket(Token left, bool_expr expr, Token right) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 3963ae2792..08f9d01574 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1570,7 +1570,13 @@ def _Dispatch(self, node, cmd_st): # This is a leaf from the parent process POV cmd_st.check_errexit = True - status = self.shell_ex.RunSubshell(node.child) + + if node.is_last_cmd: + # If the subshell is the last command in the process, just + # run it in this process. See _MarkIsLastCmd(). + status = self._Execute(node.child) + else: + status = self.shell_ex.RunSubshell(node.child) elif case(command_e.DBracket): # LEAF command node = cast(command.DBracket, UP_node) @@ -1863,7 +1869,7 @@ def LastStatus(self): """For main_loop.py to determine the exit code of the shell itself.""" return self.mem.LastStatus() - def _NoForkLast(self, node): + def _MarkIsLastCmd(self, node): # type: (command_t) -> None if 0: @@ -1879,53 +1885,30 @@ def _NoForkLast(self, node): if 0: log('Simple optimized') + elif case(command_e.Subshell): + node = cast(command.Subshell, UP_node) + node.is_last_cmd = True + elif case(command_e.Pipeline): node = cast(command.Pipeline, UP_node) # Bug fix: if we change the status, we can't exec the last # element! if node.negated is None and not self.exec_opts.pipefail(): - self._NoForkLast(node.children[-1]) + self._MarkIsLastCmd(node.children[-1]) elif case(command_e.Sentence): node = cast(command.Sentence, UP_node) - self._NoForkLast(node.child) + self._MarkIsLastCmd(node.child) elif case(command_e.CommandList): # Subshells often have a CommandList child node = cast(command.CommandList, UP_node) - self._NoForkLast(node.children[-1]) + self._MarkIsLastCmd(node.children[-1]) elif case(command_e.BraceGroup): # TODO: What about redirects? node = cast(BraceGroup, UP_node) - self._NoForkLast(node.children[-1]) - - def _NoForkSentence(self, node): - # type: (command_t) -> None - - if 0: - log('optimizing') - node.PrettyPrint(sys.stderr) - log('') - - UP_node = node - with tagswitch(node) as case: - if case(command_e.Simple): - node = cast(command.Simple, UP_node) - node.is_last_cmd = False - if 0: - log('Simple optimized') - - #elif case(command_e.Pipeline): - # node = cast(command.Pipeline, UP_node) - # if node.negated is None: - # #log ('pipe') - # self._NoForkLast(node.children[-1]) - - elif case(command_e.Sentence): - node = cast(command.Sentence, UP_node) - if node.terminator.id == Id.Op_Amp: - self._NoForkSentence(node.child) + self._MarkIsLastCmd(node.children[-1]) def _RemoveSubshells(self, node): # type: (command_t) -> command_t @@ -1963,17 +1946,8 @@ def ExecuteAndCatch(self, node, cmd_flags=0): """ if cmd_flags & Optimize: node = self._RemoveSubshells(node) - #if self.exec_opts.no_fork_last(): - - # Bug: analysis happens too early: - # - # sh -c 'trap "echo trap" EXIT; date' - #if not self.trap_state.ThisProcessHasTraps(): - - self._NoForkLast(node) # turn the last ones into exec - - # TODO: this makes a difference in job control test - #self._NoForkSentence(node) + # mark the last command in process, so we may avoid forks + self._MarkIsLastCmd(node) if 0: log('after opt:') diff --git a/osh/cmd_parse.py b/osh/cmd_parse.py index 463049650d..fb17902bc0 100644 --- a/osh/cmd_parse.py +++ b/osh/cmd_parse.py @@ -2304,7 +2304,7 @@ def ParseSubshell(self): ate = self._Eat(Id.Right_Subshell) right = word_.AsOperatorToken(ate) - return command.Subshell(left, child, right) + return command.Subshell(left, child, right, False) def ParseDBracket(self): # type: () -> command.DBracket diff --git a/test/syscall.sh b/test/syscall.sh index 05f0a7df90..7930eb4581 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -147,6 +147,14 @@ date; { date; } echo hi; (date) +echo hi; (date;) + +echo hi; (echo hi;) + +echo hi; (echo hi; date) + +( echo hi ); echo hi + # Sentence in Oil (date;) > /tmp/out.txt @@ -207,9 +215,10 @@ date | read x # osh does 4 when others do 3. So every shell optimizes this extra pipeline. ( echo a; echo b ) | wc -l -# osh does 5 when others do 3. ( echo a; echo b ) | ( wc -l ) +{ echo prefix; ( echo a; echo b ); } | ( wc -l ) + echo hi & wait date & wait From 3d54e67958b0bcdc532197e4eef0f4a63e6a1000 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 7 Aug 2024 00:24:46 -0400 Subject: [PATCH 117/506] [core] Remove even more fork() from subshells [test/syscall] Do comparison against bash 5. We are beating other shells even more now! --- osh/cmd_eval.py | 5 +++++ test/syscall.py | 6 +++--- test/syscall.sh | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 08f9d01574..a49af9d9c0 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1887,8 +1887,13 @@ def _MarkIsLastCmd(self, node): elif case(command_e.Subshell): node = cast(command.Subshell, UP_node) + # Mark ourselves as the last node.is_last_cmd = True + # Also mark 'date' as the last one + # echo 1; (echo 2; date) + self._MarkIsLastCmd(node.child) + elif case(command_e.Pipeline): node = cast(command.Pipeline, UP_node) # Bug fix: if we change the status, we can't exec the last diff --git a/test/syscall.py b/test/syscall.py index 23ded902ae..7e6310e7ea 100755 --- a/test/syscall.py +++ b/test/syscall.py @@ -95,7 +95,7 @@ def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs, else: f.write('\t') - bash_count = num_procs[case_id, 'bash-4.4'] + bash_count = num_procs[case_id, 'bash-5.2.21'] if osh_count > bash_count: more_than_bash += 1 if osh_count < bash_count: @@ -110,8 +110,8 @@ def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs, f.write('\n\n') f.write("Cases where ...\n") f.write(" OSH isn't the minimum: %d\n" % not_minimum) - f.write(" OSH starts more than bash: %d\n" % more_than_bash) - f.write(" OSH starts fewer than bash: %d\n\n" % fewer_than_bash) + f.write(" OSH starts more than bash 5: %d\n" % more_than_bash) + f.write(" OSH starts fewer than bash 5: %d\n\n" % fewer_than_bash) return not_minimum, more_than_bash, fewer_than_bash diff --git a/test/syscall.sh b/test/syscall.sh index 7930eb4581..9b78453a7b 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -17,11 +17,11 @@ YSH=${YSH:-ysh} #readonly -a SHELLS=(dash bash-4.4 bash $OSH) # Compare bash 4 vs. bash 5 -readonly -a SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash $OSH) +SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash $OSH) # yash does something fundamentally different in by-code.wrapped - it # understands functions -# SHELLS+=(yash) +#SHELLS+=(yash) readonly BASE_DIR='_tmp/syscall' # What we'll publish readonly RAW_DIR='_tmp/syscall-raw' # Raw data From 4f31306a572c50f1ad124a668b9bb8a52186cf09 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 7 Aug 2024 00:54:56 -0400 Subject: [PATCH 118/506] [cmd_eval refactor] Separate flags for 2 kinds of optimization YSH has shopt --set verbose_errexit, so I think marking the last command should be disabled in that case. --- core/main_loop.py | 7 ++++--- core/process.py | 4 +++- osh/cmd_eval.py | 33 ++++++++++++++++++--------------- osh/prompt.py | 2 +- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/core/main_loop.py b/core/main_loop.py index 3fb290c666..a03630527e 100644 --- a/core/main_loop.py +++ b/core/main_loop.py @@ -268,7 +268,7 @@ def Interactive( break try: - is_return, _ = cmd_ev.ExecuteAndCatch(node) + is_return, _ = cmd_ev.ExecuteAndCatch(node, 0) except KeyboardInterrupt: # issue 467, Ctrl-C during $(sleep 1) is_return = False display.EraseLines() @@ -354,13 +354,14 @@ def Batch(cmd_ev, c_parser, errfmt, cmd_flags=0): # Only optimize if we're on the last line like -c "echo hi" etc. if (cmd_flags & cmd_eval.IsMainProgram and c_parser.line_reader.LastLineHint()): - cmd_flags |= cmd_eval.Optimize + cmd_flags |= cmd_eval.OptimizeSubshells + cmd_flags |= cmd_eval.MarkLastCommands probe('main_loop', 'Batch_parse_exit') probe('main_loop', 'Batch_execute_enter') # can't optimize this because we haven't seen the end yet - is_return, is_fatal = cmd_ev.ExecuteAndCatch(node, cmd_flags=cmd_flags) + is_return, is_fatal = cmd_ev.ExecuteAndCatch(node, cmd_flags) status = cmd_ev.LastStatus() # e.g. 'return' in middle of script, or divide by zero if is_return or is_fatal: diff --git a/core/process.py b/core/process.py index 0c863d6fe3..2bcdced065 100644 --- a/core/process.py +++ b/core/process.py @@ -843,7 +843,9 @@ def Run(self): self.cmd_ev.mutable_opts.DisableErrExit() try: # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc. - self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize) + self.cmd_ev.ExecuteAndCatch( + self.node, + cmd_eval.OptimizeSubshells | cmd_eval.MarkLastCommands) status = self.cmd_ev.LastStatus() # NOTE: We ignore the is_fatal return value. The user should set -o # errexit so failures in subprocesses cause failures in the parent. diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index a49af9d9c0..e12fdae927 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -111,9 +111,10 @@ # ExecuteAndCatch, along with SetValue() flags. IsMainProgram = 1 << 0 # the main shell program, not eval/source/subshell RaiseControlFlow = 1 << 1 # eval/source builtins -Optimize = 1 << 2 -NoDebugTrap = 1 << 3 -NoErrTrap = 1 << 4 +OptimizeSubshells = 1 << 2 +MarkLastCommands = 1 << 3 +NoDebugTrap = 1 << 4 +NoErrTrap = 1 << 5 def MakeBuiltinArgv(argv1): @@ -1573,7 +1574,7 @@ def _Dispatch(self, node, cmd_st): if node.is_last_cmd: # If the subshell is the last command in the process, just - # run it in this process. See _MarkIsLastCmd(). + # run it in this process. See _MarkLastCommands(). status = self._Execute(node.child) else: status = self.shell_ex.RunSubshell(node.child) @@ -1869,7 +1870,7 @@ def LastStatus(self): """For main_loop.py to determine the exit code of the shell itself.""" return self.mem.LastStatus() - def _MarkIsLastCmd(self, node): + def _MarkLastCommands(self, node): # type: (command_t) -> None if 0: @@ -1892,28 +1893,28 @@ def _MarkIsLastCmd(self, node): # Also mark 'date' as the last one # echo 1; (echo 2; date) - self._MarkIsLastCmd(node.child) + self._MarkLastCommands(node.child) elif case(command_e.Pipeline): node = cast(command.Pipeline, UP_node) # Bug fix: if we change the status, we can't exec the last # element! if node.negated is None and not self.exec_opts.pipefail(): - self._MarkIsLastCmd(node.children[-1]) + self._MarkLastCommands(node.children[-1]) elif case(command_e.Sentence): node = cast(command.Sentence, UP_node) - self._MarkIsLastCmd(node.child) + self._MarkLastCommands(node.child) elif case(command_e.CommandList): # Subshells often have a CommandList child node = cast(command.CommandList, UP_node) - self._MarkIsLastCmd(node.children[-1]) + self._MarkLastCommands(node.children[-1]) elif case(command_e.BraceGroup): # TODO: What about redirects? node = cast(BraceGroup, UP_node) - self._MarkIsLastCmd(node.children[-1]) + self._MarkLastCommands(node.children[-1]) def _RemoveSubshells(self, node): # type: (command_t) -> command_t @@ -1930,7 +1931,7 @@ def _RemoveSubshells(self, node): return self._RemoveSubshells(node.child) return node - def ExecuteAndCatch(self, node, cmd_flags=0): + def ExecuteAndCatch(self, node, cmd_flags): # type: (command_t, int) -> Tuple[bool, bool] """Execute a subprogram, handling vm.IntControlFlow and fatal exceptions. @@ -1949,10 +1950,12 @@ def ExecuteAndCatch(self, node, cmd_flags=0): Note: To do what optimize does, dash has EV_EXIT flag and yash has a finally_exit boolean. We use a different algorithm. """ - if cmd_flags & Optimize: + if cmd_flags & OptimizeSubshells: node = self._RemoveSubshells(node) - # mark the last command in process, so we may avoid forks - self._MarkIsLastCmd(node) + + if cmd_flags & MarkLastCommands: + # Mark the last command in each process, so we may avoid forks + self._MarkLastCommands(node) if 0: log('after opt:') @@ -2079,7 +2082,7 @@ def MaybeRunExitTrap(self, mut_status): # RunPendingTraps() in the MAIN LOOP with dev.ctx_Tracer(self.tracer, 'trap EXIT', None): try: - is_return, is_fatal = self.ExecuteAndCatch(node) + is_return, is_fatal = self.ExecuteAndCatch(node, 0) except util.UserExit as e: # explicit exit mut_status.i = e.status return diff --git a/osh/prompt.py b/osh/prompt.py index 25cce8b126..27e6d29d2e 100644 --- a/osh/prompt.py +++ b/osh/prompt.py @@ -370,4 +370,4 @@ def Run(self): # Save this so PROMPT_COMMAND can't set $? with state.ctx_Registers(self.mem): # Catches fatal execution error - self.cmd_ev.ExecuteAndCatch(node) + self.cmd_ev.ExecuteAndCatch(node, 0) From 2bcfb1a7752b93babbf76b6f9235e14f185683aa Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 7 Aug 2024 01:32:14 -0400 Subject: [PATCH 119/506] [test/syscall] Improve report - Move summary to the top - Compare against yash, where relevant - yash never beats us anymore! - There is one case where zsh/ash/dash beats us - redirects - can probably fix this --- test/syscall.py | 45 +++++++++++++++++--------- test/syscall.sh | 86 ++++++++++++++++++++++++++----------------------- 2 files changed, 75 insertions(+), 56 deletions(-) diff --git a/test/syscall.py b/test/syscall.py index 7e6310e7ea..17f7c4ccd0 100755 --- a/test/syscall.py +++ b/test/syscall.py @@ -72,28 +72,21 @@ def WriteHeader(f, shells, col=''): def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs, procs_by_shell): - f.write('Number of Processes Started, by shell and test case\n\n') - - WriteHeader(f, proc_sh, col='osh>min') - not_minimum = 0 more_than_bash = 0 fewer_than_bash = 0 + minimum = {} # case -> number of procses for case_id in sorted(cases): - f.write(case_id + "\t") min_procs = 20 for sh in proc_sh: n = num_procs[case_id, sh] - f.write(Cell(n) + "\t") min_procs = min(n, min_procs) + minimum[case_id] = min_procs osh_count = num_procs[case_id, 'osh'] if osh_count != min_procs: - f.write('%d>%d\t' % (osh_count, min_procs)) not_minimum += 1 - else: - f.write('\t') bash_count = num_procs[case_id, 'bash-5.2.21'] if osh_count > bash_count: @@ -101,17 +94,37 @@ def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs, if osh_count < bash_count: fewer_than_bash += 1 - f.write(code_strs[case_id]) - f.write("\n") - - f.write("TOTAL\t") - for sh in proc_sh: - f.write('%6d\t' % procs_by_shell[sh]) - f.write('\n\n') + f.write('Number of Processes Started, by shell and test case\n') + f.write('\n') f.write("Cases where ...\n") f.write(" OSH isn't the minimum: %d\n" % not_minimum) f.write(" OSH starts more than bash 5: %d\n" % more_than_bash) f.write(" OSH starts fewer than bash 5: %d\n\n" % fewer_than_bash) + f.write('\n') + WriteHeader(f, proc_sh, col='osh>min') + f.write('\n') + + f.write("TOTAL\t") + for sh in proc_sh: + f.write('%6d\t' % procs_by_shell[sh]) + f.write('\n') + f.write('\n') + + for case_id in sorted(cases): + f.write(case_id + "\t") + for sh in proc_sh: + n = num_procs[case_id, sh] + f.write(Cell(n) + "\t") + + osh_count = num_procs[case_id, 'osh'] + min_procs = minimum[case_id] + if osh_count != min_procs: + f.write('%d>%d\t' % (osh_count, min_procs)) + else: + f.write('\t') + + f.write(code_strs[case_id]) + f.write("\n") return not_minimum, more_than_bash, fewer_than_bash diff --git a/test/syscall.sh b/test/syscall.sh index 9b78453a7b..781274e8fd 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -19,6 +19,8 @@ YSH=${YSH:-ysh} # Compare bash 4 vs. bash 5 SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash $OSH) +SHELLS_MORE=( ${SHELLS[@]} yash ) + # yash does something fundamentally different in by-code.wrapped - it # understands functions #SHELLS+=(yash) @@ -63,11 +65,15 @@ run-case() { local code_str=$2 local func_wrap=${3:-} + local -a shells if test -n "$func_wrap"; then code_str="wrapper() { $code_str; }; wrapper" + shells=( "${SHELLS[@]}" ) + else + shells=( "${SHELLS_MORE[@]}" ) fi - for sh in "${SHELLS[@]}"; do + for sh in "${shells[@]}"; do local out_prefix=$RAW_DIR/${sh}__${num} echo "--- $sh" count-procs $out_prefix $sh -c "$code_str" @@ -82,7 +88,7 @@ run-case-file() { echo -n "$code_str" > _tmp/$num.sh - for sh in "${SHELLS[@]}"; do + for sh in "${SHELLS_MORE[@]}"; do local out_prefix=$RAW_DIR/${sh}__${num} echo "--- $sh" count-procs $out_prefix $sh _tmp/$num.sh @@ -95,7 +101,7 @@ run-case-stdin() { local num=$1 local code_str=$2 - for sh in "${SHELLS[@]}"; do + for sh in "${SHELLS_MORE[@]}"; do local out_prefix=$RAW_DIR/${sh}__${num} echo "--- $sh" echo -n "$code_str" | count-procs $out_prefix $sh @@ -276,51 +282,51 @@ by-input() { newline2=$'date\n\ndate\n#comment' # zsh is the only shell to optimize all 6 cases! 2 processes instead of 3. - run-case 30 "$zero" - run-case 31 "$one" - run-case 32 "$two" - run-case 33 "$comment" - run-case 34 "$newline" - run-case 35 "$newline2" - - run-case-file 40 "$zero" - run-case-file 41 "$one" - run-case-file 42 "$two" - run-case-file 43 "$comment" - run-case-file 44 "$newline2" - run-case-file 45 "$newline2" + run-case 50 "$zero" + run-case 51 "$one" + run-case 52 "$two" + run-case 53 "$comment" + run-case 54 "$newline" + run-case 55 "$newline2" + + run-case-file 60 "$zero" + run-case-file 61 "$one" + run-case-file 62 "$two" + run-case-file 63 "$comment" + run-case-file 64 "$newline2" + run-case-file 65 "$newline2" # yash is the only shell to optimize the stdin case at all! # it looks for a lack of trailing newline. - run-case-stdin 50 "$zero" - run-case-stdin 51 "$one" - run-case-stdin 52 "$two" - run-case-stdin 53 "$comment" - run-case-stdin 54 "$newline2" - run-case-stdin 55 "$newline2" + run-case-stdin 70 "$zero" + run-case-stdin 71 "$one" + run-case-stdin 72 "$two" + run-case-stdin 73 "$comment" + run-case-stdin 74 "$newline2" + run-case-stdin 75 "$newline2" # This is identical for all shells #run-case 32 $'date; date\n#comment\n' cat >$BASE_DIR/cases.${suite}.txt < Date: Wed, 7 Aug 2024 01:48:57 -0400 Subject: [PATCH 120/506] [cmd_eval] Optimize last processes in command.Redirect node The benchmark cases added showed up as suboptimal on the processes.by-code.wrapped.txt report. We care about that one more than processes.by-code.txt. --- osh/cmd_eval.py | 10 ++++++++++ test/syscall.sh | 13 +++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index e12fdae927..18b5cc709b 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1906,6 +1906,16 @@ def _MarkLastCommands(self, node): node = cast(command.Sentence, UP_node) self._MarkLastCommands(node.child) + elif case(command_e.Redirect): + node = cast(command.Sentence, UP_node) + # Don't need to restore the redirect in any of these cases: + + # bin/osh -c 'echo hi 2>stderr' + # bin/osh -c '{ echo hi; date; } 2>stderr' + # echo hi 2>stderr | wc -l + + self._MarkLastCommands(node.child) + elif case(command_e.CommandList): # Subshells often have a CommandList child node = cast(command.CommandList, UP_node) diff --git a/test/syscall.sh b/test/syscall.sh index 781274e8fd..aa6fa17eae 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -161,8 +161,17 @@ echo hi; (echo hi; date) ( echo hi ); echo hi -# Sentence in Oil -(date;) > /tmp/out.txt +date > /tmp/redir.txt + +(date;) > /tmp/sentence.txt + +date 2> /tmp/stderr.txt | wc -l + +echo hi > /tmp/redir.txt + +(echo hi;) > /tmp/sentence.txt + +echo hi 2> /tmp/stderr.txt | wc -l (date; echo hi) From 5d34cddb237c633d1d2f8b83ce20eebecd955c0a Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 7 Aug 2024 02:52:19 -0400 Subject: [PATCH 121/506] [ysh] Turn off top-level fork() optimizations when shopt --set verbose_errexit. So we can see this fail: bin/ysh -c '/bin/false' After running test/syscall, we can see it mainly affects: processes.by-code.txt And not: processes.by-code-wrapped.txt --- core/main_loop.py | 3 ++- spec/ysh-usage.test.sh | 21 +++++++++++++++++++++ test/syscall.py | 30 +++++++++++++++++++++--------- test/syscall.sh | 2 +- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/core/main_loop.py b/core/main_loop.py index a03630527e..60fdbe1bd7 100644 --- a/core/main_loop.py +++ b/core/main_loop.py @@ -355,7 +355,8 @@ def Batch(cmd_ev, c_parser, errfmt, cmd_flags=0): if (cmd_flags & cmd_eval.IsMainProgram and c_parser.line_reader.LastLineHint()): cmd_flags |= cmd_eval.OptimizeSubshells - cmd_flags |= cmd_eval.MarkLastCommands + if not cmd_ev.exec_opts.verbose_errexit(): + cmd_flags |= cmd_eval.MarkLastCommands probe('main_loop', 'Batch_parse_exit') diff --git a/spec/ysh-usage.test.sh b/spec/ysh-usage.test.sh index 920bc0936d..9b3bd5aee6 100644 --- a/spec/ysh-usage.test.sh +++ b/spec/ysh-usage.test.sh @@ -45,3 +45,24 @@ no-quoting:1 "with spaces.sh":1 b'bad \yff':1 ## END + + +#### shopt --set verbose_errexit + +try { + $SH -c '/bin/false' 2>on.txt +} + +try { + $SH +o verbose_errexit -c '/bin/false' 2>off.txt +} + +wc -l on.txt off.txt +#echo +#cat on.txt off.txt + +## STDOUT: + 3 on.txt + 0 off.txt + 3 total +## END diff --git a/test/syscall.py b/test/syscall.py index 17f7c4ccd0..94a074d65b 100755 --- a/test/syscall.py +++ b/test/syscall.py @@ -56,8 +56,8 @@ def Cell(i): assert WC_LINE.match(' 68 bash-4.4__01.19610') -def WriteHeader(f, shells, col=''): - f.write("ID\t") +def WriteHeader(f, shells, more_cols=None): + f.write("ID\t",) for sh in shells: # abbreviate if sh.startswith('bash-4'): @@ -65,7 +65,9 @@ def WriteHeader(f, shells, col=''): elif sh.startswith('bash-5'): sh = 'bash-5' f.write("%6s\t" % sh) - f.write('%s\t' % col) + if more_cols: + for col in more_cols: + f.write('%s\t' % col) f.write('Description') f.write("\n") @@ -75,6 +77,7 @@ def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs, not_minimum = 0 more_than_bash = 0 fewer_than_bash = 0 + osh_not_equal_ysh = 0 minimum = {} # case -> number of procses for case_id in sorted(cases): @@ -85,9 +88,14 @@ def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs, minimum[case_id] = min_procs osh_count = num_procs[case_id, 'osh'] + ysh_count = num_procs[case_id, 'ysh'] + if osh_count != min_procs: not_minimum += 1 + if ysh_count != osh_count: + osh_not_equal_ysh += 1 + bash_count = num_procs[case_id, 'bash-5.2.21'] if osh_count > bash_count: more_than_bash += 1 @@ -100,8 +108,9 @@ def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs, f.write(" OSH isn't the minimum: %d\n" % not_minimum) f.write(" OSH starts more than bash 5: %d\n" % more_than_bash) f.write(" OSH starts fewer than bash 5: %d\n\n" % fewer_than_bash) + f.write(" YSH not equal to OSH: %d\n\n" % osh_not_equal_ysh) f.write('\n') - WriteHeader(f, proc_sh, col='osh>min') + WriteHeader(f, proc_sh, more_cols=['osh>min', 'ysh!osh']) f.write('\n') f.write("TOTAL\t") @@ -117,11 +126,16 @@ def WriteProcessReport(f, cases, code_strs, proc_sh, num_procs, f.write(Cell(n) + "\t") osh_count = num_procs[case_id, 'osh'] + ysh_count = num_procs[case_id, 'ysh'] min_procs = minimum[case_id] + if osh_count != min_procs: - f.write('%d>%d\t' % (osh_count, min_procs)) - else: - f.write('\t') + f.write('%d>%d' % (osh_count, min_procs)) + f.write('\t') + + if ysh_count != osh_count: + f.write('%d!=%d' % (ysh_count, osh_count)) + f.write('\t') f.write(code_strs[case_id]) f.write("\n") @@ -143,8 +157,6 @@ def WriteSyscallReport(f, cases, code_strs, syscall_sh, num_syscalls, f.write('%6d\t' % n) #min_procs = min(n, min_procs) - f.write('\t') - f.write(code_strs[case_id]) f.write("\n") diff --git a/test/syscall.sh b/test/syscall.sh index aa6fa17eae..e2122df91b 100755 --- a/test/syscall.sh +++ b/test/syscall.sh @@ -17,7 +17,7 @@ YSH=${YSH:-ysh} #readonly -a SHELLS=(dash bash-4.4 bash $OSH) # Compare bash 4 vs. bash 5 -SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash $OSH) +SHELLS=(dash bash-4.4 bash-5.2.21 mksh zsh ash $OSH $YSH) SHELLS_MORE=( ${SHELLS[@]} yash ) From 131be43903b2e3c7b0c5f98e46072ae1702883ef Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 7 Aug 2024 11:02:46 -0400 Subject: [PATCH 122/506] [builtin/read] read --raw-line returns 0 for line without newline And then the NEXT iteration returns 1 for EOF. This makes a 'while' loop work correctly. Aidan noticed this problem on Zulip. --- builtin/io_osh.py | 3 +-- builtin/read_osh.py | 14 +++++++------ osh/cmd_eval.py | 5 +++++ spec/ysh-builtins.test.sh | 43 +++++++++++++++++++++++++++++++++++++-- test/bugs.sh | 40 ++++++++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 10 deletions(-) diff --git a/builtin/io_osh.py b/builtin/io_osh.py index 514d1039ef..87058ab7fe 100644 --- a/builtin/io_osh.py +++ b/builtin/io_osh.py @@ -131,8 +131,7 @@ def Run(self, cmd_val): while True: # bash uses this slow algorithm; YSH could provide read --all-lines try: - line, _ = read_osh.ReadLineSlowly(self.cmd_ev, - with_eol=not arg.t) + line = read_osh.ReadLineSlowly(self.cmd_ev, with_eol=not arg.t) except pyos.ReadError as e: self.errfmt.PrintMessage("mapfile: read() error: %s" % posix.strerror(e.err_num)) diff --git a/builtin/read_osh.py b/builtin/read_osh.py index c70e1c27ed..40bfccf4af 100644 --- a/builtin/read_osh.py +++ b/builtin/read_osh.py @@ -186,14 +186,15 @@ def _ReadPortion(delim_byte, max_chars, cmd_ev): def ReadLineSlowly(cmd_ev, with_eol=True): - # type: (CommandEvaluator, bool) -> Tuple[str, bool] + # type: (CommandEvaluator, bool) -> str """Read a line from stdin, unbuffered + Used by mapfile and read --raw-line. + sys.stdin.readline() in Python has its own buffering which is incompatible with shell semantics. dash, mksh, and zsh all read a single byte at a time with read(0, 1). """ - eof = False ch_array = [] # type: List[int] while True: ch, err_num = pyos.ReadByte(0) @@ -206,7 +207,6 @@ def ReadLineSlowly(cmd_ev, with_eol=True): raise pyos.ReadError(err_num) elif ch == pyos.EOF_SENTINEL: - eof = True break else: @@ -217,7 +217,7 @@ def ReadLineSlowly(cmd_ev, with_eol=True): ch_array.pop() break - return pyutil.ChArrayToString(ch_array), eof + return pyutil.ChArrayToString(ch_array) def ReadAll(): @@ -374,8 +374,10 @@ def _ReadYsh(self, arg, arg_r, cmd_val): status = 0 elif arg.raw_line: # read --raw-line is unbuffered - contents, eof = ReadLineSlowly(self.cmd_ev, with_eol=arg.with_eol) - status = 1 if eof else 0 + contents = ReadLineSlowly(self.cmd_ev, with_eol=arg.with_eol) + #log('EOF %s', eof) + #status = 1 if eof else 0 + status = 0 if len(contents) else 1 elif arg.all: # read --all contents = ReadAll() diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 18b5cc709b..d18ac46342 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -2086,6 +2086,9 @@ def MaybeRunExitTrap(self, mut_status): Could use i & (n-1) == i & 255 because we have a power of 2. https://stackoverflow.com/questions/14997165/fastest-way-to-get-a-positive-modulo-in-c-c """ + # TODO: This calls _Execute(), but we may need ExecuteAndCatch() + #self.RunPendingTraps() + node = self.trap_state.GetHook('EXIT') # type: command_t if node: # NOTE: Don't set option_i._running_trap, because that's for @@ -2154,6 +2157,8 @@ def _MaybeRunErrTrap(self): # RunPendingTraps() in the MAIN LOOP with dev.ctx_Tracer(self.tracer, 'trap ERR', None): + # In bash, the PIPESTATUS register leaks. See spec/builtin-trap-err. + # So unlike other traps, we don't isolate registers. #with state.ctx_Registers(self.mem): # prevent setting $? etc. with state.ctx_ErrTrap(self.mem): self._Execute(node) diff --git a/spec/ysh-builtins.test.sh b/spec/ysh-builtins.test.sh index e3da89580c..5877c04658 100644 --- a/spec/ysh-builtins.test.sh +++ b/spec/ysh-builtins.test.sh @@ -172,7 +172,46 @@ len=2 pass ## END -#### Mixing read --line with read -r +#### read --raw-line handles line without end, --with-eol + +write --end '' $'a\nb\n' | while read --raw-line; do + pp test_ (_reply) +done + +echo + +write --end '' $'a\nb' | while read --raw-line; do + pp test_ (_reply) +done + +echo + +write --end '' $'a\nb\n' | while read --raw-line --with-eol; do + pp test_ (_reply) +done + +echo + +write --end '' $'a\nb' | while read --raw-line --with-eol; do + pp test_ (_reply) +done + + +## STDOUT: +(Str) "a" +(Str) "b" + +(Str) "a" +(Str) "b" + +(Str) "a\n" +(Str) "b\n" + +(Str) "a\n" +(Str) "b" +## END + +#### Mixing read --raw-line with read -r $SH $REPO_ROOT/spec/testdata/ysh-read-0.sh @@ -192,7 +231,7 @@ _reply=3 REPLY=4 ## END -#### read --line --with-eol +#### read --raw-line --with-eol $SH $REPO_ROOT/spec/testdata/ysh-read-1.sh diff --git a/test/bugs.sh b/test/bugs.sh index 9979dfdb98..c4b0a717c0 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -46,10 +46,13 @@ esrch-test() { trap-1() { local sh=${1:-bin/osh} + set +o errexit # This fails to run the trap $sh -x -c 'trap "echo int" INT; sleep 5' + + echo "$sh status=$?" } # Run with bin/ysh -x to show fork opts @@ -59,6 +62,43 @@ trap-2() { # This runs it $sh -x -c 'trap "echo int" INT; sleep 5; echo last' + + echo "$sh status=$?" +} + +trap-with-errexit() { + local sh=${1:-bin/osh} + + # This can't raise + $sh -x -c 'set -e; trap "echo false; false" INT; sleep 5' +} + +two-traps-return() { + local sh=${1:-bin/osh} + + set +o errexit + + $sh -x -c ' +trap "echo int; return 44" INT +trap "echo exit; return 55" EXIT +sleep 5 +' + # bash gives 130? + echo "$sh status=$?" +} + +two-traps-status() { + local sh=${1:-bin/osh} + + set +o errexit + + $sh -x -c ' +trap "echo int; ( exit 44 )" INT +trap "echo exit; ( exit 55 )" EXIT +sleep 5 +' + # bash gives 130? + echo "$sh status=$?" } trap-line() { From 041fdc0cbbeb5b74475e545820a38ca50fddc658 Mon Sep 17 00:00:00 2001 From: Ellen <38250543+ellen364@users.noreply.github.com> Date: Wed, 7 Aug 2024 23:25:04 +0100 Subject: [PATCH 123/506] [builtins] Add Dict => get() method (#2045) Use verbose variable name `default_value` because `default` breaks mycpp. --- builtin/method_dict.py | 17 +++++++++++++++++ core/shell.py | 2 +- doc/ref/chap-type-method.md | 15 +++++++++++++++ spec/ysh-methods.test.sh | 12 ++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/builtin/method_dict.py b/builtin/method_dict.py index 8b7193685d..648c5c18a4 100644 --- a/builtin/method_dict.py +++ b/builtin/method_dict.py @@ -61,3 +61,20 @@ def Call(self, rd): mylib.dict_erase(dictionary, key) return value.Null + + +class Get(vm._Callable): + + def __init__(self): + # type: () -> None + pass + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + + dictionary = rd.PosDict() + key = rd.PosStr() + default_value = rd.PosValue() + rd.Done() + + return dictionary.get(key, default_value) diff --git a/core/shell.py b/core/shell.py index 5f229ef321..bd4c7f70cf 100644 --- a/core/shell.py +++ b/core/shell.py @@ -745,7 +745,7 @@ def Main( 'fullMatch': None, } methods[value_e.Dict] = { - 'get': None, # doesn't raise an error + 'get': method_dict.Get(), 'erase': method_dict.Erase(), 'keys': method_dict.Keys(), 'values': method_dict.Values(), diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 948c58fab5..e9d0416893 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -341,6 +341,21 @@ Similar to `keys()`, but returns the values of the dictionary. ### get() +Return value for given key, falling back to the default value if the key +doesn't exist. Default is required. + + var book = { + title: "Hitchhiker's Guide", + published: 1979, + } + var published = book => get("published", null) + = published + # => (Int 1979) + + var author = book => get("author", "???") + = author + # => (Str "???") + ### erase() Ensures that the given key does not exist in the dictionary. diff --git a/spec/ysh-methods.test.sh b/spec/ysh-methods.test.sh index 2a94c8928b..152bb0b337 100644 --- a/spec/ysh-methods.test.sh +++ b/spec/ysh-methods.test.sh @@ -406,6 +406,18 @@ pp test_ (book) (Dict) {"title":"The Histories"} ## END +#### Dict -> get() +var book = {title: "Hitchhiker's Guide", published: 1979} +pp test_ (book => get("title", "")) +pp test_ (book => get("published", 0)) +pp test_ (book => get("author", "")) +## status: 0 +## STDOUT: +(Str) "Hitchhiker's Guide" +(Int) 1979 +(Str) "" +## END + #### Separation of -> attr and () calling const check = "abc" => startsWith pp test_ (check("a")) From 7fb2a8dbb9d8366bcef96268188abd84305240d9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 7 Aug 2024 16:08:22 -0400 Subject: [PATCH 124/506] [core] Run traps before exiting This was motivated by a 'trap INT' bug that Samuel reported. It now works for SIGUSR1, but there's still some work to do for SIGINT aka Ctrl-C. --- core/shell.py | 6 +++--- osh/cmd_eval.py | 31 +++++++++++++++++++++++++----- spec/builtin-trap.test.sh | 22 +++++++++++---------- spec/testdata/builtin-trap-int.sh | 11 +++++++++++ spec/testdata/builtin-trap-usr1.sh | 12 ++++++++++++ test/bugs.sh | 18 +++++++++++++++-- 6 files changed, 80 insertions(+), 20 deletions(-) create mode 100755 spec/testdata/builtin-trap-int.sh create mode 100755 spec/testdata/builtin-trap-usr1.sh diff --git a/core/shell.py b/core/shell.py index bd4c7f70cf..34ca744e90 100644 --- a/core/shell.py +++ b/core/shell.py @@ -990,7 +990,7 @@ def Main( # Same logic as interactive shell mut_status = IntParamBox(status) - cmd_ev.MaybeRunExitTrap(mut_status) + cmd_ev.RunTrapsOnExit(mut_status) status = mut_status.i return status @@ -1060,7 +1060,7 @@ def Main( status = e.status mut_status = IntParamBox(status) - cmd_ev.MaybeRunExitTrap(mut_status) + cmd_ev.RunTrapsOnExit(mut_status) status = mut_status.i if readline: @@ -1139,7 +1139,7 @@ def Main( except util.UserExit as e: status = e.status mut_status = IntParamBox(status) - cmd_ev.MaybeRunExitTrap(mut_status) + cmd_ev.RunTrapsOnExit(mut_status) multi_trace.WriteDumps() diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index d18ac46342..a7360f6d15 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1748,12 +1748,33 @@ def RunPendingTraps(self): with state.ctx_Option(self.mutable_opts, [option_i._running_trap], True): for trap_node in trap_nodes: - # Isolate the exit status. with state.ctx_Registers(self.mem): - # Trace it. TODO: Show the trap kind too + # TODO: show trap kind in trace with dev.ctx_Tracer(self.tracer, 'trap', None): + # Note: exit status is lost self._Execute(trap_node) + def RunPendingTrapsAndCatch(self): + # type: () -> None + """ + Like the above, but calls ExecuteAndCatch(), which may raise util.UserExit + """ + trap_nodes = self.trap_state.GetPendingTraps() + if trap_nodes is not None: + with state.ctx_Option(self.mutable_opts, [option_i._running_trap], + True): + for trap_node in trap_nodes: + with state.ctx_Registers(self.mem): + # TODO: show trap kind in trace + with dev.ctx_Tracer(self.tracer, 'trap', None): + # Note: exit status is lost + try: + self.ExecuteAndCatch(trap_node, 0) + except util.UserExit: + # If user calls 'exit', stop running traps, but + # we still run the EXIT trap later. + break + def _Execute(self, node): # type: (command_t) -> int """Call _Dispatch(), and performs the errexit check. @@ -2072,7 +2093,7 @@ def EvalCommand(self, block): return status - def MaybeRunExitTrap(self, mut_status): + def RunTrapsOnExit(self, mut_status): # type: (IntParamBox) -> None """If an EXIT trap handler exists, run it. @@ -2086,8 +2107,8 @@ def MaybeRunExitTrap(self, mut_status): Could use i & (n-1) == i & 255 because we have a power of 2. https://stackoverflow.com/questions/14997165/fastest-way-to-get-a-positive-modulo-in-c-c """ - # TODO: This calls _Execute(), but we may need ExecuteAndCatch() - #self.RunPendingTraps() + # This does not raise, even on 'exit', etc. + self.RunPendingTrapsAndCatch() node = self.trap_state.GetHook('EXIT') # type: command_t if node: diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index 18f26f732a..7ead64602a 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -272,25 +272,27 @@ end child wait status 0 ## END +#### trap USR1, sleep, SIGINT: non-interactively + +$REPO_ROOT/spec/testdata/builtin-trap-usr1.sh + +## STDOUT: +usr1 +status=0 +## END + #### trap INT, sleep, SIGINT: non-interactively # mksh behaves differently in CI -- maybe when it's not connected to a # terminal? - case $SH in mksh) echo mksh; exit ;; esac -# Without this, it succeeds in CI? -case $SH in *osh) echo osh; exit ;; esac - -$SH -c 'trap "echo int" INT; sleep 0.1' & -/usr/bin/kill -INT $! -wait - -# Only mksh shows 'int'? -# OSH shows "done" +$REPO_ROOT/spec/testdata/builtin-trap-int.sh ## STDOUT: +status=0 ## END + ## OK mksh STDOUT: mksh ## END diff --git a/spec/testdata/builtin-trap-int.sh b/spec/testdata/builtin-trap-int.sh new file mode 100755 index 0000000000..ccd1a7c573 --- /dev/null +++ b/spec/testdata/builtin-trap-int.sh @@ -0,0 +1,11 @@ + +# Why don't other shells run this trap? It's not a subshell +$SH -c 'trap "echo int" INT; sleep 0.1' & + +sleep 0.05 + +$(which kill) -INT $! + +wait + +echo status=$? diff --git a/spec/testdata/builtin-trap-usr1.sh b/spec/testdata/builtin-trap-usr1.sh new file mode 100755 index 0000000000..070f52653e --- /dev/null +++ b/spec/testdata/builtin-trap-usr1.sh @@ -0,0 +1,12 @@ + +# Why don't other shells run this trap? It's not a subshell +$SH -c 'trap "echo usr1" USR1; sleep 0.1' & +#$SH -c 'trap "echo int" INT; sleep 0.1' & + +sleep 0.05 + +$(which kill) -USR1 $! + +wait + +echo status=$? diff --git a/test/bugs.sh b/test/bugs.sh index c4b0a717c0..63692bfdd2 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -50,7 +50,7 @@ trap-1() { set +o errexit # This fails to run the trap - $sh -x -c 'trap "echo int" INT; sleep 5' + $sh -x -c 'echo pid=$$; trap "echo int" INT; sleep 5' echo "$sh status=$?" } @@ -61,7 +61,7 @@ trap-2() { set +o errexit # This runs it - $sh -x -c 'trap "echo int" INT; sleep 5; echo last' + $sh -x -c 'echo pid=$$; trap "echo int" INT; sleep 5; echo last' echo "$sh status=$?" } @@ -87,6 +87,20 @@ sleep 5 echo "$sh status=$?" } +two-traps-exit() { + local sh=${1:-bin/osh} + + set +o errexit + + $sh -x -c ' +trap "echo int; exit 44" INT +trap "echo exit; exit 55" EXIT +sleep 5 +' + # bash gives 130? + echo "$sh status=$?" +} + two-traps-status() { local sh=${1:-bin/osh} From 59da24326a87529142aaff53c31d3c3ecda3a80b Mon Sep 17 00:00:00 2001 From: Aidan <46799759+PossiblyAShrub@users.noreply.github.com> Date: Wed, 7 Aug 2024 22:14:06 -0600 Subject: [PATCH 125/506] [builtin/eval] Add optional args for var binding (#2044) --- builtin/meta_osh.py | 38 +++++++++++++++++++++----- builtin/method_str.py | 42 ++++------------------------- core/shell.py | 2 +- core/state.py | 51 +++++++++++++++++++++++++++++++---- spec/ysh-builtin-eval.test.sh | 29 +++++++++++--------- spec/ysh-proc.test.sh | 32 ++++++++++++++++++++++ 6 files changed, 132 insertions(+), 62 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index 14619d3fb4..ba31e349a8 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -6,6 +6,7 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus +from _devbuild.gen.value_asdl import value, value_e from _devbuild.gen.syntax_asdl import source, loc from core import alloc from core import dev @@ -31,7 +32,7 @@ _ = log -from typing import Dict, List, Tuple, Optional, TYPE_CHECKING +from typing import Dict, List, Tuple, Optional, cast, TYPE_CHECKING if TYPE_CHECKING: from frontend import args from frontend.parse_lib import ParseContext @@ -50,6 +51,7 @@ def __init__( cmd_ev, # type: CommandEvaluator tracer, # type: dev.Tracer errfmt, # type: ui.ErrorFormatter + mem, # type: state.Mem ): # type: (...) -> None self.parse_ctx = parse_ctx @@ -58,16 +60,38 @@ def __init__( self.cmd_ev = cmd_ev self.tracer = tracer self.errfmt = errfmt + self.mem = mem - def Run(self, cmd_val): + def RunTyped(self, cmd_val): # type: (cmd_value.Argv) -> int - - if cmd_val.proc_args: # eval (mycmd) - rd = typed_args.ReaderForProc(cmd_val) - cmd = rd.PosCommand() - rd.Done() + """For eval (mycmd)""" + rd = typed_args.ReaderForProc(cmd_val) + cmd = rd.PosCommand() + dollar0 = rd.NamedStr("dollar0", None) + pos_args_raw = rd.NamedList("pos_args", None) + vars = rd.NamedDict("vars", None) + rd.Done() + + pos_args = None # type: List[str] + if pos_args_raw is not None: + pos_args = [] + for arg in pos_args_raw: + if arg.tag() != value_e.Str: + raise error.TypeErr( + arg, + "Expected pos_args to be a list of Strs", + rd.LeftParenToken()) + + pos_args.append(cast(value.Str, arg).s) + + with state.ctx_Eval(self.mem, dollar0, pos_args, vars): return self.cmd_ev.EvalCommand(cmd) + def Run(self, cmd_val): + # type: (cmd_value.Argv) -> int + if cmd_val.proc_args: + return self.RunTyped(cmd_val) + # There are no flags, but we need it to respect -- _, arg_r = flag_util.ParseCmdVal('eval', cmd_val) diff --git a/builtin/method_str.py b/builtin/method_str.py index 4e004977cc..07b2e46c6c 100644 --- a/builtin/method_str.py +++ b/builtin/method_str.py @@ -2,10 +2,9 @@ from __future__ import print_function -from _devbuild.gen.syntax_asdl import loc_t, loc -from _devbuild.gen.runtime_asdl import scope_e +from _devbuild.gen.syntax_asdl import loc_t from _devbuild.gen.value_asdl import (value, value_e, value_t, eggex_ops, - eggex_ops_t, RegexMatch, LeftName) + eggex_ops_t, RegexMatch) from builtin import pure_ysh from core import error from core import state @@ -21,7 +20,7 @@ import libc from libc import REG_NOTBOL -from typing import cast, Any, List, Optional, Tuple +from typing import cast, List, Tuple _ = log @@ -321,37 +320,6 @@ def Call(self, rd): return RegexMatch(string, indices, capture) -class ctx_EvalReplace(object): - """For $0, $1, $2, $3, ... replacements in Str => replace()""" - - def __init__(self, mem, arg0, argv): - # type: (state.Mem, str, Optional[List[str]]) -> None - # argv will be None for Str => replace(Str, Expr) - if argv is None: - self.pushed_argv = False - else: - mem.argv_stack.append(state._ArgFrame(argv)) - self.pushed_argv = True - - # $0 needs to have lexical scoping. So we store it with other locals. - # As "0" cannot be parsed as an lvalue, we can safely store arg0 there. - assert mem.GetValue("0", scope_e.LocalOnly).tag() == value_e.Undef - self.lval = LeftName("0", loc.Missing) - mem.SetLocalName(self.lval, value.Str(arg0)) - - self.mem = mem - - def __enter__(self): - # type: () -> None - pass - - def __exit__(self, type, value_, traceback): - # type: (Any, Any, Any) -> None - self.mem.SetLocalName(self.lval, value.Undef) - if self.pushed_argv: - self.mem.argv_stack.pop() - - class Replace(vm._Callable): def __init__(self, mem, expr_ev): @@ -429,7 +397,7 @@ def Call(self, rd): s = subst_str.s if subst_expr: # Eval with $0 set to string_val (the matched substring) - with ctx_EvalReplace(self.mem, string_val.s, None): + with state.ctx_Eval(self.mem, string_val.s, None, None): s = self.EvalSubstExpr(subst_expr, rd.LeftParenToken()) assert s is not None @@ -491,7 +459,7 @@ def Call(self, rd): if subst_str: s = subst_str.s if subst_expr: - with ctx_EvalReplace(self.mem, arg0, argv): + with state.ctx_Eval(self.mem, arg0, argv, None): with pure_ysh.ctx_Shvar(self.mem, named_vars): s = self.EvalSubstExpr(subst_expr, rd.LeftParenToken()) diff --git a/core/shell.py b/core/shell.py index 34ca744e90..dd3b41cc26 100644 --- a/core/shell.py +++ b/core/shell.py @@ -602,7 +602,7 @@ def Main( b[builtin_i.source] = source_builtin b[builtin_i.dot] = source_builtin b[builtin_i.eval] = meta_osh.Eval(parse_ctx, exec_opts, cmd_ev, tracer, - errfmt) + errfmt, mem) # Module builtins guards = {} # type: Dict[str, bool] diff --git a/core/state.py b/core/state.py index 92279a86e7..8845bfbbcb 100644 --- a/core/state.py +++ b/core/state.py @@ -1136,6 +1136,49 @@ def _MakeArgvCell(argv): return Cell(False, False, False, value.List(items)) +class ctx_Eval(object): + """Push temporary variable frame and override $0, $1, $2, etc.""" + + def __init__(self, mem, dollar0, pos_args, vars): + # type: (Mem, Optional[str], Optional[List[str]], Optional[Dict[str, value_t]]) -> None + self.mem = mem + self.dollar0 = dollar0 + self.pos_args = pos_args + self.vars = vars + + # $0 needs to have lexical scoping. So we store it with other locals. + # As "0" cannot be parsed as an lvalue, we can safely store dollar0 there. + if dollar0 is not None: + assert mem.GetValue("0", scope_e.LocalOnly).tag() == value_e.Undef + self.dollar0_lval = LeftName("0", loc.Missing) + mem.SetLocalName(self.dollar0_lval, value.Str(dollar0)) + + if pos_args is not None: + mem.argv_stack.append(_ArgFrame(pos_args)) + + if vars is not None: + frame = {} # type: Dict[str, Cell] + for name in vars: + frame[name] = Cell(False, False, False, vars[name]) + + mem.var_stack.append(frame) + + def __enter__(self): + # type: () -> None + pass + + def __exit__(self, type, value_, traceback): + # type: (Any, Any, Any) -> None + if self.vars is not None: + self.mem.var_stack.pop() + + if self.pos_args is not None: + self.mem.argv_stack.pop() + + if self.dollar0 is not None: + self.mem.SetLocalName(self.dollar0_lval, value.Undef) + + class Mem(object): """For storing variables. @@ -2353,11 +2396,9 @@ def Get(self, name): First, we search for a proc, and then a sh-func. This means that procs can shadow the definition of sh-funcs. """ - vars = self.mem.var_stack[0] - if name in vars: - maybe_proc = vars[name] - if maybe_proc.val.tag() == value_e.Proc: - return cast(value.Proc, maybe_proc.val) + maybe_proc = self.mem.GetValue(name) + if maybe_proc.tag() == value_e.Proc: + return cast(value.Proc, maybe_proc) if name in self.sh_funcs: return self.sh_funcs[name] diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index c020022ac2..a7eff2aec8 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 8 +## oils_failures_allowed: 1 #### Eval does not take a literal block - can restore this later @@ -99,8 +99,8 @@ TODO ## END #### eval with argv bindings -eval (^(echo "$@")) (pos_args=:| foo bar baz |) -eval (^(pp test_ (:| $1 $2 $3 |))) (pos_args=:| foo bar baz |) +eval (^(echo "$@"), pos_args=:| foo bar baz |) +eval (^(pp test_ (:| $1 $2 $3 |)), pos_args=:| foo bar baz |) ## STDOUT: foo bar baz (List) ["foo","bar","baz"] @@ -108,21 +108,21 @@ foo bar baz #### eval lines with argv bindings proc lines (;;; block) { - while read --line { + while read --raw-line { var cols = _reply => split() eval (block, pos_args=cols) } } -printf 'a b\nc d' | lines { echo $1 } +printf 'a b\nc d\n' | lines { echo $1 } ## STDOUT: a c ## END -#### eval with custom arg0 -eval (^(write $0)) (arg0="my arg0") +#### eval with custom dollar0 +eval (^(write $0), dollar0="my arg0") ## STDOUT: my arg0 ## END @@ -136,8 +136,9 @@ eval (^(pp test_ (myVar)), vars={ 'myVar': '123' }) eval (^(pp test_ (myVar))) ## STDOUT: -abc -123 +(Str) "abc" +(Str) "123" +(Str) "abc" ## END #### dynamic binding names and mutation @@ -146,8 +147,8 @@ proc foreach (binding, in_; list ;; block) { error 'Must use the "syntax" `foreach in () { ... }`' } - for _ in (list) { - eval (block, vars={ binding: _ }) + for item in (list) { + eval (block, vars={ [binding]: item }) } } @@ -200,13 +201,17 @@ arg file #### vars initializes the variable frame, but does not remember it var vars = { 'foo': 123 } -eval (^(var bar = 321), vars=vars) +eval (^(var bar = 321;), vars=vars) pp test_ (vars) ## STDOUT: (Dict) {"foo":123} ## END +#### eval pos_args must be strings +eval (^(true), pos_args=[1, 2, 3]) +## status: 3 + #### eval 'mystring' vs. eval (myblock) eval 'echo plain' diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index de1bc0dbfa..3b996e2234 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -524,3 +524,35 @@ grep ## STDOUT: sh-func grep ## END + +#### proc resolution changes with the local scope +shopt -s ysh:upgrade + +proc foo { + echo foo +} + +proc bar { + echo bar +} + +proc inner { + var foo = bar + foo # Will now reference `proc bar` +} + +foo +inner +foo # Back to the global scope, foo still references `proc foo` + +# Without this behavior, features like `eval(b, vars={ flag: __flag })`, needed +# by parseArgs, will not work. `eval` with `vars` adds a new frame to the end of +# `mem.var_stack` with a local `flag` set to `proc __flag`. However, then we +# cannot resolve `flag` by only checking `mem.var_stack[0]` like we could with +# a proc declared normally, so we must search `mem.var_stack` from last to first. + +## STDOUT: +foo +bar +foo +## END From 59ece78259d0eeed24d1f7b51aeec952c365becd Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 00:22:15 -0400 Subject: [PATCH 126/506] [spec/ysh-builtin-eval] Use local vars in vars/pos_args binding cases I was trying to find what I thought was a design issue, but couldn't tickle it. Then I made a diversion into 'pp stacks_' to try print state.Mem in a readable way. That led to some diversions: - shopt --unset copy_env - to make the output less cluttered, - Fixed an ordering bug in printin options: bin/ysh -o pp stacks_ isn't done yet, but could be a good starting point. Right now it uses a similar mechanism as the JSON crash dump, which is supposed to dump the interpreter state in JSON. (This isn't used very much now, but is tested.) --- builtin/io_ysh.py | 17 ++++++- builtin/meta_osh.py | 3 +- core/completion_test.py | 14 +++--- core/shell.py | 29 +++++++++--- core/state.py | 15 ++---- frontend/option_def.py | 8 ++-- spec/ysh-builtin-eval.test.sh | 86 +++++++++++++++++++++++++++++++---- spec/ysh-usage.test.sh | 9 ++++ 8 files changed, 137 insertions(+), 44 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 9350c3612c..0f8aa4a5c2 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -19,7 +19,7 @@ from frontend import match from frontend import typed_args from mycpp import mylib -from mycpp.mylib import tagswitch, log +from mycpp.mylib import tagswitch, log, iteritems from typing import TYPE_CHECKING, cast if TYPE_CHECKING: @@ -175,6 +175,21 @@ def Run(self, cmd_val): self.stdout_.write('\n') return status + if action == 'stacks_': # Format may change + if mylib.PYTHON: + var_stack, argv_stack, unused = self.mem.Dump() + print(var_stack) + print('===') + print(argv_stack) + if 0: + var_stack = self.mem.var_stack + for i, frame in enumerate(var_stack): + print('=== Frame %d' % i) + for name, cell in iteritems(frame): + print('%s = %s' % (name, cell)) + + return 0 + if action == 'gc-stats_': print('TODO') return 0 diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index ba31e349a8..4c8c50eb01 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -78,8 +78,7 @@ def RunTyped(self, cmd_val): for arg in pos_args_raw: if arg.tag() != value_e.Str: raise error.TypeErr( - arg, - "Expected pos_args to be a list of Strs", + arg, "Expected pos_args to be a list of Strs", rd.LeftParenToken()) pos_args.append(cast(value.Str, arg).s) diff --git a/core/completion_test.py b/core/completion_test.py index f963366fc3..c752473bdc 100755 --- a/core/completion_test.py +++ b/core/completion_test.py @@ -303,7 +303,7 @@ def testCompletesVarNames(self): self.assertEqual(7, comp.end) print(comp) m = list(r.Matches(comp)) - self.assert_('echo $PWD' in m, 'Got %s' % m) + self.assert_('echo $PPID' in m, 'Got %s' % m) self.assert_('echo $PS4' in m, 'Got %s' % m) # @@ -322,7 +322,7 @@ def testCompletesVarNames(self): comp = MockApi(line='echo ${P') print(comp) m = list(r.Matches(comp)) - self.assert_('echo ${PWD' in m, 'Got %s' % m) + self.assert_('echo ${PPID' in m, 'Got %s' % m) self.assert_('echo ${PS4' in m, 'Got %s' % m) # Odd word break @@ -330,7 +330,7 @@ def testCompletesVarNames(self): comp = MockApi(line='echo ${undef:-$P') print(comp) m = list(r.Matches(comp)) - self.assert_('echo ${undef:-$PWD' in m, 'Got %s' % m) + self.assert_('echo ${undef:-$PPID' in m, 'Got %s' % m) self.assert_('echo ${undef:-$PS4' in m, 'Got %s' % m) comp = MockApi(line='echo ${undef:-$') @@ -353,7 +353,7 @@ def testCompletesVarNames(self): comp = MockApi(line='echo "$P') print(comp) m = list(r.Matches(comp)) - self.assert_('echo "$PWD' in m, 'Got %s' % m) + self.assert_('echo "$PPID' in m, 'Got %s' % m) self.assert_('echo "$PS4' in m, 'Got %s' % m) # @@ -370,7 +370,7 @@ def testCompletesVarNames(self): comp = MockApi(line='echo "${#P') print(comp) m = list(r.Matches(comp)) - self.assert_('echo "${#PWD' in m, 'Got %s' % m) + self.assert_('echo "${#PPID' in m, 'Got %s' % m) self.assert_('echo "${#PS4' in m, 'Got %s' % m) # @@ -380,13 +380,13 @@ def testCompletesVarNames(self): comp = MockApi(line='echo "$((PWD +P') # bare word print(comp) m = list(r.Matches(comp)) - self.assert_('echo "$((PWD +PWD' in m, 'Got %s' % m) + self.assert_('echo "$((PWD +PPID' in m, 'Got %s' % m) self.assert_('echo "$((PWD +PS4' in m, 'Got %s' % m) comp = MockApi(line='echo "$(( $P') print(comp) m = list(r.Matches(comp)) - self.assert_('echo "$(( $PWD' in m, 'Got %s' % m) # word with $ + self.assert_('echo "$(( $PPID' in m, 'Got %s' % m) # word with $ self.assert_('echo "$(( $PS4' in m, 'Got %s' % m) def testCompletesCommandSubs(self): diff --git a/core/shell.py b/core/shell.py index dd3b41cc26..437d26097e 100644 --- a/core/shell.py +++ b/core/shell.py @@ -355,13 +355,6 @@ def Main( mem.exec_opts = exec_opts # circular dep mutable_opts.Init() - version_str = pyutil.GetVersion(loader) - state.InitMem(mem, environ, version_str) - - if attrs.show_options: # special case: sh -o - mutable_opts.ShowOptions([]) - return 0 - # Set these BEFORE processing flags, so they can be overridden. if lang == 'ysh': mutable_opts.SetAnyOption('ysh:all', True) @@ -369,6 +362,28 @@ def Main( pure_osh.SetOptionsFromFlags(mutable_opts, attrs.opt_changes, attrs.shopt_changes) + version_str = pyutil.GetVersion(loader) + state.InitMem(mem, environ, version_str) + + # TODO: consider turning on no_copy_env in YSH + if exec_opts.no_copy_env(): + # Don't consult the environment + mem.SetPwd(state.GetWorkingDir()) + else: + state.InitVarsFromEnv(mem, environ) + + # MUTABLE GLOBAL that's SEPARATE from $PWD. Used by the 'pwd' builtin, but + # it can't be modified by users. + val = mem.GetValue('PWD') + # should be true since it's exported + assert val.tag() == value_e.Str, val + pwd = cast(value.Str, val).s + mem.SetPwd(pwd) + + if attrs.show_options: # special case: sh -o + mutable_opts.ShowOptions([]) + return 0 + # feedback between runtime and parser aliases = {} # type: Dict[str, str] diff --git a/core/state.py b/core/state.py index 8845bfbbcb..d7ff39f144 100644 --- a/core/state.py +++ b/core/state.py @@ -820,7 +820,7 @@ def _DumpVarFrame(frame): return vars_json -def _GetWorkingDir(): +def GetWorkingDir(): # type: () -> str """Fallback for pwd and $PWD when there's no 'cd' and no inherited $PWD.""" try: @@ -881,7 +881,7 @@ def _InitDefaults(mem): # set_home_var (); -def _InitVarsFromEnv(mem, environ): +def InitVarsFromEnv(mem, environ): # type: (Mem, Dict[str, str]) -> None # This is the way dash and bash work -- at startup, they turn everything in @@ -912,7 +912,7 @@ def _InitVarsFromEnv(mem, environ): # compute it. val = mem.GetValue('PWD') if val.tag() == value_e.Undef: - SetGlobalString(mem, 'PWD', _GetWorkingDir()) + SetGlobalString(mem, 'PWD', GetWorkingDir()) # Now mark it exported, no matter what. This is one of few variables # EXPORTED. bash and dash both do it. (e.g. env -i -- dash -c env) mem.SetNamed(location.LName('PWD'), @@ -953,15 +953,6 @@ def InitMem(mem, environ, version_str): _SetGlobalValue(mem, 'INFINITY', value.Float(pyutil.infinity())) _InitDefaults(mem) - _InitVarsFromEnv(mem, environ) - - # MUTABLE GLOBAL that's SEPARATE from $PWD. Used by the 'pwd' builtin, but - # it can't be modified by users. - val = mem.GetValue('PWD') - # should be true since it's exported - assert val.tag() == value_e.Str, val - pwd = cast(value.Str, val).s - mem.SetPwd(pwd) def InitInteractive(mem): diff --git a/frontend/option_def.py b/frontend/option_def.py index 335ae11181..7e5a32ada1 100644 --- a/frontend/option_def.py +++ b/frontend/option_def.py @@ -284,11 +284,9 @@ def _Init(opt_def): opt_def.Add('extglob') opt_def.Add('nocasematch') - # TODO: Opt-in to optimization, which may causes correctness issues: - # - running traps - # - job control restoration with set -m - # - verbose_errexit doesn't get a chance to run - opt_def.Add('no_fork_last') + # Should we copy the environment in to the global stack frame? + # TODO: This may be off in YSH + opt_def.Add('no_copy_env') # recursive parsing and evaluation - for compatibility, ble.sh, etc. opt_def.Add('eval_unsafe_arith') diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index a7eff2aec8..2fe129b91d 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -107,22 +107,79 @@ foo bar baz ## END #### eval lines with argv bindings -proc lines (;;; block) { +proc my-split (;;; block) { while read --raw-line { var cols = _reply => split() eval (block, pos_args=cols) } } -printf 'a b\nc d\n' | lines { echo $1 } +printf 'a b\nc d\n' | my-split { + echo "$2 $1" +} + +printf 'a b\nc d\n' | my-split { + var mylocal = 'mylocal' + echo "$2 $1 $mylocal" +} + +# Now do the same thing inside a proc +proc p { + printf 'a b\nc d\n' | my-split { + var local2 = 'local2' + echo "$2 $1 $local2" + } +} + +echo +p + +## STDOUT: +b a +d c +b a mylocal +d c mylocal + +b a local2 +d c local2 +## END + +#### eval lines with var bindings + +proc my-split (;;; block) { + while read --raw-line { + var cols = _reply => split() + eval (block, vars={_line: _reply, _first: cols[0]}) + } +} + +printf 'a b\nc d\n' | my-split { + var mylocal = 'mylocal' + echo "$_line | $_first $mylocal" +} + +# Now do the same thing inside a proc +proc p { + printf 'a b\nc d\n' | my-split { + var local2 = 'local2' + echo "$_line | $_first $local2" + } +} + +echo +p ## STDOUT: -a -c +a b | a mylocal +c d | c mylocal + +a b | a local2 +c d | c local2 ## END #### eval with custom dollar0 -eval (^(write $0), dollar0="my arg0") +var b = ^(write $0) +eval (b, dollar0="my arg0") ## STDOUT: my arg0 ## END @@ -154,17 +211,26 @@ proc foreach (binding, in_; list ;; block) { var mydicts = [{'a': 1}, {'b': 2}, {'c': 3}] foreach mydict in (mydicts) { + var mylocal = 'z' + setvar mydict.z = mylocal + pp test_ (mydict) setvar mydict.d = 0 } +echo -pp test_ (mydicts) +for d in (mydicts) { + pp test_ (d) +} ## STDOUT: -(Dict) {"a":1} -(Dict) {"b":2} -(Dict) {"c":3} -(List) [{"a":1,"d":0},{"b":2,"d":0},{"c":3,"d":0}] +(Dict) {"a":1,"z":"z"} +(Dict) {"b":2,"z":"z"} +(Dict) {"c":3,"z":"z"} + +(Dict) {"a":1,"z":"z","d":0} +(Dict) {"b":2,"z":"z","d":0} +(Dict) {"c":3,"z":"z","d":0} ## END #### binding procs in the eval-ed namespace diff --git a/spec/ysh-usage.test.sh b/spec/ysh-usage.test.sh index 9b3bd5aee6..d8aefa5d3e 100644 --- a/spec/ysh-usage.test.sh +++ b/spec/ysh-usage.test.sh @@ -66,3 +66,12 @@ wc -l on.txt off.txt 0 off.txt 3 total ## END + +#### YSH shows options correctly (bug fix) + +$SH -o | egrep 'errexit|pipefail' + +## STDOUT: +set -o errexit +set -o pipefail +## END From 76a717fc6208ef002ea9f0423820582cb874d1a1 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 18:51:58 -0400 Subject: [PATCH 127/506] [core refactor] Make Dict_ its own type Add a prototype chain! Not used yet. --- builtin/error_ysh.py | 6 +-- builtin/func_hay.py | 6 +-- builtin/func_misc.py | 15 +++--- builtin/hay_ysh.py | 10 ++-- core/dev.py | 10 ++-- core/error.py | 6 +-- core/shell.py | 2 +- core/state.py | 18 +++---- core/value.asdl | 5 +- data_lang/j8.py | 11 +++-- display/pp_value.py | 7 +-- frontend/typed_args.py | 7 +-- frontend/typed_args_test.py | 4 +- osh/cmd_eval.py | 6 +-- prebuilt/core/error.mycpp.cc | 81 +++++--------------------------- prebuilt/core/error.mycpp.h | 2 +- prebuilt/frontend/args.mycpp.cc | 83 +++++---------------------------- spec/ysh-json.test.sh | 4 +- ysh/expr_eval.py | 13 +++--- ysh/func_proc.py | 6 +-- ysh/val_ops.py | 28 +++++++---- 21 files changed, 115 insertions(+), 215 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index e5802892da..2f082a044c 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -4,7 +4,7 @@ from _devbuild.gen.id_kind_asdl import Id from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus from _devbuild.gen.syntax_asdl import loc, loc_t, expr, expr_e -from _devbuild.gen.value_asdl import value, value_e +from _devbuild.gen.value_asdl import value, value_e, Dict_ from core import error from core.error import e_die_status, e_usage from core import executor @@ -99,7 +99,7 @@ def Run(self, cmd_val): cmd = rd.RequiredBlock() rd.Done() - error_dict = None # type: value.Dict + error_dict = None # type: Dict_ status = 0 # success by default try: @@ -116,7 +116,7 @@ def Run(self, cmd_val): error_dict = e.ToDict() if error_dict is None: - error_dict = value.Dict({'code': num.ToBig(status)}) + error_dict = Dict_({'code': num.ToBig(status)}, None) # Always set _error self.mem.SetTryError(error_dict) diff --git a/builtin/func_hay.py b/builtin/func_hay.py index 121aae3ebd..05e03777a6 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -3,7 +3,7 @@ from __future__ import print_function from _devbuild.gen.syntax_asdl import source, loc, command_t -from _devbuild.gen.value_asdl import value +from _devbuild.gen.value_asdl import value, Dict_ from builtin import hay_ysh from core import alloc from core import error @@ -106,7 +106,7 @@ def Call(self, rd): cmd = rd.PosCommand() rd.Done() - return value.Dict(self._Call(cmd)) + return Dict_(self._Call(cmd), None) class BlockAsStr(vm._Callable): @@ -147,4 +147,4 @@ def Call(self, rd): # type: (typed_args.Reader) -> value_t # TODO: check args - return value.Dict(self._Call()) + return Dict_(self._Call(), None) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 8b0b9c6100..9f9cbcf9a7 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -5,7 +5,8 @@ from __future__ import print_function from _devbuild.gen.runtime_asdl import (scope_e) -from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str) +from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, + Dict_) from core import error from core import num @@ -49,7 +50,7 @@ def Call(self, rd): return num.ToBig(len(x.items)) elif case(value_e.Dict): - x = cast(value.Dict, UP_x) + x = cast(Dict_, UP_x) return num.ToBig(len(x.d)) elif case(value_e.Str): @@ -263,7 +264,7 @@ def Call(self, rd): it = val_ops.ListIterator(val) elif case(value_e.Dict): - val = cast(value.Dict, UP_val) + val = cast(Dict_, UP_val) it = val_ops.DictIterator(val) elif case(value_e.Range): @@ -286,7 +287,7 @@ def Call(self, rd): return value.List(l) -class Dict_(vm._Callable): +class DictFunc(vm._Callable): def __init__(self): # type: () -> None @@ -302,11 +303,11 @@ def Call(self, rd): with tagswitch(val) as case: if case(value_e.Dict): d = NewDict() # type: Dict[str, value_t] - val = cast(value.Dict, UP_val) + val = cast(Dict_, UP_val) for k, v in iteritems(val.d): d[k] = v - return value.Dict(d) + return Dict_(d, None) elif case(value_e.BashAssoc): d = NewDict() @@ -314,7 +315,7 @@ def Call(self, rd): for k, s in iteritems(val.d): d[k] = value.Str(s) - return value.Dict(d) + return Dict_(d, None) raise error.TypeErr(val, 'dict() expected Dict or BashAssoc', rd.BlamePos()) diff --git a/builtin/hay_ysh.py b/builtin/hay_ysh.py index e9bc3ba187..aff308d4d5 100644 --- a/builtin/hay_ysh.py +++ b/builtin/hay_ysh.py @@ -3,7 +3,7 @@ from _devbuild.gen.option_asdl import option_i from _devbuild.gen.runtime_asdl import (scope_e, HayNode) from _devbuild.gen.syntax_asdl import loc -from _devbuild.gen.value_asdl import (value, value_e, value_t) +from _devbuild.gen.value_asdl import (value, value_e, value_t, Dict_) from asdl import format as fmt from core import alloc @@ -157,7 +157,7 @@ def AppendResult(self, d): UP_children = self.result_stack[-1]['children'] assert UP_children.tag() == value_e.List, UP_children children = cast(value.List, UP_children) - children.items.append(value.Dict(d)) + children.items.append(Dict_(d, None)) def Result(self): # type: () -> Dict[str, value_t] @@ -206,7 +206,7 @@ def Push(self, hay_name): top = self.result_stack[-1] # TODO: Store this more efficiently? See osh/builtin_pure.py children = cast(value.List, top['children']) - last_child = cast(value.Dict, children.items[-1]) + last_child = cast(Dict_, children.items[-1]) self.result_stack.append(last_child.d) #log('> PUSH') @@ -295,7 +295,7 @@ def Run(self, cmd_val): result = self.hay_state.Result() - val = value.Dict(result) + val = Dict_(result, None) self.mem.SetNamed(location.LName(var_name), val, scope_e.LocalOnly) elif action == 'reset': @@ -426,6 +426,6 @@ def Run(self, cmd_val): attrs[name] = cell.val - result['attrs'] = value.Dict(attrs) + result['attrs'] = Dict_(attrs, None) return 0 diff --git a/core/dev.py b/core/dev.py index d4dbf427cc..68f5ad48d7 100644 --- a/core/dev.py +++ b/core/dev.py @@ -8,7 +8,7 @@ trace_t) from _devbuild.gen.syntax_asdl import assign_op_e, Token from _devbuild.gen.value_asdl import (value, value_e, value_t, sh_lvalue, - sh_lvalue_e, LeftName) + sh_lvalue_e, LeftName, Dict_) from core import error from core import optview @@ -143,7 +143,7 @@ def MaybeDump(self, status): 'var_stack': value.List(self.var_stack), 'argv_stack': value.List(self.argv_stack), 'debug_stack': value.List(self.debug_stack), - 'error': value.Dict(self.error), + 'error': Dict_(self.error, None), 'status': num.ToBig(status), 'pid': num.ToBig(my_pid), } # type: Dict[str, value_t] @@ -153,7 +153,7 @@ def MaybeDump(self, status): # TODO: This should be JSON with unicode replacement char? buf = mylib.BufWriter() - j8.PrintMessage(value.Dict(d), buf, 2) + j8.PrintMessage(Dict_(d, None), buf, 2) json_str = buf.getvalue() try: @@ -346,7 +346,7 @@ def WriteDumps(self): a = value.Str(argv0) c = value.Int(mops.IntWiden(count)) d = {'argv0': a, 'count': c} - metric_argv0.append(value.Dict(d)) + metric_argv0.append(Dict_(d, None)) # Other things we need: the reason for the crash! _ErrorWithLocation is # required I think. @@ -359,7 +359,7 @@ def WriteDumps(self): path = os_path.join(self.out_dir, '%d.argv0.json' % self.this_pid) buf = mylib.BufWriter() - j8.PrintMessage(value.Dict(j), buf, 2) + j8.PrintMessage(Dict_(j, None), buf, 2) json8_str = buf.getvalue() try: diff --git a/core/error.py b/core/error.py index a1affda08d..3ccb7f8eec 100644 --- a/core/error.py +++ b/core/error.py @@ -2,7 +2,7 @@ from __future__ import print_function from _devbuild.gen.syntax_asdl import loc_e, loc_t, loc -from _devbuild.gen.value_asdl import (value, value_t, value_str) +from _devbuild.gen.value_asdl import (value, value_t, value_str, Dict_) from core import num from mycpp.mylib import NewDict @@ -172,7 +172,7 @@ def __init__(self, status, msg, location, properties=None): self.properties = properties def ToDict(self): - # type: () -> value.Dict + # type: () -> Dict_ d = NewDict() # type: Dict[str, value_t] @@ -186,7 +186,7 @@ def ToDict(self): d['code'] = num.ToBig(self.ExitStatus()) d['message'] = value.Str(self.msg) - return value.Dict(d) + return Dict_(d, None) class AssertionErr(Expr): diff --git a/core/shell.py b/core/shell.py index 437d26097e..8cc73a5e70 100644 --- a/core/shell.py +++ b/core/shell.py @@ -851,7 +851,7 @@ def Main( _SetGlobalFunc(mem, 'float', func_misc.Float()) _SetGlobalFunc(mem, 'str', func_misc.Str_()) _SetGlobalFunc(mem, 'list', func_misc.List_()) - _SetGlobalFunc(mem, 'dict', func_misc.Dict_()) + _SetGlobalFunc(mem, 'dict', func_misc.DictFunc()) _SetGlobalFunc(mem, 'runes', func_misc.Runes()) _SetGlobalFunc(mem, 'encodeRunes', func_misc.EncodeRunes()) diff --git a/core/state.py b/core/state.py index d7ff39f144..8825e77bf5 100644 --- a/core/state.py +++ b/core/state.py @@ -19,7 +19,7 @@ from _devbuild.gen.value_asdl import (value, value_e, value_t, sh_lvalue, sh_lvalue_e, sh_lvalue_t, LeftName, y_lvalue_e, regex_match, regex_match_e, - regex_match_t, RegexMatch) + regex_match_t, RegexMatch, Dict_) from core import error from core.error import e_usage, e_die from core import num @@ -815,7 +815,7 @@ def _DumpVarFrame(frame): # TODO: should we show the object ID here? pass - vars_json[name] = value.Dict(cell_json) + vars_json[name] = Dict_(cell_json, None) return vars_json @@ -1071,7 +1071,7 @@ def __init__(self, mem): last = mem.last_status[-1] mem.last_status.append(last) mem.try_status.append(0) - mem.try_error.append(value.Dict({})) + mem.try_error.append(Dict_({}, None)) # TODO: We should also copy these values! Turn the whole thing into a # frame. @@ -1225,7 +1225,7 @@ def __init__(self, dollar0, argv, arena, debug_stack): # - push-registers builtin self.last_status = [0] # type: List[int] # a stack self.try_status = [0] # type: List[int] # a stack - self.try_error = [value.Dict({})] # type: List[value.Dict] # a stack + self.try_error = [Dict_({}, None)] # type: List[Dict_] # a stack self.pipe_status = [[]] # type: List[List[int]] # stack self.process_sub_status = [[]] # type: List[List[int]] # stack @@ -1271,9 +1271,9 @@ def Dump(self): # type: () -> Tuple[List[value_t], List[value_t], List[value_t]] """Copy state before unwinding the stack.""" var_stack = [ - value.Dict(_DumpVarFrame(frame)) for frame in self.var_stack + Dict_(_DumpVarFrame(frame), None) for frame in self.var_stack ] # type: List[value_t] - argv_stack = [value.Dict(frame.Dump()) + argv_stack = [Dict_(frame.Dump(), None) for frame in self.argv_stack] # type: List[value_t] debug_stack = [] # type: List[value_t] @@ -1308,7 +1308,7 @@ def Dump(self): frame = cast(debug_frame.Main, UP_frame) d = {'type': t_main, 'dollar0': value.Str(frame.dollar0)} - debug_stack.append(value.Dict(d)) + debug_stack.append(Dict_(d, None)) return var_stack, argv_stack, debug_stack def SetLastArgument(self, s): @@ -1376,7 +1376,7 @@ def TryStatus(self): return self.try_status[-1] def TryError(self): - # type: () -> value.Dict + # type: () -> Dict_ return self.try_error[-1] def PipeStatus(self): @@ -1392,7 +1392,7 @@ def SetTryStatus(self, x): self.try_status[-1] = x def SetTryError(self, x): - # type: (value.Dict) -> None + # type: (Dict_) -> None self.try_error[-1] = x def SetPipeStatus(self, x): diff --git a/core/value.asdl b/core/value.asdl index bb5934ea66..255b5092ac 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -58,6 +58,9 @@ module value No | Yes %RegexMatch + # prototype is for the attribute lookup chain + Dict_ = (Dict[str, value] d, Dict_? prototype) + # Commands, words, and expressions from syntax.asdl are evaluated to a VALUE. # value_t instances are stored in state.Mem(). value = @@ -90,7 +93,7 @@ module value #| Int(int i) | Float(float f) | List(List[value] items) - | Dict(Dict[str, value] d) + | Dict %Dict_ # CODE types # unevaluated: Eggex, Expr, Template, Command/Block diff --git a/data_lang/j8.py b/data_lang/j8.py index d0347546d5..ce00fa7488 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -31,7 +31,8 @@ import math from _devbuild.gen.id_kind_asdl import Id, Id_t, Id_str -from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str) +from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, + Dict_) from _devbuild.gen.nil8_asdl import (nvalue, nvalue_t) from asdl import format as fmt @@ -306,7 +307,7 @@ def _PrintList(self, val, level): self.buf.write(']') def _PrintDict(self, val, level): - # type: (value.Dict, int) -> None + # type: (Dict_, int) -> None if len(val.d) == 0: # Special case like Python/JS self.buf.write('{}') @@ -550,7 +551,7 @@ def Print(self, val, level=0): self.visited[heap_id] = FINISHED elif case(value_e.Dict): - val = cast(value.Dict, UP_val) + val = cast(Dict_, UP_val) # Cycle detection, only for containers that can be in cycles heap_id = HeapValueId(val) @@ -939,7 +940,7 @@ def _ParseDict(self): self._Next() if self.tok_id == Id.J8_RBrace: self._Next() - return value.Dict(d) + return Dict_(d, None) k, v = self._ParsePair() d[k] = v @@ -955,7 +956,7 @@ def _ParseDict(self): #log('< Dict') - return value.Dict(d) + return Dict_(d, None) def _ParseList(self): # type: () -> value_t diff --git a/display/pp_value.py b/display/pp_value.py index f4bf01eef0..dad3970cb3 100644 --- a/display/pp_value.py +++ b/display/pp_value.py @@ -8,7 +8,8 @@ import math from _devbuild.gen.pretty_asdl import (doc, Measure, MeasuredDoc) -from _devbuild.gen.value_asdl import value, value_e, value_t, value_str +from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, + Dict_) from data_lang import j8 from data_lang import j8_lite from display.pretty import (_Break, _Concat, _Flat, _Group, _IfFlat, _Indent, @@ -326,7 +327,7 @@ def _YshList(self, vlist): return self._Surrounded("[", self._Tabular(mdocs, ","), "]") def _YshDict(self, vdict): - # type: (value.Dict) -> MeasuredDoc + # type: (Dict_) -> MeasuredDoc if len(vdict.d) == 0: return UText("{}") mdocs = [] # type: List[MeasuredDoc] @@ -433,7 +434,7 @@ def _Value(self, val): return result elif case(value_e.Dict): - vdict = cast(value.Dict, val) + vdict = cast(Dict_, val) heap_id = j8.HeapValueId(vdict) if self.visiting.get(heap_id, False): return _Concat([ diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 6c0169d836..fac76b29a8 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -4,7 +4,8 @@ from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, LiteralBlock, command_t, expr_t, Token) -from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch) +from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, + Dict_) from core import error from core.error import e_usage from frontend import location @@ -265,7 +266,7 @@ def _ToList(self, val): def _ToDict(self, val): # type: (value_t) -> Dict[str, value_t] if val.tag() == value_e.Dict: - return cast(value.Dict, val).d + return cast(Dict_, val).d raise error.TypeErr(val, 'Arg %d should be a Dict' % self.pos_consumed, self.BlamePos()) @@ -553,7 +554,7 @@ def NamedDict(self, param_name, default_): val = self.named_args[param_name] UP_val = val if val.tag() == value_e.Dict: - val = cast(value.Dict, UP_val) + val = cast(Dict_, UP_val) mylib.dict_erase(self.named_args, param_name) return val.d diff --git a/frontend/typed_args_test.py b/frontend/typed_args_test.py index a3696f1709..1904583037 100755 --- a/frontend/typed_args_test.py +++ b/frontend/typed_args_test.py @@ -36,7 +36,7 @@ def testReaderPosArgs(self): value.Str('foo'), value.List([value.Int(1), value.Int(2), value.Int(3)]), - value.Dict({ + Dict_({ 'a': value.Int(0xaa), 'b': value.Int(0xbb) }), @@ -110,7 +110,7 @@ def testReaderKwargs(self): 'numbers': value.List([value.Int(1), value.Int(2), value.Int(3)]), - 'blah': value.Dict({ + 'blah': Dict_({ 'a': value.Int(0xaa), 'b': value.Int(0xbb) }), diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index a7360f6d15..ace03b269d 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -65,7 +65,7 @@ ) from _devbuild.gen.types_asdl import redir_arg_type_e from _devbuild.gen.value_asdl import (value, value_e, value_t, y_lvalue, - y_lvalue_e, y_lvalue_t, LeftName) + y_lvalue_e, y_lvalue_t, LeftName, Dict_) from core import dev from core import error @@ -743,7 +743,7 @@ def _DoMutation(self, node): obj.items[index] = rval elif case(value_e.Dict): - obj = cast(value.Dict, UP_obj) + obj = cast(Dict_, UP_obj) key = val_ops.ToStr(lval.index, 'Dict index should be Str', loc.Missing) @@ -1154,7 +1154,7 @@ def _DoForEach(self, node): node.keyword) elif case(value_e.Dict): - val = cast(value.Dict, UP_val) + val = cast(Dict_, UP_val) it2 = val_ops.DictIterator(val) if n == 1: diff --git a/prebuilt/core/error.mycpp.cc b/prebuilt/core/error.mycpp.cc index c08fe210cf..1a2b32255e 100644 --- a/prebuilt/core/error.mycpp.cc +++ b/prebuilt/core/error.mycpp.cc @@ -55,11 +55,6 @@ namespace num { // declare value::Int* ToBig(int i); mops::BigInt Exponent(mops::BigInt x, mops::BigInt y); -int Exponent2(int x, int y); -mops::BigInt IntDivide(mops::BigInt x, mops::BigInt y); -int IntDivide2(int x, int y); -mops::BigInt IntRemainder(mops::BigInt x, mops::BigInt y); -int IntRemainder2(int x, int y); } // declare namespace num @@ -104,6 +99,7 @@ using syntax_asdl::loc; using value_asdl::value; using value_asdl::value_t; using value_asdl::value_str; +using value_asdl::Dict_; BigStr* _ValType(value_asdl::value_t* val) { StackRoot _root0(&val); @@ -163,13 +159,17 @@ Structured::Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Di this->properties = properties; } -value::Dict* Structured::ToDict() { - if (this->properties == nullptr) { - this->properties = Alloc>(); +value_asdl::Dict_* Structured::ToDict() { + Dict* d = nullptr; + StackRoot _root0(&d); + + d = Alloc>(); + if (this->properties != nullptr) { + d->update(this->properties); } - this->properties->set(str6, num::ToBig(this->ExitStatus())); - this->properties->set(str7, Alloc(this->msg)); - return Alloc(this->properties); + d->set(str6, num::ToBig(this->ExitStatus())); + d->set(str7, Alloc(this->msg)); + return Alloc(d, nullptr); } AssertionErr::AssertionErr(BigStr* msg, syntax_asdl::loc_t* location) : ::error::Expr(msg, location) { @@ -277,64 +277,5 @@ mops::BigInt Exponent(mops::BigInt x, mops::BigInt y) { return result; } -int Exponent2(int x, int y) { - return mops::BigTruncate(Exponent(mops::IntWiden(x), mops::IntWiden(y))); -} - -mops::BigInt IntDivide(mops::BigInt x, mops::BigInt y) { - mops::BigInt ZERO; - int sign; - mops::BigInt ax; - mops::BigInt ay; - ZERO = mops::BigInt(0); - sign = 1; - if (mops::Greater(ZERO, x)) { - ax = mops::Negate(x); - sign = -1; - } - else { - ax = x; - } - if (mops::Greater(ZERO, y)) { - ay = mops::Negate(y); - sign = -sign; - } - else { - ay = y; - } - return mops::Mul(mops::IntWiden(sign), mops::Div(ax, ay)); -} - -int IntDivide2(int x, int y) { - return mops::BigTruncate(IntDivide(mops::IntWiden(x), mops::IntWiden(y))); -} - -mops::BigInt IntRemainder(mops::BigInt x, mops::BigInt y) { - mops::BigInt ZERO; - mops::BigInt ax; - int sign; - mops::BigInt ay; - ZERO = mops::BigInt(0); - if (mops::Greater(ZERO, x)) { - ax = mops::Negate(x); - sign = -1; - } - else { - ax = x; - sign = 1; - } - if (mops::Greater(ZERO, y)) { - ay = mops::Negate(y); - } - else { - ay = y; - } - return mops::Mul(mops::IntWiden(sign), mops::Rem(ax, ay)); -} - -int IntRemainder2(int x, int y) { - return mops::BigTruncate(IntRemainder(mops::IntWiden(x), mops::IntWiden(y))); -} - } // define namespace num diff --git a/prebuilt/core/error.mycpp.h b/prebuilt/core/error.mycpp.h index 584979c783..d3cfc60554 100644 --- a/prebuilt/core/error.mycpp.h +++ b/prebuilt/core/error.mycpp.h @@ -187,7 +187,7 @@ class Expr : public ::error::FatalRuntime { class Structured : public ::error::FatalRuntime { public: Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties = nullptr); - value::Dict* ToDict(); + value_asdl::Dict_* ToDict(); Dict* properties; diff --git a/prebuilt/frontend/args.mycpp.cc b/prebuilt/frontend/args.mycpp.cc index dc3c03ee28..b19b043a0c 100644 --- a/prebuilt/frontend/args.mycpp.cc +++ b/prebuilt/frontend/args.mycpp.cc @@ -387,7 +387,7 @@ class Expr : public ::error::FatalRuntime { class Structured : public ::error::FatalRuntime { public: Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties = nullptr); - value::Dict* ToDict(); + value_asdl::Dict_* ToDict(); Dict* properties; @@ -504,11 +504,6 @@ namespace num { // declare value::Int* ToBig(int i); mops::BigInt Exponent(mops::BigInt x, mops::BigInt y); -int Exponent2(int x, int y); -mops::BigInt IntDivide(mops::BigInt x, mops::BigInt y); -int IntDivide2(int x, int y); -mops::BigInt IntRemainder(mops::BigInt x, mops::BigInt y); -int IntRemainder2(int x, int y); } // declare namespace num @@ -1415,6 +1410,7 @@ using syntax_asdl::loc; using value_asdl::value; using value_asdl::value_t; using value_asdl::value_str; +using value_asdl::Dict_; BigStr* _ValType(value_asdl::value_t* val) { StackRoot _root0(&val); @@ -1474,13 +1470,17 @@ Structured::Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Di this->properties = properties; } -value::Dict* Structured::ToDict() { - if (this->properties == nullptr) { - this->properties = Alloc>(); +value_asdl::Dict_* Structured::ToDict() { + Dict* d = nullptr; + StackRoot _root0(&d); + + d = Alloc>(); + if (this->properties != nullptr) { + d->update(this->properties); } - this->properties->set(str62, num::ToBig(this->ExitStatus())); - this->properties->set(str63, Alloc(this->msg)); - return Alloc(this->properties); + d->set(str62, num::ToBig(this->ExitStatus())); + d->set(str63, Alloc(this->msg)); + return Alloc(d, nullptr); } AssertionErr::AssertionErr(BigStr* msg, syntax_asdl::loc_t* location) : ::error::Expr(msg, location) { @@ -1588,65 +1588,6 @@ mops::BigInt Exponent(mops::BigInt x, mops::BigInt y) { return result; } -int Exponent2(int x, int y) { - return mops::BigTruncate(Exponent(mops::IntWiden(x), mops::IntWiden(y))); -} - -mops::BigInt IntDivide(mops::BigInt x, mops::BigInt y) { - mops::BigInt ZERO; - int sign; - mops::BigInt ax; - mops::BigInt ay; - ZERO = mops::BigInt(0); - sign = 1; - if (mops::Greater(ZERO, x)) { - ax = mops::Negate(x); - sign = -1; - } - else { - ax = x; - } - if (mops::Greater(ZERO, y)) { - ay = mops::Negate(y); - sign = -sign; - } - else { - ay = y; - } - return mops::Mul(mops::IntWiden(sign), mops::Div(ax, ay)); -} - -int IntDivide2(int x, int y) { - return mops::BigTruncate(IntDivide(mops::IntWiden(x), mops::IntWiden(y))); -} - -mops::BigInt IntRemainder(mops::BigInt x, mops::BigInt y) { - mops::BigInt ZERO; - mops::BigInt ax; - int sign; - mops::BigInt ay; - ZERO = mops::BigInt(0); - if (mops::Greater(ZERO, x)) { - ax = mops::Negate(x); - sign = -1; - } - else { - ax = x; - sign = 1; - } - if (mops::Greater(ZERO, y)) { - ay = mops::Negate(y); - } - else { - ay = y; - } - return mops::Mul(mops::IntWiden(sign), mops::Rem(ax, ay)); -} - -int IntRemainder2(int x, int y) { - return mops::BigTruncate(IntRemainder(mops::IntWiden(x), mops::IntWiden(y))); -} - } // define namespace num namespace args { // define diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index 2b7ba48699..1714ec60bb 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -153,14 +153,14 @@ echo '{"age": 42}' > $TMP/foo.txt json read (&x) < $TMP/foo.txt pp cell_ x ## STDOUT: -x = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:42)])) +x = (Cell exported:F readonly:F nameref:F val:(Dict_ d:[Dict age (value.Int i:42)])) ## END #### json read at end of pipeline (relies on lastpipe) echo '{"age": 43}' | json read (&y) pp cell_ y ## STDOUT: -y = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:43)])) +y = (Cell exported:F readonly:F nameref:F val:(Dict_ d:[Dict age (value.Int i:43)])) ## END #### invalid JSON diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 5506a95dea..77ccea1f23 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -45,7 +45,8 @@ Piece, ) from _devbuild.gen.value_asdl import (value, value_e, value_t, y_lvalue, - y_lvalue_e, y_lvalue_t, IntBox, LeftName) + y_lvalue_e, y_lvalue_t, IntBox, LeftName, + Dict_) from core import error from core.error import e_die, e_die_status from core import num @@ -238,7 +239,7 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): loc.Missing) elif case(value_e.Dict): - obj = cast(value.Dict, UP_obj) + obj = cast(Dict_, UP_obj) index = -1 # silence C++ warning key = val_ops.ToStr(lval.index, 'Dict index should be Str', @@ -267,7 +268,7 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): obj.items[index] = new_val_ elif case(value_e.Dict): - obj = cast(value.Dict, UP_obj) + obj = cast(Dict_, UP_obj) obj.d[key] = new_val_ else: @@ -912,7 +913,7 @@ def _EvalSubscript(self, obj, index): loc.Missing) elif case(value_e.Dict): - obj = cast(value.Dict, UP_obj) + obj = cast(Dict_, UP_obj) if index.tag() != value_e.Str: raise error.TypeErr(index, 'Dict index expected Str', loc.Missing) @@ -938,7 +939,7 @@ def _EvalDot(self, node, obj): UP_obj = obj with tagswitch(obj) as case: if case(value_e.Dict): - obj = cast(value.Dict, UP_obj) + obj = cast(Dict_, UP_obj) attr_name = node.attr_name try: result = obj.d[attr_name] @@ -1178,7 +1179,7 @@ def _EvalExpr(self, node): loc.Missing) d[k] = values[i] - return value.Dict(d) + return Dict_(d, None) elif case(expr_e.ListComp): e_die_status( diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 27d35c1da8..de185d002d 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -10,7 +10,7 @@ NamedArg, Func, loc, ArgList, expr, expr_e, expr_t) from _devbuild.gen.value_asdl import (value, value_e, value_t, ProcDefaults, - LeftName) + LeftName, Dict_) from core import error from core.error import e_die @@ -168,7 +168,7 @@ def _EvalNamedArgs(expr_ev, named_exprs): if val.tag() != value_e.Dict: raise error.TypeErr(val, 'Spread expected a Dict', val_expr.left) - named_args.update(cast(value.Dict, val).d) + named_args.update(cast(Dict_, val).d) else: val = expr_ev.EvalExpr(n.value, n.name) name = lexer.TokenVal(n.name) @@ -403,7 +403,7 @@ def _BindNamed( rest = group.rest_of if rest: lval = LeftName(rest.name, rest.blame_tok) - mem.SetLocalName(lval, value.Dict(named_args)) + mem.SetLocalName(lval, Dict_(named_args, None)) else: num_args = len(named_args) num_params = len(group.params) diff --git a/ysh/val_ops.py b/ysh/val_ops.py index 9da110afce..66cd3843b1 100644 --- a/ysh/val_ops.py +++ b/ysh/val_ops.py @@ -4,7 +4,8 @@ from _devbuild.gen.syntax_asdl import loc, loc_t, command_t from _devbuild.gen.value_asdl import (value, value_e, value_t, eggex_ops, - eggex_ops_t, regex_match, RegexMatch) + eggex_ops_t, regex_match, RegexMatch, + Dict_) from core import error from core.error import e_die from display import ui @@ -23,6 +24,15 @@ if TYPE_CHECKING: from core import state +if 0: + + def PlainDict(d): + # type: (Dict[str, value_t]) -> Dict_ + """ + Shorthand for "plain old data", i.e. data without behavior + """ + return Dict_(d, None) + def ToInt(val, msg, blame_loc): # type: (value_t, str, loc_t) -> int @@ -68,7 +78,7 @@ def ToDict(val, msg, blame_loc): # type: (value_t, str, loc_t) -> Dict[str, value_t] UP_val = val if val.tag() == value_e.Dict: - val = cast(value.Dict, UP_val) + val = cast(Dict_, UP_val) return val.d raise error.TypeErr(val, msg, blame_loc) @@ -299,7 +309,7 @@ class DictIterator(Iterator): """ for x in (mydict) { """ def __init__(self, val): - # type: (value.Dict) -> None + # type: (Dict_) -> None Iterator.__init__(self) # TODO: Don't materialize these Lists @@ -364,7 +374,7 @@ def ToBool(val): return len(val.items) > 0 elif case(value_e.Dict): - val = cast(value.Dict, UP_val) + val = cast(Dict_, UP_val) return len(val.d) > 0 else: @@ -433,8 +443,8 @@ def ExactlyEqual(left, right, blame_loc): return True elif case(value_e.BashAssoc): - left = cast(value.Dict, UP_left) - right = cast(value.Dict, UP_right) + left = cast(Dict_, UP_left) + right = cast(Dict_, UP_right) if len(left.d) != len(right.d): return False @@ -445,8 +455,8 @@ def ExactlyEqual(left, right, blame_loc): return True elif case(value_e.Dict): - left = cast(value.Dict, UP_left) - right = cast(value.Dict, UP_right) + left = cast(Dict_, UP_left) + right = cast(Dict_, UP_right) if len(left.d) != len(right.d): return False @@ -471,7 +481,7 @@ def Contains(needle, haystack): UP_haystack = haystack with tagswitch(haystack) as case: if case(value_e.Dict): - haystack = cast(value.Dict, UP_haystack) + haystack = cast(Dict_, UP_haystack) s = ToStr(needle, "LHS of 'in' should be Str", loc.Missing) return s in haystack.d From b57c586f6bcb171143af3035ec65d5adf89e3d6b Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 19:37:23 -0400 Subject: [PATCH 128/506] [test/unit] Fix build --- frontend/typed_args_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/typed_args_test.py b/frontend/typed_args_test.py index 1904583037..a900188640 100755 --- a/frontend/typed_args_test.py +++ b/frontend/typed_args_test.py @@ -6,7 +6,7 @@ import unittest from _devbuild.gen.syntax_asdl import ArgList, expr -from _devbuild.gen.value_asdl import value +from _devbuild.gen.value_asdl import value, Dict_ from core import error from core import test_lib from frontend import typed_args # module under test @@ -39,7 +39,7 @@ def testReaderPosArgs(self): Dict_({ 'a': value.Int(0xaa), 'b': value.Int(0xbb) - }), + }, None), value.Float(3.14), value.Int(0xdead), value.Int(0xbeef), @@ -113,7 +113,7 @@ def testReaderKwargs(self): 'blah': Dict_({ 'a': value.Int(0xaa), 'b': value.Int(0xbb) - }), + }, None), 'pi': value.Float(3.14), 'a': value.Int(0xdead), 'b': value.Int(0xbeef), From d75d710a0609ab2ced768edd1705eda915a8a5c0 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 19:45:05 -0400 Subject: [PATCH 129/506] [ysh] Object() allows prototype chain lookup So we can have polymorphic methods / prototypal inheritance, and not "flat objects" This isn't documented yet. We need to iron it out on a few use cases. Probably the main one is: = _io.stdin # this is the value.Stdin object, which is iterable call _io->eval(myblock) --- builtin/func_misc.py | 22 +++++++++++++++++++++ core/shell.py | 2 ++ frontend/typed_args.py | 13 +++++++++++++ spec/ysh-object.test.sh | 31 ++++++++++++++++++++++++++++++ test/spec.sh | 4 ++++ ysh/expr_eval.py | 42 +++++++++++++++++++++++++++++++++++------ 6 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 spec/ysh-object.test.sh diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 9f9cbcf9a7..b90a4cd1f9 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -31,6 +31,28 @@ _ = log +class Object(vm._Callable): + """ + Create an object. The order of params follows JavaScript's Object.create() + + var obj = Object(prototype, props) + """ + + def __init__(self): + # type: () -> None + pass + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + + prototype = rd.PosObject() + props = rd.PosDict() + rd.Done() + + # Opposite order + return Dict_(props, prototype) + + class Len(vm._Callable): def __init__(self): diff --git a/core/shell.py b/core/shell.py index 8cc73a5e70..fbc22ec71f 100644 --- a/core/shell.py +++ b/core/shell.py @@ -845,6 +845,8 @@ def Main( _SetGlobalFunc(mem, 'evalExpr', func_misc.EvalExpr(expr_ev)) + _SetGlobalFunc(mem, 'Object', func_misc.Object()) + # type conversions _SetGlobalFunc(mem, 'bool', func_misc.Bool()) _SetGlobalFunc(mem, 'int', func_misc.Int()) diff --git a/frontend/typed_args.py b/frontend/typed_args.py index fac76b29a8..f0fa4585df 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -271,6 +271,14 @@ def _ToDict(self, val): raise error.TypeErr(val, 'Arg %d should be a Dict' % self.pos_consumed, self.BlamePos()) + def _ToObject(self, val): + # type: (value_t) -> Dict_ + if val.tag() == value_e.Dict: + return cast(Dict_, val) + + raise error.TypeErr(val, 'Arg %d should be a Dict' % self.pos_consumed, + self.BlamePos()) + def _ToPlace(self, val): # type: (value_t) -> value.Place if val.tag() == value_e.Place: @@ -404,6 +412,11 @@ def PosDict(self): val = self.PosValue() return self._ToDict(val) + def PosObject(self): + # type: () -> Dict_ + val = self.PosValue() + return self._ToObject(val) + def PosPlace(self): # type: () -> value.Place val = self.PosValue() diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh new file mode 100644 index 0000000000..b644280279 --- /dev/null +++ b/spec/ysh-object.test.sh @@ -0,0 +1,31 @@ +## our_shell: ysh + +#### Object() creates prototype chain + +func Rect_area(this) { + return (this.x * this.y) +} + +var Rect = {area: Rect_area} + +var rect1 = Object(Rect, {x: 3, y: 4}) +var rect2 = Object(Rect, {x: 10, y: 20}) + +# This could change to show the object? +# pp test_ (rect) + +# TODO: This should be a bound function +#pp asdl_ (rect) +#pp (rect.area) +#pp (rect->area) + +var area1 = rect1.area() +var area2 = rect2.area() + +echo "area1 = $area1" +echo "area2 = $area2" + +## STDOUT: +area1 = 12 +area2 = 200 +## END diff --git a/test/spec.sh b/test/spec.sh index 7fc7b20d92..731d082832 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -831,6 +831,10 @@ ysh-method-io() { run-file ysh-method-io "$@" } +ysh-object() { + run-file ysh-object "$@" +} + ysh-func() { run-file ysh-func "$@" } diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 77ccea1f23..1874c89305 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -929,6 +929,27 @@ def _EvalSubscript(self, obj, index): raise error.TypeErr(obj, 'Subscript expected Str, List, or Dict', loc.Missing) + def _ChainedLookup(self, obj, current, attr_name): + # type: (Dict_, Dict_, str) -> Optional[value_t] + """Prototype chain lookup. + + Args: + obj: properties we might bind to + current: our location in the prototype chain + """ + val = current.d.get(attr_name) + if val is not None: + # Special bound method logic for objects, but NOT modules + if val.tag() in (value_e.Func, value_e.BuiltinFunc): + return value.BoundFunc(obj, val) + else: + return val + + if current.prototype is not None: + return self._ChainedLookup(obj, current.prototype, attr_name) + + return None + def _EvalDot(self, node, obj): # type: (Attribute, value_t) -> value_t """ obj.attr on RHS or LHS @@ -941,16 +962,25 @@ def _EvalDot(self, node, obj): if case(value_e.Dict): obj = cast(Dict_, UP_obj) attr_name = node.attr_name - try: - result = obj.d[attr_name] - except KeyError: - raise error.Expr('Dict entry %r not found' % attr_name, - node.op) + + # Dict key / normal attribute lookup + result = obj.d.get(attr_name) + if result is not None: + return result + + # Prototype lookup - with special logic for BoundMethod + if obj.prototype is not None: + result = self._ChainedLookup(obj, obj.prototype, attr_name) + if result is not None: + return result + + raise error.Expr('Dict entry %r not found' % attr_name, + node.op) else: raise error.TypeErr(obj, 'Dot operator expected Dict', node.op) - return result + raise AssertionError() def _EvalAttribute(self, node): # type: (Attribute) -> value_t From a146fd95e1e6888b85b1288f3ffce15506c2c839 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 20:11:01 -0400 Subject: [PATCH 130/506] [ysh] Use separate value.Obj for methods, not Dict Dicts have builtin methods, and we don't want them to be confused with user-defined methods. --- builtin/func_misc.py | 20 ++++++++++++++++---- core/value.asdl | 5 +++++ frontend/typed_args.py | 18 +++++++++--------- spec/ysh-object.test.sh | 9 ++++++++- ysh/expr_eval.py | 20 ++++++++++++++++---- 5 files changed, 54 insertions(+), 18 deletions(-) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index b90a4cd1f9..f3d4208a2c 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -6,7 +6,7 @@ from _devbuild.gen.runtime_asdl import (scope_e) from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, - Dict_) + Dict_, Obj) from core import error from core import num @@ -23,7 +23,7 @@ from ysh import expr_eval from ysh import val_ops -from typing import TYPE_CHECKING, Dict, List, cast +from typing import TYPE_CHECKING, Dict, List, Optional, cast if TYPE_CHECKING: from osh import glob_ from osh import split @@ -45,12 +45,24 @@ def __init__(self): def Call(self, rd): # type: (typed_args.Reader) -> value_t - prototype = rd.PosObject() + prototype = rd.PosValue() props = rd.PosDict() rd.Done() + chain = None # type: Optional[Obj] + UP_prototype = prototype + with tagswitch(prototype) as case: + if case(value_e.Null): + pass + elif case(value_e.Obj): + prototype = cast(Obj, UP_prototype) + chain = prototype + else: + raise error.TypeErr(prototype, 'Object() expected Obj or Null', + rd.BlamePos()) + # Opposite order - return Dict_(props, prototype) + return Obj(props, chain) class Len(vm._Callable): diff --git a/core/value.asdl b/core/value.asdl index 255b5092ac..c7dc97f33b 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -60,6 +60,7 @@ module value # prototype is for the attribute lookup chain Dict_ = (Dict[str, value] d, Dict_? prototype) + Obj = (Dict[str, value] d, Obj? prototype) # Commands, words, and expressions from syntax.asdl are evaluated to a VALUE. # value_t instances are stored in state.Mem(). @@ -95,6 +96,10 @@ module value | List(List[value] items) | Dict %Dict_ + # for polymorphism - should replace value.{IO,Module} too + # because they have attributes (functions), methods - not just methods + | Obj %Obj + # CODE types # unevaluated: Eggex, Expr, Template, Command/Block # callable, in separate namespaces: Func, BoundFunc, Proc diff --git a/frontend/typed_args.py b/frontend/typed_args.py index f0fa4585df..8cc5303f04 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -5,7 +5,7 @@ from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, LiteralBlock, command_t, expr_t, Token) from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, - Dict_) + Dict_, Obj) from core import error from core.error import e_usage from frontend import location @@ -271,12 +271,12 @@ def _ToDict(self, val): raise error.TypeErr(val, 'Arg %d should be a Dict' % self.pos_consumed, self.BlamePos()) - def _ToObject(self, val): - # type: (value_t) -> Dict_ - if val.tag() == value_e.Dict: - return cast(Dict_, val) + def _ToObj(self, val): + # type: (value_t) -> Obj + if val.tag() == value_e.Obj: + return cast(Obj, val) - raise error.TypeErr(val, 'Arg %d should be a Dict' % self.pos_consumed, + raise error.TypeErr(val, 'Arg %d should be an Obj' % self.pos_consumed, self.BlamePos()) def _ToPlace(self, val): @@ -412,10 +412,10 @@ def PosDict(self): val = self.PosValue() return self._ToDict(val) - def PosObject(self): - # type: () -> Dict_ + def PosObj(self): + # type: () -> Obj val = self.PosValue() - return self._ToObject(val) + return self._ToObj(val) def PosPlace(self): # type: () -> value.Place diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index b644280279..dd24b931d9 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -6,7 +6,7 @@ func Rect_area(this) { return (this.x * this.y) } -var Rect = {area: Rect_area} +var Rect = Object(null, {area: Rect_area}) var rect1 = Object(Rect, {x: 3, y: 4}) var rect2 = Object(Rect, {x: 10, y: 20}) @@ -22,10 +22,17 @@ var rect2 = Object(Rect, {x: 10, y: 20}) var area1 = rect1.area() var area2 = rect2.area() +pp test_ ([rect1.x, rect1.y]) echo "area1 = $area1" + +pp test_ ([rect2.x, rect2.y]) echo "area2 = $area2" +#pp test_ (rect1.nonexistent) + ## STDOUT: +(List) [3,4] area1 = 12 +(List) [10,20] area2 = 200 ## END diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 1874c89305..dd382b4960 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -46,7 +46,7 @@ ) from _devbuild.gen.value_asdl import (value, value_e, value_t, y_lvalue, y_lvalue_e, y_lvalue_t, IntBox, LeftName, - Dict_) + Dict_, Obj) from core import error from core.error import e_die, e_die_status from core import num @@ -930,7 +930,7 @@ def _EvalSubscript(self, obj, index): loc.Missing) def _ChainedLookup(self, obj, current, attr_name): - # type: (Dict_, Dict_, str) -> Optional[value_t] + # type: (Obj, Obj, str) -> Optional[value_t] """Prototype chain lookup. Args: @@ -960,7 +960,19 @@ def _EvalDot(self, node, obj): UP_obj = obj with tagswitch(obj) as case: if case(value_e.Dict): - obj = cast(Dict_, UP_obj) + obj = cast(Obj, UP_obj) + attr_name = node.attr_name + + # Dict key / normal attribute lookup + result = obj.d.get(attr_name) + if result is not None: + return result + + raise error.Expr('Dict entry %r not found' % attr_name, + node.op) + + elif case(value_e.Obj): + obj = cast(Obj, UP_obj) attr_name = node.attr_name # Dict key / normal attribute lookup @@ -974,7 +986,7 @@ def _EvalDot(self, node, obj): if result is not None: return result - raise error.Expr('Dict entry %r not found' % attr_name, + raise error.Expr('Obj attribute %r not found' % attr_name, node.op) else: From b903b852e6fefba0a40c787ef1857ea7dbd96031 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 21:41:38 -0400 Subject: [PATCH 131/506] [core refactor] Restore value.Dict after value.Obj change --- builtin/error_ysh.py | 6 +++--- builtin/func_hay.py | 6 +++--- builtin/func_misc.py | 13 ++++++------- builtin/hay_ysh.py | 10 +++++----- core/dev.py | 10 +++++----- core/error.py | 6 +++--- core/state.py | 18 +++++++++--------- core/value.asdl | 3 +-- data_lang/j8.py | 11 +++++------ display/pp_value.py | 7 +++---- frontend/typed_args.py | 20 +++----------------- frontend/typed_args_test.py | 10 +++++----- osh/cmd_eval.py | 6 +++--- spec/ysh-json.test.sh | 4 ++-- ysh/expr_eval.py | 10 +++++----- ysh/func_proc.py | 6 +++--- ysh/val_ops.py | 28 +++++++++------------------- 17 files changed, 73 insertions(+), 101 deletions(-) diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index 2f082a044c..e5802892da 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -4,7 +4,7 @@ from _devbuild.gen.id_kind_asdl import Id from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus from _devbuild.gen.syntax_asdl import loc, loc_t, expr, expr_e -from _devbuild.gen.value_asdl import value, value_e, Dict_ +from _devbuild.gen.value_asdl import value, value_e from core import error from core.error import e_die_status, e_usage from core import executor @@ -99,7 +99,7 @@ def Run(self, cmd_val): cmd = rd.RequiredBlock() rd.Done() - error_dict = None # type: Dict_ + error_dict = None # type: value.Dict status = 0 # success by default try: @@ -116,7 +116,7 @@ def Run(self, cmd_val): error_dict = e.ToDict() if error_dict is None: - error_dict = Dict_({'code': num.ToBig(status)}, None) + error_dict = value.Dict({'code': num.ToBig(status)}) # Always set _error self.mem.SetTryError(error_dict) diff --git a/builtin/func_hay.py b/builtin/func_hay.py index 05e03777a6..121aae3ebd 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -3,7 +3,7 @@ from __future__ import print_function from _devbuild.gen.syntax_asdl import source, loc, command_t -from _devbuild.gen.value_asdl import value, Dict_ +from _devbuild.gen.value_asdl import value from builtin import hay_ysh from core import alloc from core import error @@ -106,7 +106,7 @@ def Call(self, rd): cmd = rd.PosCommand() rd.Done() - return Dict_(self._Call(cmd), None) + return value.Dict(self._Call(cmd)) class BlockAsStr(vm._Callable): @@ -147,4 +147,4 @@ def Call(self, rd): # type: (typed_args.Reader) -> value_t # TODO: check args - return Dict_(self._Call(), None) + return value.Dict(self._Call()) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index f3d4208a2c..42a6ef0d4a 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -5,8 +5,7 @@ from __future__ import print_function from _devbuild.gen.runtime_asdl import (scope_e) -from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, - Dict_, Obj) +from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj) from core import error from core import num @@ -84,7 +83,7 @@ def Call(self, rd): return num.ToBig(len(x.items)) elif case(value_e.Dict): - x = cast(Dict_, UP_x) + x = cast(value.Dict, UP_x) return num.ToBig(len(x.d)) elif case(value_e.Str): @@ -298,7 +297,7 @@ def Call(self, rd): it = val_ops.ListIterator(val) elif case(value_e.Dict): - val = cast(Dict_, UP_val) + val = cast(value.Dict, UP_val) it = val_ops.DictIterator(val) elif case(value_e.Range): @@ -337,11 +336,11 @@ def Call(self, rd): with tagswitch(val) as case: if case(value_e.Dict): d = NewDict() # type: Dict[str, value_t] - val = cast(Dict_, UP_val) + val = cast(value.Dict, UP_val) for k, v in iteritems(val.d): d[k] = v - return Dict_(d, None) + return value.Dict(d) elif case(value_e.BashAssoc): d = NewDict() @@ -349,7 +348,7 @@ def Call(self, rd): for k, s in iteritems(val.d): d[k] = value.Str(s) - return Dict_(d, None) + return value.Dict(d) raise error.TypeErr(val, 'dict() expected Dict or BashAssoc', rd.BlamePos()) diff --git a/builtin/hay_ysh.py b/builtin/hay_ysh.py index aff308d4d5..e9bc3ba187 100644 --- a/builtin/hay_ysh.py +++ b/builtin/hay_ysh.py @@ -3,7 +3,7 @@ from _devbuild.gen.option_asdl import option_i from _devbuild.gen.runtime_asdl import (scope_e, HayNode) from _devbuild.gen.syntax_asdl import loc -from _devbuild.gen.value_asdl import (value, value_e, value_t, Dict_) +from _devbuild.gen.value_asdl import (value, value_e, value_t) from asdl import format as fmt from core import alloc @@ -157,7 +157,7 @@ def AppendResult(self, d): UP_children = self.result_stack[-1]['children'] assert UP_children.tag() == value_e.List, UP_children children = cast(value.List, UP_children) - children.items.append(Dict_(d, None)) + children.items.append(value.Dict(d)) def Result(self): # type: () -> Dict[str, value_t] @@ -206,7 +206,7 @@ def Push(self, hay_name): top = self.result_stack[-1] # TODO: Store this more efficiently? See osh/builtin_pure.py children = cast(value.List, top['children']) - last_child = cast(Dict_, children.items[-1]) + last_child = cast(value.Dict, children.items[-1]) self.result_stack.append(last_child.d) #log('> PUSH') @@ -295,7 +295,7 @@ def Run(self, cmd_val): result = self.hay_state.Result() - val = Dict_(result, None) + val = value.Dict(result) self.mem.SetNamed(location.LName(var_name), val, scope_e.LocalOnly) elif action == 'reset': @@ -426,6 +426,6 @@ def Run(self, cmd_val): attrs[name] = cell.val - result['attrs'] = Dict_(attrs, None) + result['attrs'] = value.Dict(attrs) return 0 diff --git a/core/dev.py b/core/dev.py index 68f5ad48d7..d4dbf427cc 100644 --- a/core/dev.py +++ b/core/dev.py @@ -8,7 +8,7 @@ trace_t) from _devbuild.gen.syntax_asdl import assign_op_e, Token from _devbuild.gen.value_asdl import (value, value_e, value_t, sh_lvalue, - sh_lvalue_e, LeftName, Dict_) + sh_lvalue_e, LeftName) from core import error from core import optview @@ -143,7 +143,7 @@ def MaybeDump(self, status): 'var_stack': value.List(self.var_stack), 'argv_stack': value.List(self.argv_stack), 'debug_stack': value.List(self.debug_stack), - 'error': Dict_(self.error, None), + 'error': value.Dict(self.error), 'status': num.ToBig(status), 'pid': num.ToBig(my_pid), } # type: Dict[str, value_t] @@ -153,7 +153,7 @@ def MaybeDump(self, status): # TODO: This should be JSON with unicode replacement char? buf = mylib.BufWriter() - j8.PrintMessage(Dict_(d, None), buf, 2) + j8.PrintMessage(value.Dict(d), buf, 2) json_str = buf.getvalue() try: @@ -346,7 +346,7 @@ def WriteDumps(self): a = value.Str(argv0) c = value.Int(mops.IntWiden(count)) d = {'argv0': a, 'count': c} - metric_argv0.append(Dict_(d, None)) + metric_argv0.append(value.Dict(d)) # Other things we need: the reason for the crash! _ErrorWithLocation is # required I think. @@ -359,7 +359,7 @@ def WriteDumps(self): path = os_path.join(self.out_dir, '%d.argv0.json' % self.this_pid) buf = mylib.BufWriter() - j8.PrintMessage(Dict_(j, None), buf, 2) + j8.PrintMessage(value.Dict(j), buf, 2) json8_str = buf.getvalue() try: diff --git a/core/error.py b/core/error.py index 3ccb7f8eec..a1affda08d 100644 --- a/core/error.py +++ b/core/error.py @@ -2,7 +2,7 @@ from __future__ import print_function from _devbuild.gen.syntax_asdl import loc_e, loc_t, loc -from _devbuild.gen.value_asdl import (value, value_t, value_str, Dict_) +from _devbuild.gen.value_asdl import (value, value_t, value_str) from core import num from mycpp.mylib import NewDict @@ -172,7 +172,7 @@ def __init__(self, status, msg, location, properties=None): self.properties = properties def ToDict(self): - # type: () -> Dict_ + # type: () -> value.Dict d = NewDict() # type: Dict[str, value_t] @@ -186,7 +186,7 @@ def ToDict(self): d['code'] = num.ToBig(self.ExitStatus()) d['message'] = value.Str(self.msg) - return Dict_(d, None) + return value.Dict(d) class AssertionErr(Expr): diff --git a/core/state.py b/core/state.py index 8825e77bf5..d7ff39f144 100644 --- a/core/state.py +++ b/core/state.py @@ -19,7 +19,7 @@ from _devbuild.gen.value_asdl import (value, value_e, value_t, sh_lvalue, sh_lvalue_e, sh_lvalue_t, LeftName, y_lvalue_e, regex_match, regex_match_e, - regex_match_t, RegexMatch, Dict_) + regex_match_t, RegexMatch) from core import error from core.error import e_usage, e_die from core import num @@ -815,7 +815,7 @@ def _DumpVarFrame(frame): # TODO: should we show the object ID here? pass - vars_json[name] = Dict_(cell_json, None) + vars_json[name] = value.Dict(cell_json) return vars_json @@ -1071,7 +1071,7 @@ def __init__(self, mem): last = mem.last_status[-1] mem.last_status.append(last) mem.try_status.append(0) - mem.try_error.append(Dict_({}, None)) + mem.try_error.append(value.Dict({})) # TODO: We should also copy these values! Turn the whole thing into a # frame. @@ -1225,7 +1225,7 @@ def __init__(self, dollar0, argv, arena, debug_stack): # - push-registers builtin self.last_status = [0] # type: List[int] # a stack self.try_status = [0] # type: List[int] # a stack - self.try_error = [Dict_({}, None)] # type: List[Dict_] # a stack + self.try_error = [value.Dict({})] # type: List[value.Dict] # a stack self.pipe_status = [[]] # type: List[List[int]] # stack self.process_sub_status = [[]] # type: List[List[int]] # stack @@ -1271,9 +1271,9 @@ def Dump(self): # type: () -> Tuple[List[value_t], List[value_t], List[value_t]] """Copy state before unwinding the stack.""" var_stack = [ - Dict_(_DumpVarFrame(frame), None) for frame in self.var_stack + value.Dict(_DumpVarFrame(frame)) for frame in self.var_stack ] # type: List[value_t] - argv_stack = [Dict_(frame.Dump(), None) + argv_stack = [value.Dict(frame.Dump()) for frame in self.argv_stack] # type: List[value_t] debug_stack = [] # type: List[value_t] @@ -1308,7 +1308,7 @@ def Dump(self): frame = cast(debug_frame.Main, UP_frame) d = {'type': t_main, 'dollar0': value.Str(frame.dollar0)} - debug_stack.append(Dict_(d, None)) + debug_stack.append(value.Dict(d)) return var_stack, argv_stack, debug_stack def SetLastArgument(self, s): @@ -1376,7 +1376,7 @@ def TryStatus(self): return self.try_status[-1] def TryError(self): - # type: () -> Dict_ + # type: () -> value.Dict return self.try_error[-1] def PipeStatus(self): @@ -1392,7 +1392,7 @@ def SetTryStatus(self, x): self.try_status[-1] = x def SetTryError(self, x): - # type: (Dict_) -> None + # type: (value.Dict) -> None self.try_error[-1] = x def SetPipeStatus(self, x): diff --git a/core/value.asdl b/core/value.asdl index c7dc97f33b..7ff03ef7a8 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -59,7 +59,6 @@ module value | Yes %RegexMatch # prototype is for the attribute lookup chain - Dict_ = (Dict[str, value] d, Dict_? prototype) Obj = (Dict[str, value] d, Obj? prototype) # Commands, words, and expressions from syntax.asdl are evaluated to a VALUE. @@ -94,7 +93,7 @@ module value #| Int(int i) | Float(float f) | List(List[value] items) - | Dict %Dict_ + | Dict(Dict[str, value] d) # for polymorphism - should replace value.{IO,Module} too # because they have attributes (functions), methods - not just methods diff --git a/data_lang/j8.py b/data_lang/j8.py index ce00fa7488..d0347546d5 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -31,8 +31,7 @@ import math from _devbuild.gen.id_kind_asdl import Id, Id_t, Id_str -from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, - Dict_) +from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str) from _devbuild.gen.nil8_asdl import (nvalue, nvalue_t) from asdl import format as fmt @@ -307,7 +306,7 @@ def _PrintList(self, val, level): self.buf.write(']') def _PrintDict(self, val, level): - # type: (Dict_, int) -> None + # type: (value.Dict, int) -> None if len(val.d) == 0: # Special case like Python/JS self.buf.write('{}') @@ -551,7 +550,7 @@ def Print(self, val, level=0): self.visited[heap_id] = FINISHED elif case(value_e.Dict): - val = cast(Dict_, UP_val) + val = cast(value.Dict, UP_val) # Cycle detection, only for containers that can be in cycles heap_id = HeapValueId(val) @@ -940,7 +939,7 @@ def _ParseDict(self): self._Next() if self.tok_id == Id.J8_RBrace: self._Next() - return Dict_(d, None) + return value.Dict(d) k, v = self._ParsePair() d[k] = v @@ -956,7 +955,7 @@ def _ParseDict(self): #log('< Dict') - return Dict_(d, None) + return value.Dict(d) def _ParseList(self): # type: () -> value_t diff --git a/display/pp_value.py b/display/pp_value.py index dad3970cb3..f4bf01eef0 100644 --- a/display/pp_value.py +++ b/display/pp_value.py @@ -8,8 +8,7 @@ import math from _devbuild.gen.pretty_asdl import (doc, Measure, MeasuredDoc) -from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, - Dict_) +from _devbuild.gen.value_asdl import value, value_e, value_t, value_str from data_lang import j8 from data_lang import j8_lite from display.pretty import (_Break, _Concat, _Flat, _Group, _IfFlat, _Indent, @@ -327,7 +326,7 @@ def _YshList(self, vlist): return self._Surrounded("[", self._Tabular(mdocs, ","), "]") def _YshDict(self, vdict): - # type: (Dict_) -> MeasuredDoc + # type: (value.Dict) -> MeasuredDoc if len(vdict.d) == 0: return UText("{}") mdocs = [] # type: List[MeasuredDoc] @@ -434,7 +433,7 @@ def _Value(self, val): return result elif case(value_e.Dict): - vdict = cast(Dict_, val) + vdict = cast(value.Dict, val) heap_id = j8.HeapValueId(vdict) if self.visiting.get(heap_id, False): return _Concat([ diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 8cc5303f04..6c0169d836 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -4,8 +4,7 @@ from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, LiteralBlock, command_t, expr_t, Token) -from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, - Dict_, Obj) +from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch) from core import error from core.error import e_usage from frontend import location @@ -266,19 +265,11 @@ def _ToList(self, val): def _ToDict(self, val): # type: (value_t) -> Dict[str, value_t] if val.tag() == value_e.Dict: - return cast(Dict_, val).d + return cast(value.Dict, val).d raise error.TypeErr(val, 'Arg %d should be a Dict' % self.pos_consumed, self.BlamePos()) - def _ToObj(self, val): - # type: (value_t) -> Obj - if val.tag() == value_e.Obj: - return cast(Obj, val) - - raise error.TypeErr(val, 'Arg %d should be an Obj' % self.pos_consumed, - self.BlamePos()) - def _ToPlace(self, val): # type: (value_t) -> value.Place if val.tag() == value_e.Place: @@ -412,11 +403,6 @@ def PosDict(self): val = self.PosValue() return self._ToDict(val) - def PosObj(self): - # type: () -> Obj - val = self.PosValue() - return self._ToObj(val) - def PosPlace(self): # type: () -> value.Place val = self.PosValue() @@ -567,7 +553,7 @@ def NamedDict(self, param_name, default_): val = self.named_args[param_name] UP_val = val if val.tag() == value_e.Dict: - val = cast(Dict_, UP_val) + val = cast(value.Dict, UP_val) mylib.dict_erase(self.named_args, param_name) return val.d diff --git a/frontend/typed_args_test.py b/frontend/typed_args_test.py index a900188640..a3696f1709 100755 --- a/frontend/typed_args_test.py +++ b/frontend/typed_args_test.py @@ -6,7 +6,7 @@ import unittest from _devbuild.gen.syntax_asdl import ArgList, expr -from _devbuild.gen.value_asdl import value, Dict_ +from _devbuild.gen.value_asdl import value from core import error from core import test_lib from frontend import typed_args # module under test @@ -36,10 +36,10 @@ def testReaderPosArgs(self): value.Str('foo'), value.List([value.Int(1), value.Int(2), value.Int(3)]), - Dict_({ + value.Dict({ 'a': value.Int(0xaa), 'b': value.Int(0xbb) - }, None), + }), value.Float(3.14), value.Int(0xdead), value.Int(0xbeef), @@ -110,10 +110,10 @@ def testReaderKwargs(self): 'numbers': value.List([value.Int(1), value.Int(2), value.Int(3)]), - 'blah': Dict_({ + 'blah': value.Dict({ 'a': value.Int(0xaa), 'b': value.Int(0xbb) - }, None), + }), 'pi': value.Float(3.14), 'a': value.Int(0xdead), 'b': value.Int(0xbeef), diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index ace03b269d..a7360f6d15 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -65,7 +65,7 @@ ) from _devbuild.gen.types_asdl import redir_arg_type_e from _devbuild.gen.value_asdl import (value, value_e, value_t, y_lvalue, - y_lvalue_e, y_lvalue_t, LeftName, Dict_) + y_lvalue_e, y_lvalue_t, LeftName) from core import dev from core import error @@ -743,7 +743,7 @@ def _DoMutation(self, node): obj.items[index] = rval elif case(value_e.Dict): - obj = cast(Dict_, UP_obj) + obj = cast(value.Dict, UP_obj) key = val_ops.ToStr(lval.index, 'Dict index should be Str', loc.Missing) @@ -1154,7 +1154,7 @@ def _DoForEach(self, node): node.keyword) elif case(value_e.Dict): - val = cast(Dict_, UP_val) + val = cast(value.Dict, UP_val) it2 = val_ops.DictIterator(val) if n == 1: diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index 1714ec60bb..2b7ba48699 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -153,14 +153,14 @@ echo '{"age": 42}' > $TMP/foo.txt json read (&x) < $TMP/foo.txt pp cell_ x ## STDOUT: -x = (Cell exported:F readonly:F nameref:F val:(Dict_ d:[Dict age (value.Int i:42)])) +x = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:42)])) ## END #### json read at end of pipeline (relies on lastpipe) echo '{"age": 43}' | json read (&y) pp cell_ y ## STDOUT: -y = (Cell exported:F readonly:F nameref:F val:(Dict_ d:[Dict age (value.Int i:43)])) +y = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:43)])) ## END #### invalid JSON diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index dd382b4960..6aec3bea66 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -46,7 +46,7 @@ ) from _devbuild.gen.value_asdl import (value, value_e, value_t, y_lvalue, y_lvalue_e, y_lvalue_t, IntBox, LeftName, - Dict_, Obj) + Obj) from core import error from core.error import e_die, e_die_status from core import num @@ -239,7 +239,7 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): loc.Missing) elif case(value_e.Dict): - obj = cast(Dict_, UP_obj) + obj = cast(value.Dict, UP_obj) index = -1 # silence C++ warning key = val_ops.ToStr(lval.index, 'Dict index should be Str', @@ -268,7 +268,7 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): obj.items[index] = new_val_ elif case(value_e.Dict): - obj = cast(Dict_, UP_obj) + obj = cast(value.Dict, UP_obj) obj.d[key] = new_val_ else: @@ -913,7 +913,7 @@ def _EvalSubscript(self, obj, index): loc.Missing) elif case(value_e.Dict): - obj = cast(Dict_, UP_obj) + obj = cast(value.Dict, UP_obj) if index.tag() != value_e.Str: raise error.TypeErr(index, 'Dict index expected Str', loc.Missing) @@ -1221,7 +1221,7 @@ def _EvalExpr(self, node): loc.Missing) d[k] = values[i] - return Dict_(d, None) + return value.Dict(d) elif case(expr_e.ListComp): e_die_status( diff --git a/ysh/func_proc.py b/ysh/func_proc.py index de185d002d..27d35c1da8 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -10,7 +10,7 @@ NamedArg, Func, loc, ArgList, expr, expr_e, expr_t) from _devbuild.gen.value_asdl import (value, value_e, value_t, ProcDefaults, - LeftName, Dict_) + LeftName) from core import error from core.error import e_die @@ -168,7 +168,7 @@ def _EvalNamedArgs(expr_ev, named_exprs): if val.tag() != value_e.Dict: raise error.TypeErr(val, 'Spread expected a Dict', val_expr.left) - named_args.update(cast(Dict_, val).d) + named_args.update(cast(value.Dict, val).d) else: val = expr_ev.EvalExpr(n.value, n.name) name = lexer.TokenVal(n.name) @@ -403,7 +403,7 @@ def _BindNamed( rest = group.rest_of if rest: lval = LeftName(rest.name, rest.blame_tok) - mem.SetLocalName(lval, Dict_(named_args, None)) + mem.SetLocalName(lval, value.Dict(named_args)) else: num_args = len(named_args) num_params = len(group.params) diff --git a/ysh/val_ops.py b/ysh/val_ops.py index 66cd3843b1..9da110afce 100644 --- a/ysh/val_ops.py +++ b/ysh/val_ops.py @@ -4,8 +4,7 @@ from _devbuild.gen.syntax_asdl import loc, loc_t, command_t from _devbuild.gen.value_asdl import (value, value_e, value_t, eggex_ops, - eggex_ops_t, regex_match, RegexMatch, - Dict_) + eggex_ops_t, regex_match, RegexMatch) from core import error from core.error import e_die from display import ui @@ -24,15 +23,6 @@ if TYPE_CHECKING: from core import state -if 0: - - def PlainDict(d): - # type: (Dict[str, value_t]) -> Dict_ - """ - Shorthand for "plain old data", i.e. data without behavior - """ - return Dict_(d, None) - def ToInt(val, msg, blame_loc): # type: (value_t, str, loc_t) -> int @@ -78,7 +68,7 @@ def ToDict(val, msg, blame_loc): # type: (value_t, str, loc_t) -> Dict[str, value_t] UP_val = val if val.tag() == value_e.Dict: - val = cast(Dict_, UP_val) + val = cast(value.Dict, UP_val) return val.d raise error.TypeErr(val, msg, blame_loc) @@ -309,7 +299,7 @@ class DictIterator(Iterator): """ for x in (mydict) { """ def __init__(self, val): - # type: (Dict_) -> None + # type: (value.Dict) -> None Iterator.__init__(self) # TODO: Don't materialize these Lists @@ -374,7 +364,7 @@ def ToBool(val): return len(val.items) > 0 elif case(value_e.Dict): - val = cast(Dict_, UP_val) + val = cast(value.Dict, UP_val) return len(val.d) > 0 else: @@ -443,8 +433,8 @@ def ExactlyEqual(left, right, blame_loc): return True elif case(value_e.BashAssoc): - left = cast(Dict_, UP_left) - right = cast(Dict_, UP_right) + left = cast(value.Dict, UP_left) + right = cast(value.Dict, UP_right) if len(left.d) != len(right.d): return False @@ -455,8 +445,8 @@ def ExactlyEqual(left, right, blame_loc): return True elif case(value_e.Dict): - left = cast(Dict_, UP_left) - right = cast(Dict_, UP_right) + left = cast(value.Dict, UP_left) + right = cast(value.Dict, UP_right) if len(left.d) != len(right.d): return False @@ -481,7 +471,7 @@ def Contains(needle, haystack): UP_haystack = haystack with tagswitch(haystack) as case: if case(value_e.Dict): - haystack = cast(Dict_, UP_haystack) + haystack = cast(value.Dict, UP_haystack) s = ToStr(needle, "LHS of 'in' should be Str", loc.Missing) return s in haystack.d From 20c151dbb25a47b515663b19ea28615de09e757d Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 22:14:58 -0400 Subject: [PATCH 132/506] [prebuilt] Rebuild files --- prebuilt/core/error.mycpp.cc | 5 ++--- prebuilt/core/error.mycpp.h | 2 +- prebuilt/frontend/args.mycpp.cc | 7 +++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/prebuilt/core/error.mycpp.cc b/prebuilt/core/error.mycpp.cc index 1a2b32255e..cee7d3cf20 100644 --- a/prebuilt/core/error.mycpp.cc +++ b/prebuilt/core/error.mycpp.cc @@ -99,7 +99,6 @@ using syntax_asdl::loc; using value_asdl::value; using value_asdl::value_t; using value_asdl::value_str; -using value_asdl::Dict_; BigStr* _ValType(value_asdl::value_t* val) { StackRoot _root0(&val); @@ -159,7 +158,7 @@ Structured::Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Di this->properties = properties; } -value_asdl::Dict_* Structured::ToDict() { +value::Dict* Structured::ToDict() { Dict* d = nullptr; StackRoot _root0(&d); @@ -169,7 +168,7 @@ value_asdl::Dict_* Structured::ToDict() { } d->set(str6, num::ToBig(this->ExitStatus())); d->set(str7, Alloc(this->msg)); - return Alloc(d, nullptr); + return Alloc(d); } AssertionErr::AssertionErr(BigStr* msg, syntax_asdl::loc_t* location) : ::error::Expr(msg, location) { diff --git a/prebuilt/core/error.mycpp.h b/prebuilt/core/error.mycpp.h index d3cfc60554..584979c783 100644 --- a/prebuilt/core/error.mycpp.h +++ b/prebuilt/core/error.mycpp.h @@ -187,7 +187,7 @@ class Expr : public ::error::FatalRuntime { class Structured : public ::error::FatalRuntime { public: Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties = nullptr); - value_asdl::Dict_* ToDict(); + value::Dict* ToDict(); Dict* properties; diff --git a/prebuilt/frontend/args.mycpp.cc b/prebuilt/frontend/args.mycpp.cc index b19b043a0c..e8be119da6 100644 --- a/prebuilt/frontend/args.mycpp.cc +++ b/prebuilt/frontend/args.mycpp.cc @@ -387,7 +387,7 @@ class Expr : public ::error::FatalRuntime { class Structured : public ::error::FatalRuntime { public: Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties = nullptr); - value_asdl::Dict_* ToDict(); + value::Dict* ToDict(); Dict* properties; @@ -1410,7 +1410,6 @@ using syntax_asdl::loc; using value_asdl::value; using value_asdl::value_t; using value_asdl::value_str; -using value_asdl::Dict_; BigStr* _ValType(value_asdl::value_t* val) { StackRoot _root0(&val); @@ -1470,7 +1469,7 @@ Structured::Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Di this->properties = properties; } -value_asdl::Dict_* Structured::ToDict() { +value::Dict* Structured::ToDict() { Dict* d = nullptr; StackRoot _root0(&d); @@ -1480,7 +1479,7 @@ value_asdl::Dict_* Structured::ToDict() { } d->set(str62, num::ToBig(this->ExitStatus())); d->set(str63, Alloc(this->msg)); - return Alloc(d, nullptr); + return Alloc(d); } AssertionErr::AssertionErr(BigStr* msg, syntax_asdl::loc_t* location) : ::error::Expr(msg, location) { From b198701ede2805fbf3b7bec73514ce1059d822c2 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 22:26:55 -0400 Subject: [PATCH 133/506] [builtin/pp] Support value.Obj TODO: - The cycle detection is a little messed up - The pretty printer should also support it - pp (value) --- data_lang/j8.py | 64 ++++++++++++++++++++++++++++++----- display/ui.py | 2 +- spec/ysh-object.test.sh | 71 +++++++++++++++++++++++++++++++++++++++ spec/ysh-printing.test.sh | 20 +++++------ ysh/expr_eval.py | 15 ++++++++- 5 files changed, 150 insertions(+), 22 deletions(-) diff --git a/data_lang/j8.py b/data_lang/j8.py index d0347546d5..dbb7783ec0 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -31,7 +31,7 @@ import math from _devbuild.gen.id_kind_asdl import Id, Id_t, Id_str -from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str) +from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj) from _devbuild.gen.nil8_asdl import (nvalue, nvalue_t) from asdl import format as fmt @@ -305,16 +305,15 @@ def _PrintList(self, val, level): self._BracketIndent(level) self.buf.write(']') - def _PrintDict(self, val, level): - # type: (value.Dict, int) -> None - - if len(val.d) == 0: # Special case like Python/JS + def _PrintMapping(self, d, level): + # type: (Dict[str, value_t], int) -> None + if len(d) == 0: # Special case like Python/JS self.buf.write('{}') else: self.buf.write('{') self._MaybeNewline() i = 0 - for k, v in iteritems(val.d): + for k, v in iteritems(d): if i != 0: self.buf.write(',') self._MaybeNewline() @@ -334,6 +333,19 @@ def _PrintDict(self, val, level): self._BracketIndent(level) self.buf.write('}') + def _PrintDict(self, val, level): + # type: (value.Dict, int) -> None + self._PrintMapping(val.d, level) + + def _PrintObj(self, val, level): + # type: (Obj, int) -> None + + self._PrintMapping(val.d, level) + + if val.prototype: + self.buf.write(' ==> ') + self._PrintObj(val.prototype, level) + def _PrintBashPrefix(self, type_str, level): # type: (str, int) -> None @@ -576,6 +588,41 @@ def Print(self, val, level=0): self._PrintDict(val, level) self.visited[heap_id] = FINISHED + elif case(value_e.Obj): + val = cast(Obj, UP_val) + + if not (self.options & SHOW_NON_DATA): + raise error.Encode("Can't encode value of type Obj") + + # Cycle detection, only for containers that can be in cycles + heap_id = HeapValueId(val) + + node_state = self.visited.get(heap_id, UNSEEN) + if node_state == FINISHED: + # Print it AGAIN. We print a JSON tree, which means we can + # visit and print nodes MANY TIMES, as long as they're not + # in a cycle. + self._PrintObj(val, level) + return + if node_state == EXPLORING: + if self.options & SHOW_CYCLES: + self.buf.write('{ -->%s }' % ValueIdString(val)) + return + else: + # node.js prints which key closes the cycle + raise error.Encode( + "Can't encode Obj%s in object cycle" % + ValueIdString(val)) + + # TODO: cycle detection is a bit wrong, I think because the + # properties are a Dict[str, value_t], not something with an + # identity + # + # This is only used for pp test_, because SHOW_NON_DATA. + self.visited[heap_id] = EXPLORING + self._PrintObj(val, level) + self.visited[heap_id] = FINISHED + elif case(value_e.SparseArray): val = cast(value.SparseArray, UP_val) self._PrintSparseArray(val, level) @@ -594,8 +641,9 @@ def Print(self, val, level=0): # Similar to = operator, ui.DebugPrint() # TODO: that prints value.Range in a special way ysh_type = ValType(val) - id_str = ValueIdString(val) - self.buf.write('<%s%s>' % (ysh_type, id_str)) + # Don't show ID in 'pp test_' + #id_str = ValueIdString(val) + self.buf.write('<%s>' % ysh_type) else: raise error.Encode("Can't serialize object of type %s" % ValType(val)) diff --git a/display/ui.py b/display/ui.py index 9a91ed1346..ad5503d497 100644 --- a/display/ui.py +++ b/display/ui.py @@ -550,7 +550,7 @@ def TypeNotPrinted(val): # type: (value_t) -> bool return val.tag() in (value_e.Null, value_e.Bool, value_e.Int, value_e.Float, value_e.Str, value_e.List, - value_e.Dict) + value_e.Dict, value_e.Obj) def _GetMaxWidth(): diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index dd24b931d9..709ad6a222 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -1,4 +1,5 @@ ## our_shell: ysh +## oils_failures_allowed: 2 #### Object() creates prototype chain @@ -36,3 +37,73 @@ area1 = 12 (List) [10,20] area2 = 200 ## END + +#### can't encode objects as JSON + +var Rect = Object(null, {}) + +json write (Rect) +echo 'nope' + +## status: 1 +## STDOUT: +## END + +#### pretty printing of cycles + +var d = {k: 42} +setvar d.cycle = d + +pp test_ (d) + +var o = Object(null, d) + +pp test_ (o) + +var o2 = Object(o, {z: 99}) + +pp test_ (o2) + +## STDOUT: +## END + +#### setvar obj.attr + +func Rect_area(this) { + return (this.x * this.y) +} + +var Rect = Object(null, {area: Rect_area}) + +var rect1 = Object(Rect, {x: 3, y: 4}) + +pp test_ (rect1) + +# Right now it's not mutable +setvar rect1.x = 99 + +pp test_ (rect1) + +## STDOUT: +(Obj) {"x":3,"y":4} ==> {"area":} +## END + +#### Can all builtin methods with s.upper() + +var s = 'foo' +var x = s.upper() +var y = "--$[x.lower()]" + +pp test_ (x) +pp test_ (y) + +# TODO: +# keys(d) values(d) instead of d.keys() and d.values() +# +# mutating methods are OK? +# call d->inc(x) + +## STDOUT: +(Str) "FOO" +(Str) "--foo" +## END diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index 52330e4acd..99622cfce6 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -45,19 +45,15 @@ pp value ({k: x}) echo -remove-addr() { - sed 's/0x[0-9a-f]\+/0x---/' -} - -pp test_ (x) | remove-addr -pp test_ ({k: x}) | remove-addr +pp test_ (x) +pp test_ ({k: x}) ## STDOUT: (Range 1 .. 100) (Dict) {k: (Range 1 .. 100)} - -(Dict) {"k":} + +(Dict) {"k":} ## END @@ -76,15 +72,15 @@ pp value ({k: pat}) | remove-addr echo -pp test_ (pat) | remove-addr -pp test_ ({k: pat}) | remove-addr +pp test_ (pat) +pp test_ ({k: pat}) ## STDOUT: (Dict) {k: } - -(Dict) {"k":} + +(Dict) {"k":} ## END #### SparseArray, new representation for bash array diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 6aec3bea66..ef2619c7a5 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -990,7 +990,20 @@ def _EvalDot(self, node, obj): node.op) else: - raise error.TypeErr(obj, 'Dot operator expected Dict', node.op) + # Method lookup on builtin types. + # They don't have attributes or prototype chains -- we only + # have a flat dict. + type_methods = self.methods.get(obj.tag()) + name = node.attr_name + vm_callable = (type_methods.get(name) + if type_methods is not None else None) + if vm_callable: + func_val = value.BuiltinFunc(vm_callable) + return value.BoundFunc(obj, func_val) + + raise error.TypeErrVerbose( + 'Method %r does not exist on builtin type %s' % + (name, ui.ValType(obj)), node.attr) raise AssertionError() From b491e104a0556bf6fc7fd28994fb726009cc4831 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 8 Aug 2024 23:06:34 -0400 Subject: [PATCH 134/506] [ysh] Implement operations on Obj - setvar obj.attr = 99 - setvar obj.attr += 3 - dict() to copy --- builtin/func_misc.py | 30 ++++++++++-- core/shell.py | 15 +++++- osh/cmd_eval.py | 11 ++++- spec/ysh-object.test.sh | 103 +++++++++++++++++++++++++++++----------- ysh/expr_eval.py | 34 ++++++++++++- 5 files changed, 157 insertions(+), 36 deletions(-) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 42a6ef0d4a..a590955fd9 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -31,10 +31,10 @@ class Object(vm._Callable): - """ - Create an object. The order of params follows JavaScript's Object.create() + """Create a value.Obj - var obj = Object(prototype, props) + The order of params follows JavaScript's Object.create(): + var obj = Object(prototype, props) """ def __init__(self): @@ -64,6 +64,20 @@ def Call(self, rd): return Obj(props, chain) +class Prototype(vm._Callable): + """Get an object's prototype.""" + + def __init__(self): + # type: () -> None + pass + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + + # TODO + return value.Null + + class Len(vm._Callable): def __init__(self): @@ -342,6 +356,14 @@ def Call(self, rd): return value.Dict(d) + elif case(value_e.Obj): + d = NewDict() + val = cast(Obj, UP_val) + for k, v in iteritems(val.d): + d[k] = v + + return value.Dict(d) + elif case(value_e.BashAssoc): d = NewDict() val = cast(value.BashAssoc, UP_val) @@ -350,7 +372,7 @@ def Call(self, rd): return value.Dict(d) - raise error.TypeErr(val, 'dict() expected Dict or BashAssoc', + raise error.TypeErr(val, 'dict() expected Dict, Obj, or BashAssoc', rd.BlamePos()) diff --git a/core/shell.py b/core/shell.py index fbc22ec71f..31fe05cee6 100644 --- a/core/shell.py +++ b/core/shell.py @@ -760,8 +760,13 @@ def Main( 'fullMatch': None, } methods[value_e.Dict] = { - 'get': method_dict.Get(), + # TODO: __mut_erase 'erase': method_dict.Erase(), + + # Dict.get() + # Dict.keys() + # Dict.values() + 'get': method_dict.Get(), 'keys': method_dict.Keys(), 'values': method_dict.Values(), @@ -778,6 +783,7 @@ def Main( 'accum': None, } methods[value_e.List] = { + # TODO: __mut_{reverse,append,extend,pop,insert,remove} 'reverse': method_list.Reverse(), 'append': method_list.Append(), 'extend': method_list.Extend(), @@ -798,6 +804,10 @@ def Main( } methods[value_e.IO] = { + # TODO: io.eval() or io->eval()? + # We are not mutating the object itself - we are mutating the system. + # That is already captured by io, so let's make it io.eval(). + # io->eval(myblock) is the functional version of eval (myblock) # Should we also have expr->eval() instead of evalExpr? 'eval': method_io.Eval(cmd_ev), @@ -810,6 +820,8 @@ def Main( } methods[value_e.Place] = { + # __mut_setValue() + # instead of setplace keyword 'setValue': method_other.SetValue(mem), } @@ -846,6 +858,7 @@ def Main( _SetGlobalFunc(mem, 'evalExpr', func_misc.EvalExpr(expr_ev)) _SetGlobalFunc(mem, 'Object', func_misc.Object()) + _SetGlobalFunc(mem, 'prototype', func_misc.Prototype()) # type conversions _SetGlobalFunc(mem, 'bool', func_misc.Bool()) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index a7360f6d15..bdb40f6136 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -65,7 +65,7 @@ ) from _devbuild.gen.types_asdl import redir_arg_type_e from _devbuild.gen.value_asdl import (value, value_e, value_t, y_lvalue, - y_lvalue_e, y_lvalue_t, LeftName) + y_lvalue_e, y_lvalue_t, LeftName, Obj) from core import dev from core import error @@ -749,9 +749,16 @@ def _DoMutation(self, node): loc.Missing) obj.d[key] = rval + elif case(value_e.Obj): + obj = cast(Obj, UP_obj) + key = val_ops.ToStr(lval.index, + 'Obj index should be Str', + loc.Missing) + obj.d[key] = rval + else: raise error.TypeErr( - obj, "obj[index] expected List or Dict", + obj, "obj[index] expected List, Dict, or Obj", loc.Missing) else: diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index 709ad6a222..89405d1b3f 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -1,5 +1,5 @@ ## our_shell: ysh -## oils_failures_allowed: 2 +## oils_failures_allowed: 3 #### Object() creates prototype chain @@ -38,6 +38,65 @@ area1 = 12 area2 = 200 ## END +#### prototype() + +func Rect_area(this) { + return (this.x * this.y) +} + +var Rect = Object(null, {area: Rect_area}) + +var obj = Object(Rect, {x: 3, y: 4}) + +pp test_ (prototype(Rect)) +pp test_ (prototype(obj)) + +## STDOUT: +## END + +#### Copy to Dict with dict(), and mutate + +var rect = Object(null, {x: 3, y: 4}) +var d = dict(rect) + +pp test_ (rect) +pp test_ (d) + +# Right now, object attributes aren't mutable! Could change this. +# +setvar rect.x = 99 +setvar d.x = 100 + +pp test_ (rect) +pp test_ (d) +## STDOUT: +(Obj) {"x":3,"y":4} +(Dict) {"x":3,"y":4} +(Obj) {"x":99,"y":4} +(Dict) {"x":100,"y":4} +## END + +#### setvar obj.attr = and += and ... + +var rect = Object(null, {x: 3, y: 4}) +pp test_ (rect) + +setvar rect.y = 99 +pp test_ (rect) + +setvar rect.y += 3 +pp test_ (rect) + +setvar rect.x *= 5 +pp test_ (rect) + +## STDOUT: +(Obj) {"x":3,"y":4} +(Obj) {"x":3,"y":99} +(Obj) {"x":3,"y":102} +(Obj) {"x":15,"y":102} +## END + #### can't encode objects as JSON var Rect = Object(null, {}) @@ -67,27 +126,6 @@ pp test_ (o2) ## STDOUT: ## END -#### setvar obj.attr - -func Rect_area(this) { - return (this.x * this.y) -} - -var Rect = Object(null, {area: Rect_area}) - -var rect1 = Object(Rect, {x: 3, y: 4}) - -pp test_ (rect1) - -# Right now it's not mutable -setvar rect1.x = 99 - -pp test_ (rect1) - -## STDOUT: -(Obj) {"x":3,"y":4} ==> {"area":} -## END - #### Can all builtin methods with s.upper() var s = 'foo' @@ -97,13 +135,24 @@ var y = "--$[x.lower()]" pp test_ (x) pp test_ (y) -# TODO: -# keys(d) values(d) instead of d.keys() and d.values() -# +## STDOUT: +(Str) "FOO" +(Str) "--foo" +## END + + +#### Dict.keys(d), Dict.values(d), Dict.get(d, key) + +var d = {a: 42, b: 99} + +pp test_ (Dict.keys(d)) +pp test_ (Dict.values(d)) + +pp test_ (Dict.get(d, 'key', 'default')) + # mutating methods are OK? # call d->inc(x) ## STDOUT: -(Str) "FOO" -(Str) "--foo" ## END + diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index ef2619c7a5..66a6591d34 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -242,14 +242,27 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): obj = cast(value.Dict, UP_obj) index = -1 # silence C++ warning key = val_ops.ToStr(lval.index, - 'Dict index should be Str', + 'Dict key should be Str', loc.Missing) try: lhs_val_ = obj.d[key] except KeyError: - raise error.Expr('Dict entry not found: %r' % key, + raise error.Expr('Dict key not found: %r' % key, loc.Missing) + elif case(value_e.Obj): + obj = cast(Obj, UP_obj) + index = -1 # silence C++ warning + key = val_ops.ToStr(lval.index, + 'Obj attribute should be Str', + loc.Missing) + try: + lhs_val_ = obj.d[key] + except KeyError: + raise error.Expr( + 'Obj attribute not found: %r' % key, + loc.Missing) + else: raise error.TypeErr( obj, "obj[index] expected List or Dict", @@ -271,6 +284,13 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): obj = cast(value.Dict, UP_obj) obj.d[key] = new_val_ + elif case(value_e.Obj): + obj = cast(Obj, UP_obj) + obj.d[key] = new_val_ + + else: + raise AssertionError() + else: raise AssertionError() @@ -1014,6 +1034,16 @@ def _EvalAttribute(self, node): UP_o = o with switch(node.op.id) as case: + # TODO: + # -> add value.Obj rule - mut_mymethod() + # then change value.List to have __mut_append()? + # this means you can no longer do call foo => end(), which we want + # + # => eventually remove method lookup - it's only the chaining + # operator + # s => upper() => strip() might be OK though + # versus s.upper().strip() + # Right now => is a synonym for -> # Later we may enforce that => is pure, and -> is for mutation and # I/O. From 4023cbc2dce2db127191f384c26a7c2a66249a34 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 9 Aug 2024 11:10:01 -0400 Subject: [PATCH 135/506] [spec/hay-meta] Failing test case for scope issue Samuel found Thread on #oil-discuss-public on Zulip I think we can fix the problem with the new ctx_Eval(...) from Aidan, instead of ctx_Temp(...) --- spec/hay-meta.test.sh | 69 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/spec/hay-meta.test.sh b/spec/hay-meta.test.sh index 58cdc61e88..702c400350 100644 --- a/spec/hay-meta.test.sh +++ b/spec/hay-meta.test.sh @@ -1,7 +1,9 @@ +## oils_failures_allowed: 1 + # Hay Metaprogramming #### Conditional Inside Blocks -shopt --set oil:all +shopt --set ysh:all hay define Rule @@ -35,7 +37,7 @@ EOF #### Conditional Outside Block -shopt --set oil:all +shopt --set ysh:all hay define Rule @@ -64,7 +66,7 @@ EOF #### Iteration Inside Block -shopt --set oil:all +shopt --set ysh:all hay define Rule @@ -98,7 +100,7 @@ EOF #### Iteration Outside Block -shopt --set oil:all +shopt --set ysh:all hay define Rule @@ -126,8 +128,63 @@ EOF ## END +#### Iteration outside Hay node - example from Samuel + +shopt --set ysh:all + +hay define task + +# BUG with hay eval! +hay eval :result { + var all_hellos = [ "You", "lovely", "people", "Chuck Norris" ] + for hello in (all_hellos) { + task "Say $hello" { + var extend = "Say Hello" + var overrides = { + WORD: hello + } + } + } +} + +json write (result) | jq '.children[].attrs' > actual.txt + +#json write (_hay()) | jq '.children[].attrs' > actual.txt + +diff -u - actual.txt < Date: Fri, 9 Aug 2024 12:40:22 -0600 Subject: [PATCH 136/506] [builtin/eval] Set/restore new vars rather than pushing a new frame (#2047) * Add test for eval scoping in local and global scopes * Use ctx_Eval instead of ctx_Shvar everywhere * Use NewDict() --- builtin/method_str.py | 13 ++++----- builtin/pure_ysh.py | 53 +++++------------------------------ core/state.py | 30 +++++++++++++++----- spec/ysh-builtin-eval.test.sh | 20 +++++++++++++ 4 files changed, 55 insertions(+), 61 deletions(-) diff --git a/builtin/method_str.py b/builtin/method_str.py index 07b2e46c6c..f864caf0b9 100644 --- a/builtin/method_str.py +++ b/builtin/method_str.py @@ -5,7 +5,6 @@ from _devbuild.gen.syntax_asdl import loc_t from _devbuild.gen.value_asdl import (value, value_e, value_t, eggex_ops, eggex_ops_t, RegexMatch) -from builtin import pure_ysh from core import error from core import state from core import vm @@ -20,7 +19,7 @@ import libc from libc import REG_NOTBOL -from typing import cast, List, Tuple +from typing import cast, Dict, List, Tuple _ = log @@ -423,7 +422,7 @@ def Call(self, rd): # Collect captures arg0 = None # type: str argv = [] # type: List[str] - named_vars = [] # type: List[Tuple[str, value_t]] + named_vars = {} # type: Dict[str, value_t] num_groups = len(indices) / 2 for group in xrange(num_groups): start = indices[2 * group] @@ -454,15 +453,13 @@ def Call(self, rd): if group != 0: name = eggex_val.capture_names[group - 2] if name is not None: - named_vars.append((name, val)) + named_vars[name] = val if subst_str: s = subst_str.s if subst_expr: - with state.ctx_Eval(self.mem, arg0, argv, None): - with pure_ysh.ctx_Shvar(self.mem, named_vars): - s = self.EvalSubstExpr(subst_expr, - rd.LeftParenToken()) + with state.ctx_Eval(self.mem, arg0, argv, named_vars): + s = self.EvalSubstExpr(subst_expr, rd.LeftParenToken()) assert s is not None start = indices[0] diff --git a/builtin/pure_ysh.py b/builtin/pure_ysh.py index 4a9c483c63..810ed19c96 100644 --- a/builtin/pure_ysh.py +++ b/builtin/pure_ysh.py @@ -3,63 +3,24 @@ """ from __future__ import print_function -from _devbuild.gen.runtime_asdl import (cmd_value, scope_e) +from _devbuild.gen.runtime_asdl import cmd_value from _devbuild.gen.syntax_asdl import command_t, loc, loc_t -from _devbuild.gen.value_asdl import (value, value_e, value_t, LeftName) +from _devbuild.gen.value_asdl import value, value_e, value_t from core import error from core import state from core import vm from frontend import flag_util -from frontend import location from frontend import typed_args from mycpp import mylib -from mycpp.mylib import tagswitch +from mycpp.mylib import tagswitch, NewDict -from typing import TYPE_CHECKING, cast, Any, Dict, List, Tuple +from typing import TYPE_CHECKING, cast, Any, Dict, List if TYPE_CHECKING: from display import ui from osh.cmd_eval import CommandEvaluator -class ctx_Shvar(object): - """For shvar LANG=C _ESCAPER=posix-sh-word _DIALECT=ninja.""" - - def __init__(self, mem, pairs): - # type: (state.Mem, List[Tuple[str, value_t]]) -> None - #log('pairs %s', pairs) - self.mem = mem - self.restore = [] # type: List[Tuple[LeftName, value_t]] - self._Push(pairs) - - def __enter__(self): - # type: () -> None - pass - - def __exit__(self, type, value, traceback): - # type: (Any, Any, Any) -> None - self._Pop() - - # Note: _Push and _Pop are separate methods because the C++ translation - # doesn't like when they are inline in __init__ and __exit__. - def _Push(self, pairs): - # type: (List[Tuple[str, value_t]]) -> None - for name, v in pairs: - lval = location.LName(name) - # LocalOnly because we are only overwriting the current scope - old_val = self.mem.GetValue(name, scope_e.LocalOnly) - self.restore.append((lval, old_val)) - self.mem.SetNamed(lval, v, scope_e.LocalOnly) - - def _Pop(self): - # type: () -> None - for lval, old_val in self.restore: - if old_val.tag() == value_e.Undef: - self.mem.Unset(lval, scope_e.LocalOnly) - else: - self.mem.SetNamed(lval, old_val, scope_e.LocalOnly) - - class Shvar(vm._Builtin): def __init__(self, mem, search_path, cmd_ev): @@ -80,7 +41,7 @@ def Run(self, cmd_val): # But should there be a whitelist? raise error.Usage('expected a block', loc.Missing) - pairs = [] # type: List[Tuple[str, value_t]] + vars = NewDict() # type: Dict[str, value_t] args, arg_locs = arg_r.Rest2() if len(args) == 0: raise error.Usage('Expected name=value', loc.Missing) @@ -90,13 +51,13 @@ def Run(self, cmd_val): if s is None: raise error.Usage('Expected name=value', arg_locs[i]) v = value.Str(s) # type: value_t - pairs.append((name, v)) + vars[name] = v # Important fix: shvar PATH='' { } must make all binaries invisible if name == 'PATH': self.search_path.ClearCache() - with ctx_Shvar(self.mem, pairs): + with state.ctx_Eval(self.mem, None, None, vars): unused = self.cmd_ev.EvalCommand(cmd) return 0 diff --git a/core/state.py b/core/state.py index d7ff39f144..008e24e977 100644 --- a/core/state.py +++ b/core/state.py @@ -1128,7 +1128,7 @@ def _MakeArgvCell(argv): class ctx_Eval(object): - """Push temporary variable frame and override $0, $1, $2, etc.""" + """Push temporary set of variables, $0, $1, $2, etc.""" def __init__(self, mem, dollar0, pos_args, vars): # type: (Mem, Optional[str], Optional[List[str]], Optional[Dict[str, value_t]]) -> None @@ -1148,11 +1148,8 @@ def __init__(self, mem, dollar0, pos_args, vars): mem.argv_stack.append(_ArgFrame(pos_args)) if vars is not None: - frame = {} # type: Dict[str, Cell] - for name in vars: - frame[name] = Cell(False, False, False, vars[name]) - - mem.var_stack.append(frame) + self.restore = [] # type: List[Tuple[LeftName, value_t]] + self._Push(vars) def __enter__(self): # type: () -> None @@ -1161,7 +1158,7 @@ def __enter__(self): def __exit__(self, type, value_, traceback): # type: (Any, Any, Any) -> None if self.vars is not None: - self.mem.var_stack.pop() + self._Pop() if self.pos_args is not None: self.mem.argv_stack.pop() @@ -1169,6 +1166,25 @@ def __exit__(self, type, value_, traceback): if self.dollar0 is not None: self.mem.SetLocalName(self.dollar0_lval, value.Undef) + # Note: _Push and _Pop are separate methods because the C++ translation + # doesn't like when they are inline in __init__ and __exit__. + def _Push(self, vars): + # type: (Dict[str, value_t]) -> None + for name in vars: + lval = location.LName(name) + # LocalOnly because we are only overwriting the current scope + old_val = self.mem.GetValue(name, scope_e.LocalOnly) + self.restore.append((lval, old_val)) + self.mem.SetNamed(lval, vars[name], scope_e.LocalOnly) + + def _Pop(self): + # type: () -> None + for lval, old_val in self.restore: + if old_val.tag() == value_e.Undef: + self.mem.Unset(lval, scope_e.LocalOnly) + else: + self.mem.SetNamed(lval, old_val, scope_e.LocalOnly) + class Mem(object): """For storing variables. diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 2fe129b91d..0db5ec8741 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -278,6 +278,26 @@ pp test_ (vars) eval (^(true), pos_args=[1, 2, 3]) ## status: 3 +#### eval with vars follows same scoping as without +proc local-scope { + var myVar = "foo" + eval (^(echo $myVar), vars={ someOtherVar: "bar" }) + eval (^(echo $myVar)) +} + +# In global scope +var myVar = "baz" +eval (^(echo $myVar), vars={ someOtherVar: "bar" }) +eval (^(echo $myVar)) + +local-scope +## STDOUT: +baz +baz +foo +foo +## END + #### eval 'mystring' vs. eval (myblock) eval 'echo plain' From ad56419948918615005630a02467b5eaccadd2bc Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 9 Aug 2024 21:42:56 -0400 Subject: [PATCH 137/506] [job control] Tweak messages to be more consistent Use the [%5] syntax where we can to indicate a job. TODO: some messages should be omitted when the shell is not interactive. --- builtin/process_osh.py | 12 +++++++----- core/executor.py | 2 +- core/process.py | 8 ++++---- spec/stateful/job_control.py | 18 +++++++++--------- spec/testdata/builtin-trap-int.sh | 2 +- test/bugs.sh | 26 ++++++++++++++++++++++++++ 6 files changed, 48 insertions(+), 20 deletions(-) diff --git a/builtin/process_osh.py b/builtin/process_osh.py index 470be69af5..bbfc0b1e90 100644 --- a/builtin/process_osh.py +++ b/builtin/process_osh.py @@ -36,6 +36,8 @@ from core.state import Mem, SearchPath from display import ui +_ = log + class Jobs(vm._Builtin): """List jobs.""" @@ -83,7 +85,7 @@ def Run(self, cmd_val): job = self.job_list.GetJobWithSpec(job_spec) if job is None: - log('No job to put in the foreground') + print_stderr('fg: No job to put in the foreground') return 1 pgid = job.ProcessGroupId() @@ -91,7 +93,7 @@ def Run(self, cmd_val): 'Processes put in the background should have a PGID' # TODO: Print job ID rather than the PID - log('Continue PID %d', pgid) + print_stderr('fg: PID %d Continued' % pgid) # Put the job's process group back into the foreground. GiveTerminal() must # be called before sending SIGCONT or else the process might immediately get # suspsended again if it tries to read/write on the terminal. @@ -380,7 +382,7 @@ def Run(self, cmd_val): except ValueError: # NOTE: This also happens when we have '8' or '9' in the input. print_stderr( - "osh warning: umask with symbolic input isn't implemented") + "oils warning: umask with symbolic input isn't implemented") return 1 posix.umask(new_mask) @@ -575,7 +577,7 @@ def Run(self, cmd_val): except (ValueError, resource.error) as e: # Annoying: Python binding changes IOError -> ValueError - print_stderr('ulimit error: %s' % e) + print_stderr('oils: ulimit error: %s' % e) # Extra info we could expose in C++ too print_stderr('soft=%s hard=%s -> soft=%s hard=%s' % ( @@ -589,7 +591,7 @@ def Run(self, cmd_val): try: pyos.SetRLimit(what, soft, hard) except (IOError, OSError) as e: - print_stderr('ulimit error: %s' % pyutil.strerror(e)) + print_stderr('oils: ulimit error: %s' % pyutil.strerror(e)) return 1 return 0 diff --git a/core/executor.py b/core/executor.py index 6eb79490c7..dbb587a846 100644 --- a/core/executor.py +++ b/core/executor.py @@ -426,7 +426,7 @@ def RunBackgroundJob(self, node): if self.exec_opts.interactive(): # Print it like %1 to show it's a job - print_stderr('[%%%d] %d' % (job_id, self.mem.last_bg_pid)) + print_stderr('[%%%d] PID %d Started' % (job_id, self.mem.last_bg_pid)) return 0 diff --git a/core/process.py b/core/process.py index 2bcdced065..eb33623761 100644 --- a/core/process.py +++ b/core/process.py @@ -1162,7 +1162,7 @@ def WhenDone(self, pid, status): # assigned a job ID. if self.in_background: # TODO: bash only prints this interactively - print_stderr('[%d] Done PID %d' % (self.job_id, self.pid)) + print_stderr('[%%%d] PID %d Done' % (self.job_id, self.pid)) self.job_list.RemoveJob(self.job_id) @@ -1445,7 +1445,7 @@ def WhenDone(self, pid, status): # Job might have been brought to the foreground after being # assigned a job ID. if self.in_background: - print_stderr('[%d] Done PGID %d' % + print_stderr('[%%%d] PGID %d Done' % (self.job_id, self.pids[0])) self.job_list.RemoveJob(self.job_id) @@ -1914,7 +1914,7 @@ def WaitForOne(self, waitpid_options=0): # notification of its exit, even though we didn't start it. We can't have # any knowledge of such processes, so print a warning. if pid not in self.job_list.child_procs: - print_stderr("osh: PID %d stopped, but osh didn't start it" % pid) + print_stderr("oils: PID %d Stopped, but osh didn't start it" % pid) return W1_OK proc = self.job_list.child_procs[pid] @@ -1941,7 +1941,7 @@ def WaitForOne(self, waitpid_options=0): stop_sig = WSTOPSIG(status) print_stderr('') - print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig)) + print_stderr('oils: PID %d Stopped with signal %d' % (pid, stop_sig)) proc.WhenStopped(stop_sig) else: diff --git a/spec/stateful/job_control.py b/spec/stateful/job_control.py index 9ba58a0a22..cc942d06aa 100755 --- a/spec/stateful/job_control.py +++ b/spec/stateful/job_control.py @@ -48,7 +48,7 @@ def expect_no_job(sh): def expect_continued(sh): if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') else: sh.expect('cat') @@ -89,7 +89,7 @@ def bug_1004(sh): sh.sendline('fg') if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') else: sh.expect('cat') @@ -209,7 +209,7 @@ def stopped_process(sh): sh.sendline('fg') if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') else: sh.expect('cat') @@ -245,7 +245,7 @@ def stopped_pipeline(sh): sh.sendline('fg') if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') else: sh.expect('cat') @@ -354,7 +354,7 @@ def fg_current_previous(sh): # Bring back the newest stopped job sh.sendline('fg %+') if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') sh.sendline('foo') sh.expect('foo') @@ -363,7 +363,7 @@ def fg_current_previous(sh): # Bring back the second-newest stopped job sh.sendline('fg %-') if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') sh.sendline('') sh.expect('bar') @@ -376,7 +376,7 @@ def fg_current_previous(sh): # Now that cat is gone, %- should refer to the running job sh.sendline('fg %-') if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') sh.sendline('true') time.sleep(0.5) @@ -386,14 +386,14 @@ def fg_current_previous(sh): # %+ and %- should refer to the same thing now that there's only one job sh.sendline('fg %+') if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') sh.sendline('woof') sh.expect('woof') ctrl_z(sh) sh.sendline('fg %-') if 'osh' in sh.shell_label: - sh.expect(r'Continue PID \d+') + sh.expect(r'.*PID \d+ Continue') sh.sendline('meow') sh.expect('meow') diff --git a/spec/testdata/builtin-trap-int.sh b/spec/testdata/builtin-trap-int.sh index ccd1a7c573..2f4c68b9fb 100755 --- a/spec/testdata/builtin-trap-int.sh +++ b/spec/testdata/builtin-trap-int.sh @@ -4,7 +4,7 @@ $SH -c 'trap "echo int" INT; sleep 0.1' & sleep 0.05 -$(which kill) -INT $! +$(command -v kill) -INT $! wait diff --git a/test/bugs.sh b/test/bugs.sh index 63692bfdd2..ed5f044523 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -66,6 +66,32 @@ trap-2() { echo "$sh status=$?" } +spec-sig() { + ### Run spec test outside the sh-spec framework + + local sh=${1:-bin/osh} + local sig=${2:-int} + + SH=$sh $sh spec/testdata/builtin-trap-$sig.sh +} + +spec-sig-all() { + local sig=${1:-int} + + # they all run usr1 + # they differ with respect int - only zsh prints it, and bin/osh + # + # zsh prints 'int' + + for sh in bin/osh bash dash mksh zsh; do + echo '-----' + echo "$sh" + echo + + spec-sig $sh $sig + done +} + trap-with-errexit() { local sh=${1:-bin/osh} From 1aac8b81e3521c3c1491f6b25e36de4d4e0dedc9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 9 Aug 2024 22:09:31 -0400 Subject: [PATCH 138/506] [spec/ysh-xtrace] Fix assertions --- spec/ysh-xtrace.test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/ysh-xtrace.test.sh b/spec/ysh-xtrace.test.sh index 6801922ba6..7e95d8e9d8 100644 --- a/spec/ysh-xtrace.test.sh +++ b/spec/ysh-xtrace.test.sh @@ -345,7 +345,7 @@ status=0 . builtin set '+x' < wait > wait -[1] Done PGID 12345 +[%1] PGID 12345 Done | part 12345 | part 12345 | part 12345 @@ -385,8 +385,8 @@ sed --regexp-extended 's/[[:digit:]]{2,}/12345/g' err.txt | < wait > wait > wait -[1] Done PID 12345 -[1] Done PID 12345 +[%1] PID 12345 Done +[%1] PID 12345 Done | fork 12345 | fork 12345 ## END From 5edf3cee81e5ebecbde5325df685386f83e6de76 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 9 Aug 2024 22:22:49 -0400 Subject: [PATCH 139/506] [job control] Make Enabled() check more efficient Check the condition that doesn't require a syscall. Also fix tracebacks in core/process_test.py. --- builtin/process_osh.py | 3 ++- core/executor.py | 3 ++- core/process.py | 18 +++++++++++------- core/process_test.py | 25 +++++++++++++------------ 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/builtin/process_osh.py b/builtin/process_osh.py index bbfc0b1e90..db8ab6558d 100644 --- a/builtin/process_osh.py +++ b/builtin/process_osh.py @@ -382,7 +382,8 @@ def Run(self, cmd_val): except ValueError: # NOTE: This also happens when we have '8' or '9' in the input. print_stderr( - "oils warning: umask with symbolic input isn't implemented") + "oils warning: umask with symbolic input isn't implemented" + ) return 1 posix.umask(new_mask) diff --git a/core/executor.py b/core/executor.py index dbb587a846..3b3e65b206 100644 --- a/core/executor.py +++ b/core/executor.py @@ -426,7 +426,8 @@ def RunBackgroundJob(self, node): if self.exec_opts.interactive(): # Print it like %1 to show it's a job - print_stderr('[%%%d] PID %d Started' % (job_id, self.mem.last_bg_pid)) + print_stderr('[%%%d] PID %d Started' % + (job_id, self.mem.last_bg_pid)) return 0 diff --git a/core/process.py b/core/process.py index eb33623761..a8ffba6463 100644 --- a/core/process.py +++ b/core/process.py @@ -1162,7 +1162,8 @@ def WhenDone(self, pid, status): # assigned a job ID. if self.in_background: # TODO: bash only prints this interactively - print_stderr('[%%%d] PID %d Done' % (self.job_id, self.pid)) + print_stderr('[%%%d] PID %d Done' % + (self.job_id, self.pid)) self.job_list.RemoveJob(self.job_id) @@ -1542,12 +1543,14 @@ def InitJobControl(self): def Enabled(self): # type: () -> bool + """ + Only the main shell process should bother with job control functions. + """ + #log('ENABLED? %d', self.shell_tty_fd) - # TODO: get rid of this syscall? SubProgramThunk should set a flag I - # think. - curr_pid = posix.getpid() - # Only the main shell should bother with job control functions. - return curr_pid == self.shell_pid and self.shell_tty_fd != -1 + # TODO: get rid of getpid()? I think SubProgramThunk should set a + # flag. + return self.shell_tty_fd != -1 and posix.getpid() == self.shell_pid # TODO: This isn't a PID. This is a process group ID? # @@ -1941,7 +1944,8 @@ def WaitForOne(self, waitpid_options=0): stop_sig = WSTOPSIG(status) print_stderr('') - print_stderr('oils: PID %d Stopped with signal %d' % (pid, stop_sig)) + print_stderr('oils: PID %d Stopped with signal %d' % + (pid, stop_sig)) proc.WhenStopped(stop_sig) else: diff --git a/core/process_test.py b/core/process_test.py index 781d47572d..922f2a9769 100755 --- a/core/process_test.py +++ b/core/process_test.py @@ -64,9 +64,10 @@ def setUp(self): self.trap_state = trap_osh.TrapState(signal_safe) fd_state = None - multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state) + self.multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', + fd_state) self.tracer = dev.Tracer(None, exec_opts, mutable_opts, mem, - mylib.Stderr(), multi_trace) + mylib.Stderr(), self.multi_trace) self.waiter = process.Waiter(self.job_list, exec_opts, self.trap_state, self.tracer) errfmt = ui.ErrorFormatter() @@ -181,12 +182,12 @@ def testPipeline2(self): node2 = _CommandNode('head', self.arena) node3 = _CommandNode('sort --reverse', self.arena) - thunk1 = process.SubProgramThunk(cmd_ev, node1, self.trap_state, None, - True, False) - thunk2 = process.SubProgramThunk(cmd_ev, node2, self.trap_state, None, - True, False) - thunk3 = process.SubProgramThunk(cmd_ev, node3, self.trap_state, None, - True, False) + thunk1 = process.SubProgramThunk(cmd_ev, node1, self.trap_state, + self.multi_trace, True, False) + thunk2 = process.SubProgramThunk(cmd_ev, node2, self.trap_state, + self.multi_trace, True, False) + thunk3 = process.SubProgramThunk(cmd_ev, node3, self.trap_state, + self.multi_trace, True, False) p = process.Pipeline(False, self.job_control, self.job_list, self.tracer) @@ -224,10 +225,10 @@ def makeTestPipeline(self, jc): node1 = _CommandNode('/bin/echo testpipeline', self.arena) node2 = _CommandNode('cat', self.arena) - thunk1 = process.SubProgramThunk(cmd_ev, node1, self.trap_state, None, - True, False) - thunk2 = process.SubProgramThunk(cmd_ev, node2, self.trap_state, None, - True, False) + thunk1 = process.SubProgramThunk(cmd_ev, node1, self.trap_state, + self.multi_trace, True, False) + thunk2 = process.SubProgramThunk(cmd_ev, node2, self.trap_state, + self.multi_trace, True, False) pi.Add(Process(thunk1, jc, self.job_list, self.tracer)) pi.Add(Process(thunk2, jc, self.job_list, self.tracer)) From dd89cc234ae8f6845e63b240c082d67426773888 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 10 Aug 2024 00:50:18 -0400 Subject: [PATCH 140/506] [spec/builtin-trap] Another test case I think I figured out the problem with C++ - it doesn't disable KeyboardInterrupt upon signal(SIGINT), like CPython does! --- builtin/trap_osh.py | 10 ++++++++-- spec/builtin-trap.test.sh | 10 ++++++++++ spec/testdata/builtin-trap-exit.sh | 12 ++++++++++++ test/bugs.sh | 9 +++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100755 spec/testdata/builtin-trap-exit.sh diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index 16539b9342..7fef379dda 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -106,6 +106,12 @@ def GetPendingTraps(self): """Transfer ownership of the current queue of pending trap handlers to the caller.""" signals = self.signal_safe.TakePendingSignals() + if 0: + log('*** GetPendingTraps') + for si in signals: + log('SIGNAL %d', si) + #import traceback + #traceback.print_stack() # Optimization for the common case: do not allocate a list. This function # is called in the interpreter loop. @@ -128,8 +134,8 @@ def GetPendingTraps(self): def ThisProcessHasTraps(self): # type: () -> bool """ - nolastfork optimizations should be disabled when the process has code - to run after fork! + noforklast optimizations are not enabled when the process has code to + run after fork! """ if 0: log('traps %d', len(self.traps)) diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index 7ead64602a..216a3e5c7a 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -296,3 +296,13 @@ status=0 ## OK mksh STDOUT: mksh ## END + +#### trap EXIT, sleep, SIGINT: non-interactively + +$REPO_ROOT/spec/testdata/builtin-trap-exit.sh + +## STDOUT: +on exit +status=0 +## END + diff --git a/spec/testdata/builtin-trap-exit.sh b/spec/testdata/builtin-trap-exit.sh new file mode 100755 index 0000000000..6ef722c03d --- /dev/null +++ b/spec/testdata/builtin-trap-exit.sh @@ -0,0 +1,12 @@ + +# Why don't other shells run this trap? It's not a subshell +$SH -c 'trap "echo on exit" EXIT; sleep 0.1' & + +sleep 0.05 + +# Note: this is SIGINT, for the KeyboardInterrupt problem +$(command -v kill) -INT $! + +wait + +echo status=$? diff --git a/test/bugs.sh b/test/bugs.sh index ed5f044523..03880f6131 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -92,6 +92,15 @@ spec-sig-all() { done } +sigint-loop() { + local sh=${1:-bin/osh} + + # Hm _bin/cxx-asan/osh behaves differently here -- it doesn't run it 5 times + # It quits the first time. + # bin/osh works like bash/dash/mksh/zsh - they all agree + $sh -c 'trap "echo int" INT; for i in 1 2 3 4 5; do sleep 1; done' +} + trap-with-errexit() { local sh=${1:-bin/osh} From c17d704a24333c29b5a09d480bd7278bdb2edeff Mon Sep 17 00:00:00 2001 From: Melvin Walls Date: Sun, 11 Aug 2024 21:33:48 -0400 Subject: [PATCH 141/506] [mycpp/dataflow] Add stack roots solver with --minimize-stack-roots (#2023) This patch adds a stack roots solver written in souffle. It uses the facts and control flow graph emitted by control_flow_pass to compute a minimal (approximately) set of stack roots required for safe execution of the source program. The relations and rules for this program, AKA our rooting policy, are derived from the examples enumerated in the translation example added in #2030. If mycpp is run with the --minimize-stack-roots flag, it will execute the stack roots solver and feed its output into cppgen_pass where it will be queried to determine if a StackRoot should be emitted for a local variable. When --minimize-stack-roots is set, cppgen_pass will also omit stack roots for loop index variables. Without --minimize-stack-roots there are no changes to the generated code produced by cppgen_pass. Running mycpp with --minimize-stack-roots reduces the number of stack roots in _gen/bin/oils_for_unix.mycpp.cc by ~78%! Before $ grep StackRoot _gen/bin/oils_for_unix.mycpp.cc | wc -l 4744 After $ grep StackRoot _gen/bin/oils_for_unix.mycpp.cc | wc -l 1001 --- mycpp/NINJA_subgraph.py | 1 + mycpp/control_flow_pass.py | 68 +- mycpp/cppgen_pass.py | 23 +- mycpp/datalog/call-graph.dl | 1 + mycpp/datalog/control-flow.dl | 3 + mycpp/datalog/dataflow.dl | 67 + mycpp/mycpp_main.py | 16 +- mycpp/pass_state.py | 136 +- prebuilt/datalog.sh | 1 + prebuilt/datalog/call-graph.cc | 15 +- prebuilt/datalog/dataflow.cc | 2072 +++++++++++++++++ prebuilt/ninja/mycpp.mycpp_main/deps.txt | 1 + .../control-flow-graph/classes/assign.facts | 81 +- .../control-flow-graph/classes/define.facts | 55 - 14 files changed, 2433 insertions(+), 107 deletions(-) create mode 100644 mycpp/datalog/dataflow.dl create mode 100644 prebuilt/datalog/dataflow.cc delete mode 100644 testdata/control-flow-graph/classes/define.facts diff --git a/mycpp/NINJA_subgraph.py b/mycpp/NINJA_subgraph.py index 24187c3faa..f6fc7e9ced 100644 --- a/mycpp/NINJA_subgraph.py +++ b/mycpp/NINJA_subgraph.py @@ -415,4 +415,5 @@ def NinjaGraph(ru): n.newline() ru.souffle_binary('prebuilt/datalog/call-graph.cc') + ru.souffle_binary('prebuilt/datalog/dataflow.cc') ru.souffle_binary('prebuilt/datalog/smoke-test.cc') diff --git a/mycpp/control_flow_pass.py b/mycpp/control_flow_pass.py index 610f187548..e995bf3eda 100644 --- a/mycpp/control_flow_pass.py +++ b/mycpp/control_flow_pass.py @@ -50,7 +50,9 @@ def __init__(self, types: Dict[Expression, Type], virtual, local_vars, self.virtual = virtual self.local_vars = local_vars self.dot_exprs = dot_exprs + self.heap_counter = 0 self.callees = {} # statement object -> SymbolPath of the callee + self.current_lval = None def current_cfg(self): if not self.current_func_node: @@ -145,7 +147,7 @@ def resolve_callee(self, o: CallExpr) -> Optional[util.SymbolPath]: # Don't currently get here raise AssertionError() - def get_variable_name(self, expr: Expression) -> Optional[util.SymbolPath]: + def get_ref_name(self, expr: Expression) -> Optional[util.SymbolPath]: """ To do dataflow analysis we need to track changes to objects, which requires naming them. This function returns the name of the object @@ -207,19 +209,23 @@ def get_variable_name(self, expr: Expression) -> Optional[util.SymbolPath]: return dot_expr.module_path + (dot_expr.member, ) elif isinstance(dot_expr, pass_state.HeapObjectMember): - return GetObjectTypeName( - dot_expr.object_type) + (dot_expr.member, ) + obj_name = self.get_ref_name(dot_expr.object_expr) + if obj_name: + # XXX: add a new case like pass_state.ExpressionMember for + # cases when the LHS of . isn't a reference (e.g. + # builtin/assign_osh.py:54) + return obj_name + (dot_expr.member, ) elif isinstance(dot_expr, pass_state.StackObjectMember): - return GetObjectTypeName( - dot_expr.object_type) + (dot_expr.member, ) + return self.get_ref_name( + dot_expr.object_expr) + (dot_expr.member, ) elif isinstance(expr, IndexExpr): if isinstance(self.types[expr.base], TupleType): assert isinstance(expr.index, IntExpr) - return self.get_variable_name(expr.base) + (str(expr.index.value),) + return self.get_ref_name(expr.base) + (str(expr.index.value), ) - return self.get_variable_name(expr.base) + return self.get_ref_name(expr.base) return None @@ -348,7 +354,8 @@ def visit_func_def(self, o: 'mypy.nodes.FuncDef') -> T: self.current_func_node = o cfg = self.current_cfg() for arg in o.arguments: - cfg.AddFact(0, pass_state.Definition((arg.variable.name,))) + cfg.AddFact(0, + pass_state.Definition((arg.variable.name, ), '$Empty')) self.accept(o.body) self.current_func_node = None @@ -439,10 +446,10 @@ def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: lval_names = [] if isinstance(lval, TupleExpr): lval_names.extend( - [self.get_variable_name(item) for item in lval.items]) + [self.get_ref_name(item) for item in lval.items]) else: - lval_names.append(self.get_variable_name(lval)) + lval_names.append(self.get_ref_name(lval)) assert lval_names, o @@ -464,20 +471,20 @@ def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: base + (str(i), ) for i in range(len(o.rvalue.items)) ] rval_names = [ - self.get_variable_name(item) for item in o.rvalue.items + self.get_ref_name(item) for item in o.rvalue.items ] elif isinstance(rval_type, TupleType): # We're unpacking a tuple. Like the tuple construction case, # give each element a name. - rval_name = self.get_variable_name(o.rvalue) + rval_name = self.get_ref_name(o.rvalue) assert rval_name, o.rvalue rval_names = [ rval_name + (str(i), ) for i in range(len(lval_names)) ] else: - rval_names = [self.get_variable_name(o.rvalue)] + rval_names = [self.get_ref_name(o.rvalue)] assert len(rval_names) == len(lval_names) @@ -494,16 +501,43 @@ def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: # statement as an (re-)definition of a variable. cfg.AddFact( self.current_statement_id, - pass_state.Definition(lhs), + pass_state.Definition( + lhs, '$HeapObject(h{})'.format(self.heap_counter)), ) + self.heap_counter += 1 for lval in o.lvalues: + self.current_lval = lval self.accept(lval) + self.current_lval = None self.accept(o.rvalue) # Expressions + def visit_member_expr(self, o: 'mypy.nodes.MemberExpr') -> T: + self.accept(o.expr) + cfg = self.current_cfg() + if (cfg and + not isinstance(self.dot_exprs[o], pass_state.ModuleMember) and + o != self.current_lval): + ref = self.get_ref_name(o) + if ref: + cfg.AddFact(self.current_statement_id, pass_state.Use(ref)) + + def visit_name_expr(self, o: 'mypy.nodes.NameExpr') -> T: + cfg = self.current_cfg() + if cfg and o != self.current_lval: + is_local = False + for name, t in self.local_vars.get(self.current_func_node, []): + if name == o.name: + is_local = True + break + + ref = self.get_ref_name(o) + if ref and is_local: + cfg.AddFact(self.current_statement_id, pass_state.Use(ref)) + def visit_call_expr(self, o: 'mypy.nodes.CallExpr') -> T: cfg = self.current_cfg() if self.current_func_node: @@ -514,6 +548,12 @@ def visit_call_expr(self, o: 'mypy.nodes.CallExpr') -> T: self.current_statement_id, pass_state.FunctionCall(join_name(full_callee, delim='.'))) + for i, arg in enumerate(o.args): + arg_ref = self.get_ref_name(arg) + if arg_ref: + cfg.AddFact(self.current_statement_id, + pass_state.Bind(arg_ref, full_callee, i)) + self.accept(o.callee) for arg in o.args: self.accept(arg) diff --git a/mycpp/cppgen_pass.py b/mycpp/cppgen_pass.py index 830bb10023..c9e397caaa 100644 --- a/mycpp/cppgen_pass.py +++ b/mycpp/cppgen_pass.py @@ -13,8 +13,8 @@ PartialType, TypeAliasType) from mypy.nodes import (Expression, Statement, NameExpr, IndexExpr, MemberExpr, TupleExpr, ExpressionStmt, IfStmt, StrExpr, SliceExpr, - FuncDef, UnaryExpr, OpExpr, CallExpr, - ListExpr, DictExpr, ListComprehension) + FuncDef, UnaryExpr, OpExpr, CallExpr, ListExpr, + DictExpr, ListComprehension) from mycpp import format_strings from mycpp.crash import catch_errors @@ -436,7 +436,8 @@ def __init__(self, decl=False, forward_decl=False, stack_roots_warn=None, - dot_exprs=None): + dot_exprs=None, + stack_roots=None): self.types = types self.const_lookup = const_lookup self.f = f @@ -475,6 +476,7 @@ def __init__(self, self.current_method_name = None self.dot_exprs = dot_exprs + self.stack_roots = stack_roots # So we can report multiple at once # module path, line number, message @@ -679,7 +681,9 @@ def visit_member_expr(self, o: 'mypy.nodes.MemberExpr') -> T: if isinstance(dot_expr, pass_state.StackObjectMember): op = '.' - elif isinstance(dot_expr, pass_state.StaticObjectMember) or isinstance(dot_expr, pass_state.ModuleMember): + elif isinstance(dot_expr, + pass_state.StaticObjectMember) or isinstance( + dot_expr, pass_state.ModuleMember): op = '::' elif isinstance(dot_expr, pass_state.HeapObjectMember): @@ -1883,7 +1887,7 @@ def visit_for_stmt(self, o: 'mypy.nodes.ForStmt') -> T: # it's called in a loop by _ExecuteList(). Although the 'child' # variable is already live by other means. # TODO: Test how much this affects performance. - if CTypeIsManaged(c_item_type): + if CTypeIsManaged(c_item_type) and not self.stack_roots: self.def_write_ind(' StackRoot _for(&') self.accept(index_expr) self.def_write_ind(');\n') @@ -2851,10 +2855,17 @@ def visit_block(self, block: 'mypy.nodes.Block') -> T: # Figure out if we have any roots to write with StackRoots roots = [] # keep it sorted + full_func_name = None + if self.current_func_node: + full_func_name = split_py_name(self.current_func_node.fullname) + for lval_name, c_type, is_param in self.prepend_to_block: #self.log('%s %s %s', lval_name, c_type, is_param) if lval_name not in roots and CTypeIsManaged(c_type): - roots.append(lval_name) + if (not self.stack_roots or self.stack_roots.needs_root( + full_func_name, split_py_name(lval_name))): + roots.append(lval_name) + #self.log('roots %s', roots) if len(roots): diff --git a/mycpp/datalog/call-graph.dl b/mycpp/datalog/call-graph.dl index db7b4ecff9..adbb204f71 100644 --- a/mycpp/datalog/call-graph.dl +++ b/mycpp/datalog/call-graph.dl @@ -11,5 +11,6 @@ .decl might_collect(f: Function, s: Statement) .output might_collect +might_collect("mylib.MaybeCollect", 0). might_collect(f, s) :- call(f, s, "mylib.MaybeCollect"). might_collect(f, s) :- call(f, s, g), might_collect(g, _). diff --git a/mycpp/datalog/control-flow.dl b/mycpp/datalog/control-flow.dl index 208ea96927..2bae3091f5 100644 --- a/mycpp/datalog/control-flow.dl +++ b/mycpp/datalog/control-flow.dl @@ -1,5 +1,8 @@ .once +// Facts and Relations (Inputs) +// ============================ + .type Function <: symbol .type Statement <: number diff --git a/mycpp/datalog/dataflow.dl b/mycpp/datalog/dataflow.dl new file mode 100644 index 0000000000..0951b3db4d --- /dev/null +++ b/mycpp/datalog/dataflow.dl @@ -0,0 +1,67 @@ +.once + +.include "control-flow.dl" +.include "call-graph.dl" + +// Types +// ===== + +// Objects can be refered to by either local variables or object members. +.type Reference = LocalVariable { f: Function, v: symbol } + | ObjectMember { o: symbol, m: symbol } + +.type Value = HeapObject { h: symbol } | Ref { r: Reference } | Empty {} + +// Facts and Relations +// =================== +// The facts and relations below use live variable analysis to determine when +// variables need stack roots. See +// https://en.wikipedia.org/wiki/Live-variable_analysis for more details. +// +// A variable is considered *live* at given statement if it might be used by a +// future statement. + +// `f` assigns `v` is assigned to `r` in statement `s`. +.decl assign(f:Function, s:Statement, r:Reference, v:Value) +.input assign + +// `f` uses `r` in statement `s`. +.decl use(f:Function, s:Statement, r:Reference) +.input use + +// `caller` binds `r` to positional argument `arg_pos` of `callee` in statement `s`. +.decl bind(caller:Function, s:Statement, r:Reference, callee:Function, arg_pos:number) +.input bind + +// The set of variables considered live on the way in to a statement. +.decl live_vars_in(f:Function, s:Statement, r:Reference) + +// The set of variables considered live on the way out of a statement. +.decl live_vars_out(f:Function, s:Statement, r:Reference) + +// The set of references that a function should generate stack roots for. +.decl stack_root_vars(f:Function, r: Reference) +.output stack_root_vars(IO=file, filename="stack_root_vars.tsv", delimeter="\t") + +// Rules +// ===== + +// See the definition of the GEN set at https://en.wikipedia.org/wiki/Live-variable_analysis +live_vars_in(f, s, r) :- use(f, s, r). +// See the definition of the KILL set at https://en.wikipedia.org/wiki/Live-variable_analysis +live_vars_in(f, s, r) :- !assign(f, s, r, _), live_vars_out(f, s, r). + +// The set of live variables leaving a statement is the union of the inbound +// live variables of the statements sucessors in the control flow graph. +live_vars_out(f, s1, r) :- cf_edge(f, s1, s2), live_vars_in(f, s2, r). + +// All variables considered live after a statement that, directly or indirectly, +// invokes the GC must be rooted. +stack_root_vars(f, r) :- call(f, s, g), might_collect(g, _), !bind(f, s, r, g, _), live_vars_out(f, s, r). + +// If a function invokes the GC, directly or indirectly, all of its heap-managed +// arguments must be rooted. +stack_root_vars(f, $LocalVariable(f, v)) :- might_collect(f, _), assign(f, 0, $LocalVariable(f, v), $Empty()). + +// All members of context managers must be rooted. +stack_root_vars(f, $ObjectMember("self", m)) :- match(".*ctx_.*__init__", f), assign(f, _, $ObjectMember("self", m), _). diff --git a/mycpp/mycpp_main.py b/mycpp/mycpp_main.py index 2803f91b56..3caafb2ebd 100755 --- a/mycpp/mycpp_main.py +++ b/mycpp/mycpp_main.py @@ -57,6 +57,12 @@ def Options(): type='int', help='Emit warnings about functions with too many stack roots') + p.add_option( + '--minimize-stack-roots', + dest='minimize_stack_roots', + default=False, + help='Try to minimize the number of GC stack roots.') + return p @@ -360,7 +366,12 @@ def main(argv): cfg_pass.visit_mypy_file(module) cfgs.update(cfg_pass.cfgs) - pass_state.DumpControlFlowGraphs(cfgs) + log('\tmycpp pass: DATAFLOW') + stack_roots = None + if opts.minimize_stack_roots: + stack_roots = pass_state.ComputeMinimalStackRoots(cfgs) + else: + pass_state.DumpControlFlowGraphs(cfgs) log('\tmycpp pass: IMPL') @@ -374,7 +385,8 @@ def main(argv): local_vars=local_vars, ctx_member_vars=ctx_member_vars, stack_roots_warn=opts.stack_roots_warn, - dot_exprs=dot_exprs[module.path]) + dot_exprs=dot_exprs[module.path], + stack_roots=stack_roots) p4.visit_mypy_file(module) MaybeExitWithErrors(p4) diff --git a/mycpp/pass_state.py b/mycpp/pass_state.py index 9dc170ea55..b677456c60 100644 --- a/mycpp/pass_state.py +++ b/mycpp/pass_state.py @@ -4,12 +4,14 @@ from __future__ import print_function import os +import re +import subprocess from collections import defaultdict from mypy.types import Type from mypy.nodes import Expression -from mycpp.util import join_name, log, SymbolPath +from mycpp.util import join_name, log, split_py_name, SymbolPath from typing import Optional @@ -49,7 +51,7 @@ class HeapObjectMember(object): def __init__(self, object_expr: Expression, object_type: Type, member: str) -> None: - self.ojbect_expr = object_expr + self.object_expr = object_expr self.object_type = object_type self.member = member @@ -152,11 +154,12 @@ def CanReorderFields(self, class_name: SymbolPath) -> bool: return True # by default they can be reordered -def SymbolPathToSouffle(p: SymbolPath) -> str: +def SymbolPathToReference(func: str, p: SymbolPath) -> str: if len(p) > 1: - return '$Member({}, {})'.format(join_name(p[:-1], delim='.'), p[-1]) + return '$ObjectMember({}, {})'.format(join_name(p[:-1], delim='.'), + p[-1]) - return '$Variable({})'.format(p[0]) + return '$LocalVariable({}, {})'.format(func, p[0]) class Fact(object): @@ -191,15 +194,17 @@ class Definition(Fact): The definition of a variable. This corresponds to an allocation. """ - def __init__(self, variable: SymbolPath) -> None: - self.variable = variable + def __init__(self, ref: SymbolPath, obj: str) -> None: + self.ref = ref + self.obj = obj def name(self) -> str: - return 'define' + return 'assign' def Generate(self, func: str, statement: int) -> str: - return '{}\t{}\t{}\n'.format(func, statement, - SymbolPathToSouffle(self.variable)) + return '{}\t{}\t{}\t{}\n'.format(func, statement, + SymbolPathToReference(func, self.ref), + self.obj) class Assignment(Fact): @@ -215,9 +220,62 @@ def name(self) -> str: return 'assign' def Generate(self, func: str, statement: int) -> str: - return '{}\t{}\t{}\t{}\n'.format(func, statement, - SymbolPathToSouffle(self.lhs), - SymbolPathToSouffle(self.rhs)) + return '{}\t{}\t{}\t$Ref({})\n'.format( + func, statement, SymbolPathToReference(func, self.lhs), + SymbolPathToReference(func, self.rhs)) + + +class Use(Fact): + """ + The use of a reference. + + In the last assignment below, we would emit Use(foo) and Use(x). We would, + however, not emit Use(foo.a) since it is an lvalue and would instead be + covered by the Assign fact. Similarly, the first two assignments do not + generate Use facts. + + foo = Foo() + x = Bar() + foo.a = x + + Any time a reference appears in an expression (or expression-statement) it + will be considered used. + + some_function(a) => Use(a) + a + b => Use(a), Use(b) + print(thing.dict[key]) => Use(thing), Use(thing.dict), Use(key) + obj.func() => Use(obj) + """ + + def __init__(self, ref: SymbolPath) -> None: + self.ref = ref + + def name(self) -> str: + return 'use' + + def Generate(self, func: str, statement: int) -> str: + return '{}\t{}\t{}\n'.format(func, statement, + SymbolPathToReference(func, self.ref)) + + +class Bind(Fact): + """ + Binding a reference to a positional function parameter. + """ + + def __init__(self, ref: SymbolPath, callee: SymbolPath, + arg_pos: int) -> None: + self.ref = ref + self.callee = callee + self.arg_pos = arg_pos + + def name(self) -> str: + return 'bind' + + def Generate(self, func: str, statement: int) -> str: + return '{}\t{}\t{}\t{}\t{}\n'.format( + func, statement, SymbolPathToReference(func, self.ref), + join_name(self.callee, delim='.'), self.arg_pos) class ControlFlowGraph(object): @@ -458,6 +516,21 @@ def __exit__(self, *args) -> None: self.cfg.predecessors.add(b) +class StackRoots(object): + """ + Output of the souffle stack roots solver. + """ + + def __init__(self, tuples: set[tuple[SymbolPath, SymbolPath]]) -> None: + self.root_tuples = tuples + + def needs_root(self, func: SymbolPath, reference: SymbolPath) -> bool: + """ + Returns true if the given reference should have a stack root. + """ + return (func, reference) in self.root_tuples + + def DumpControlFlowGraphs(cfgs: dict[str, ControlFlowGraph], facts_dir='_tmp/mycpp-facts') -> None: """ @@ -485,3 +558,40 @@ def DumpControlFlowGraphs(cfgs: dict[str, ControlFlowGraph], for f in fact_files.values(): f.close() + + +def ComputeMinimalStackRoots(cfgs: dict[str, ControlFlowGraph], + facts_dir: str = '_tmp/mycpp-facts', + souffle_output_dir: str = '_tmp') -> StackRoots: + """ + Run the the souffle stack roots solver and translate its output in a format + that can be queried by cppgen_pass. + """ + DumpControlFlowGraphs(cfgs, facts_dir=facts_dir) + subprocess.check_call([ + '_bin/datalog/dataflow', + '-F', + facts_dir, + '-D', + souffle_output_dir, + ]) + + tuples: set[tuple[SymbolPath, SymbolPath]] = set({}) + with open('{}/stack_root_vars.tsv'.format(souffle_output_dir), + 'r') as roots_f: + pat = re.compile(r'\$(.*)\((.*), (.*)\)') + for line in roots_f: + function, ref = line.split('\t') + m = pat.match(ref) + assert m.group(1) in ('LocalVariable', 'ObjectMember') + if m.group(1) == 'LocalVariable': + _, ref_func, var_name = m.groups() + assert ref_func == function + tuples.add((split_py_name(function), (var_name, ))) + + if m.group(1) == 'ObjectMember': + _, base_obj, member_name = m.groups() + tuples.add((split_py_name(function), + split_py_name(base_obj) + (member_name, ))) + + return StackRoots(tuples) diff --git a/prebuilt/datalog.sh b/prebuilt/datalog.sh index 6c582ef517..4a14502d1b 100755 --- a/prebuilt/datalog.sh +++ b/prebuilt/datalog.sh @@ -22,6 +22,7 @@ compile_one() { compile_all() { compile_one mycpp/datalog/call-graph.dl + compile_one mycpp/datalog/dataflow.dl compile_one deps/source.medo/souffle/smoke-test.dl } diff --git a/prebuilt/datalog/call-graph.cc b/prebuilt/datalog/call-graph.cc index d94ffbaa22..8378c03188 100644 --- a/prebuilt/datalog/call-graph.cc +++ b/prebuilt/datalog/call-graph.cc @@ -1,4 +1,4 @@ -#define SOUFFLE_GENERATOR_VERSION "UNKNOWN" +#define SOUFFLE_GENERATOR_VERSION "39d42a366" #include "souffle/CompiledSouffle.h" #include "souffle/SignalHandler.h" #include "souffle/SouffleInterface.h" @@ -522,9 +522,16 @@ rel_might_collect_ef1d0b06d36e4ddc(&rel_might_collect_ef1d0b06d36e4ddc){ } void Stratum_might_collect_beadc513d07ff032::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ -signalHandler->setMsg(R"_(might_collect(f,s) :- +signalHandler->setMsg(R"_(might_collect("mylib.MaybeCollect",0). +in file call-graph.dl [14:1-14:40])_"); +[&](){ +CREATE_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt,rel_might_collect_ef1d0b06d36e4ddc->createContext()); +Tuple tuple{{ramBitCast(RamSigned(0)),ramBitCast(RamSigned(0))}}; +rel_might_collect_ef1d0b06d36e4ddc->insert(tuple,READ_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt)); +} +();signalHandler->setMsg(R"_(might_collect(f,s) :- call(f,s,"mylib.MaybeCollect"). -in file call-graph.dl [14:1-14:57])_"); +in file call-graph.dl [15:1-15:57])_"); if(!(rel_call_ee1d8972d66cc25f->empty())) { [&](){ CREATE_OP_CONTEXT(rel_call_ee1d8972d66cc25f_op_ctxt,rel_call_ee1d8972d66cc25f->createContext()); @@ -550,7 +557,7 @@ for(;;) { signalHandler->setMsg(R"_(might_collect(f,s) :- call(f,s,g), might_collect(g,_). -in file call-graph.dl [15:1-15:59])_"); +in file call-graph.dl [16:1-16:59])_"); if(!(rel_call_ee1d8972d66cc25f->empty()) && !(rel_delta_might_collect_d651f71586aafe59->empty())) { [&](){ CREATE_OP_CONTEXT(rel_delta_might_collect_d651f71586aafe59_op_ctxt,rel_delta_might_collect_d651f71586aafe59->createContext()); diff --git a/prebuilt/datalog/dataflow.cc b/prebuilt/datalog/dataflow.cc new file mode 100644 index 0000000000..e5106e415f --- /dev/null +++ b/prebuilt/datalog/dataflow.cc @@ -0,0 +1,2072 @@ +#define SOUFFLE_GENERATOR_VERSION "39d42a366" +#include "souffle/CompiledSouffle.h" +#include "souffle/SignalHandler.h" +#include "souffle/SouffleInterface.h" +#include "souffle/datastructure/BTree.h" +#include "souffle/io/IOSystem.h" +#include +namespace functors { +extern "C" { +} +} //namespace functors +namespace souffle::t_btree_iiii__0_1_2_3__1110__1111__1100 { +using namespace souffle; +struct Type { +static constexpr Relation::arity_type Arity = 4; +using t_tuple = Tuple; +struct t_comparator_0{ + int operator()(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0])) ? -1 : (ramBitCast(a[0]) > ramBitCast(b[0])) ? 1 :((ramBitCast(a[1]) < ramBitCast(b[1])) ? -1 : (ramBitCast(a[1]) > ramBitCast(b[1])) ? 1 :((ramBitCast(a[2]) < ramBitCast(b[2])) ? -1 : (ramBitCast(a[2]) > ramBitCast(b[2])) ? 1 :((ramBitCast(a[3]) < ramBitCast(b[3])) ? -1 : (ramBitCast(a[3]) > ramBitCast(b[3])) ? 1 :(0)))); + } +bool less(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0]))|| ((ramBitCast(a[0]) == ramBitCast(b[0])) && ((ramBitCast(a[1]) < ramBitCast(b[1]))|| ((ramBitCast(a[1]) == ramBitCast(b[1])) && ((ramBitCast(a[2]) < ramBitCast(b[2]))|| ((ramBitCast(a[2]) == ramBitCast(b[2])) && ((ramBitCast(a[3]) < ramBitCast(b[3])))))))); + } +bool equal(const t_tuple& a, const t_tuple& b) const { +return (ramBitCast(a[0]) == ramBitCast(b[0]))&&(ramBitCast(a[1]) == ramBitCast(b[1]))&&(ramBitCast(a[2]) == ramBitCast(b[2]))&&(ramBitCast(a[3]) == ramBitCast(b[3])); + } +}; +using t_ind_0 = btree_set; +t_ind_0 ind_0; +using iterator = t_ind_0::iterator; +struct context { +t_ind_0::operation_hints hints_0_lower; +t_ind_0::operation_hints hints_0_upper; +}; +context createContext() { return context(); } +bool insert(const t_tuple& t); +bool insert(const t_tuple& t, context& h); +bool insert(const RamDomain* ramDomain); +bool insert(RamDomain a0,RamDomain a1,RamDomain a2,RamDomain a3); +bool contains(const t_tuple& t, context& h) const; +bool contains(const t_tuple& t) const; +std::size_t size() const; +iterator find(const t_tuple& t, context& h) const; +iterator find(const t_tuple& t) const; +range lowerUpperRange_0000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const; +range lowerUpperRange_0000(const t_tuple& /* lower */, const t_tuple& /* upper */) const; +range lowerUpperRange_1110(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_1110(const t_tuple& lower, const t_tuple& upper) const; +range lowerUpperRange_1111(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_1111(const t_tuple& lower, const t_tuple& upper) const; +range lowerUpperRange_1100(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_1100(const t_tuple& lower, const t_tuple& upper) const; +bool empty() const; +std::vector> partition() const; +void purge(); +iterator begin() const; +iterator end() const; +void printStatistics(std::ostream& o) const; +}; +} // namespace souffle::t_btree_iiii__0_1_2_3__1110__1111__1100 +namespace souffle::t_btree_iiii__0_1_2_3__1110__1111__1100 { +using namespace souffle; +using t_ind_0 = Type::t_ind_0; +using iterator = Type::iterator; +using context = Type::context; +bool Type::insert(const t_tuple& t) { +context h; +return insert(t, h); +} +bool Type::insert(const t_tuple& t, context& h) { +if (ind_0.insert(t, h.hints_0_lower)) { +return true; +} else return false; +} +bool Type::insert(const RamDomain* ramDomain) { +RamDomain data[4]; +std::copy(ramDomain, ramDomain + 4, data); +const t_tuple& tuple = reinterpret_cast(data); +context h; +return insert(tuple, h); +} +bool Type::insert(RamDomain a0,RamDomain a1,RamDomain a2,RamDomain a3) { +RamDomain data[4] = {a0,a1,a2,a3}; +return insert(data); +} +bool Type::contains(const t_tuple& t, context& h) const { +return ind_0.contains(t, h.hints_0_lower); +} +bool Type::contains(const t_tuple& t) const { +context h; +return contains(t, h); +} +std::size_t Type::size() const { +return ind_0.size(); +} +iterator Type::find(const t_tuple& t, context& h) const { +return ind_0.find(t, h.hints_0_lower); +} +iterator Type::find(const t_tuple& t) const { +context h; +return find(t, h); +} +range Type::lowerUpperRange_0000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_0000(const t_tuple& /* lower */, const t_tuple& /* upper */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_1110(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_1110(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_1110(lower,upper,h); +} +range Type::lowerUpperRange_1111(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp == 0) { + auto pos = ind_0.find(lower, h.hints_0_lower); + auto fin = ind_0.end(); + if (pos != fin) {fin = pos; ++fin;} + return make_range(pos, fin); +} +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_1111(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_1111(lower,upper,h); +} +range Type::lowerUpperRange_1100(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_1100(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_1100(lower,upper,h); +} +bool Type::empty() const { +return ind_0.empty(); +} +std::vector> Type::partition() const { +return ind_0.getChunks(400); +} +void Type::purge() { +ind_0.clear(); +} +iterator Type::begin() const { +return ind_0.begin(); +} +iterator Type::end() const { +return ind_0.end(); +} +void Type::printStatistics(std::ostream& o) const { +o << " arity 4 direct b-tree index 0 lex-order [0,1,2,3]\n"; +ind_0.printStats(o); +} +} // namespace souffle::t_btree_iiii__0_1_2_3__1110__1111__1100 +namespace souffle::t_btree_iiiii__0_1_2_3_4__11111__11110 { +using namespace souffle; +struct Type { +static constexpr Relation::arity_type Arity = 5; +using t_tuple = Tuple; +struct t_comparator_0{ + int operator()(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0])) ? -1 : (ramBitCast(a[0]) > ramBitCast(b[0])) ? 1 :((ramBitCast(a[1]) < ramBitCast(b[1])) ? -1 : (ramBitCast(a[1]) > ramBitCast(b[1])) ? 1 :((ramBitCast(a[2]) < ramBitCast(b[2])) ? -1 : (ramBitCast(a[2]) > ramBitCast(b[2])) ? 1 :((ramBitCast(a[3]) < ramBitCast(b[3])) ? -1 : (ramBitCast(a[3]) > ramBitCast(b[3])) ? 1 :((ramBitCast(a[4]) < ramBitCast(b[4])) ? -1 : (ramBitCast(a[4]) > ramBitCast(b[4])) ? 1 :(0))))); + } +bool less(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0]))|| ((ramBitCast(a[0]) == ramBitCast(b[0])) && ((ramBitCast(a[1]) < ramBitCast(b[1]))|| ((ramBitCast(a[1]) == ramBitCast(b[1])) && ((ramBitCast(a[2]) < ramBitCast(b[2]))|| ((ramBitCast(a[2]) == ramBitCast(b[2])) && ((ramBitCast(a[3]) < ramBitCast(b[3]))|| ((ramBitCast(a[3]) == ramBitCast(b[3])) && ((ramBitCast(a[4]) < ramBitCast(b[4])))))))))); + } +bool equal(const t_tuple& a, const t_tuple& b) const { +return (ramBitCast(a[0]) == ramBitCast(b[0]))&&(ramBitCast(a[1]) == ramBitCast(b[1]))&&(ramBitCast(a[2]) == ramBitCast(b[2]))&&(ramBitCast(a[3]) == ramBitCast(b[3]))&&(ramBitCast(a[4]) == ramBitCast(b[4])); + } +}; +using t_ind_0 = btree_set; +t_ind_0 ind_0; +using iterator = t_ind_0::iterator; +struct context { +t_ind_0::operation_hints hints_0_lower; +t_ind_0::operation_hints hints_0_upper; +}; +context createContext() { return context(); } +bool insert(const t_tuple& t); +bool insert(const t_tuple& t, context& h); +bool insert(const RamDomain* ramDomain); +bool insert(RamDomain a0,RamDomain a1,RamDomain a2,RamDomain a3,RamDomain a4); +bool contains(const t_tuple& t, context& h) const; +bool contains(const t_tuple& t) const; +std::size_t size() const; +iterator find(const t_tuple& t, context& h) const; +iterator find(const t_tuple& t) const; +range lowerUpperRange_00000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const; +range lowerUpperRange_00000(const t_tuple& /* lower */, const t_tuple& /* upper */) const; +range lowerUpperRange_11111(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_11111(const t_tuple& lower, const t_tuple& upper) const; +range lowerUpperRange_11110(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_11110(const t_tuple& lower, const t_tuple& upper) const; +bool empty() const; +std::vector> partition() const; +void purge(); +iterator begin() const; +iterator end() const; +void printStatistics(std::ostream& o) const; +}; +} // namespace souffle::t_btree_iiiii__0_1_2_3_4__11111__11110 +namespace souffle::t_btree_iiiii__0_1_2_3_4__11111__11110 { +using namespace souffle; +using t_ind_0 = Type::t_ind_0; +using iterator = Type::iterator; +using context = Type::context; +bool Type::insert(const t_tuple& t) { +context h; +return insert(t, h); +} +bool Type::insert(const t_tuple& t, context& h) { +if (ind_0.insert(t, h.hints_0_lower)) { +return true; +} else return false; +} +bool Type::insert(const RamDomain* ramDomain) { +RamDomain data[5]; +std::copy(ramDomain, ramDomain + 5, data); +const t_tuple& tuple = reinterpret_cast(data); +context h; +return insert(tuple, h); +} +bool Type::insert(RamDomain a0,RamDomain a1,RamDomain a2,RamDomain a3,RamDomain a4) { +RamDomain data[5] = {a0,a1,a2,a3,a4}; +return insert(data); +} +bool Type::contains(const t_tuple& t, context& h) const { +return ind_0.contains(t, h.hints_0_lower); +} +bool Type::contains(const t_tuple& t) const { +context h; +return contains(t, h); +} +std::size_t Type::size() const { +return ind_0.size(); +} +iterator Type::find(const t_tuple& t, context& h) const { +return ind_0.find(t, h.hints_0_lower); +} +iterator Type::find(const t_tuple& t) const { +context h; +return find(t, h); +} +range Type::lowerUpperRange_00000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_00000(const t_tuple& /* lower */, const t_tuple& /* upper */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_11111(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp == 0) { + auto pos = ind_0.find(lower, h.hints_0_lower); + auto fin = ind_0.end(); + if (pos != fin) {fin = pos; ++fin;} + return make_range(pos, fin); +} +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_11111(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_11111(lower,upper,h); +} +range Type::lowerUpperRange_11110(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_11110(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_11110(lower,upper,h); +} +bool Type::empty() const { +return ind_0.empty(); +} +std::vector> Type::partition() const { +return ind_0.getChunks(400); +} +void Type::purge() { +ind_0.clear(); +} +iterator Type::begin() const { +return ind_0.begin(); +} +iterator Type::end() const { +return ind_0.end(); +} +void Type::printStatistics(std::ostream& o) const { +o << " arity 5 direct b-tree index 0 lex-order [0,1,2,3,4]\n"; +ind_0.printStats(o); +} +} // namespace souffle::t_btree_iiiii__0_1_2_3_4__11111__11110 +namespace souffle::t_btree_iii__2_0_1__001__111 { +using namespace souffle; +struct Type { +static constexpr Relation::arity_type Arity = 3; +using t_tuple = Tuple; +struct t_comparator_0{ + int operator()(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[2]) < ramBitCast(b[2])) ? -1 : (ramBitCast(a[2]) > ramBitCast(b[2])) ? 1 :((ramBitCast(a[0]) < ramBitCast(b[0])) ? -1 : (ramBitCast(a[0]) > ramBitCast(b[0])) ? 1 :((ramBitCast(a[1]) < ramBitCast(b[1])) ? -1 : (ramBitCast(a[1]) > ramBitCast(b[1])) ? 1 :(0))); + } +bool less(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[2]) < ramBitCast(b[2]))|| ((ramBitCast(a[2]) == ramBitCast(b[2])) && ((ramBitCast(a[0]) < ramBitCast(b[0]))|| ((ramBitCast(a[0]) == ramBitCast(b[0])) && ((ramBitCast(a[1]) < ramBitCast(b[1])))))); + } +bool equal(const t_tuple& a, const t_tuple& b) const { +return (ramBitCast(a[2]) == ramBitCast(b[2]))&&(ramBitCast(a[0]) == ramBitCast(b[0]))&&(ramBitCast(a[1]) == ramBitCast(b[1])); + } +}; +using t_ind_0 = btree_set; +t_ind_0 ind_0; +using iterator = t_ind_0::iterator; +struct context { +t_ind_0::operation_hints hints_0_lower; +t_ind_0::operation_hints hints_0_upper; +}; +context createContext() { return context(); } +bool insert(const t_tuple& t); +bool insert(const t_tuple& t, context& h); +bool insert(const RamDomain* ramDomain); +bool insert(RamDomain a0,RamDomain a1,RamDomain a2); +bool contains(const t_tuple& t, context& h) const; +bool contains(const t_tuple& t) const; +std::size_t size() const; +iterator find(const t_tuple& t, context& h) const; +iterator find(const t_tuple& t) const; +range lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const; +range lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const; +range lowerUpperRange_001(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_001(const t_tuple& lower, const t_tuple& upper) const; +range lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const; +bool empty() const; +std::vector> partition() const; +void purge(); +iterator begin() const; +iterator end() const; +void printStatistics(std::ostream& o) const; +}; +} // namespace souffle::t_btree_iii__2_0_1__001__111 +namespace souffle::t_btree_iii__2_0_1__001__111 { +using namespace souffle; +using t_ind_0 = Type::t_ind_0; +using iterator = Type::iterator; +using context = Type::context; +bool Type::insert(const t_tuple& t) { +context h; +return insert(t, h); +} +bool Type::insert(const t_tuple& t, context& h) { +if (ind_0.insert(t, h.hints_0_lower)) { +return true; +} else return false; +} +bool Type::insert(const RamDomain* ramDomain) { +RamDomain data[3]; +std::copy(ramDomain, ramDomain + 3, data); +const t_tuple& tuple = reinterpret_cast(data); +context h; +return insert(tuple, h); +} +bool Type::insert(RamDomain a0,RamDomain a1,RamDomain a2) { +RamDomain data[3] = {a0,a1,a2}; +return insert(data); +} +bool Type::contains(const t_tuple& t, context& h) const { +return ind_0.contains(t, h.hints_0_lower); +} +bool Type::contains(const t_tuple& t) const { +context h; +return contains(t, h); +} +std::size_t Type::size() const { +return ind_0.size(); +} +iterator Type::find(const t_tuple& t, context& h) const { +return ind_0.find(t, h.hints_0_lower); +} +iterator Type::find(const t_tuple& t) const { +context h; +return find(t, h); +} +range Type::lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_001(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_001(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_001(lower,upper,h); +} +range Type::lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp == 0) { + auto pos = ind_0.find(lower, h.hints_0_lower); + auto fin = ind_0.end(); + if (pos != fin) {fin = pos; ++fin;} + return make_range(pos, fin); +} +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_111(lower,upper,h); +} +bool Type::empty() const { +return ind_0.empty(); +} +std::vector> Type::partition() const { +return ind_0.getChunks(400); +} +void Type::purge() { +ind_0.clear(); +} +iterator Type::begin() const { +return ind_0.begin(); +} +iterator Type::end() const { +return ind_0.end(); +} +void Type::printStatistics(std::ostream& o) const { +o << " arity 3 direct b-tree index 0 lex-order [2,0,1]\n"; +ind_0.printStats(o); +} +} // namespace souffle::t_btree_iii__2_0_1__001__111 +namespace souffle::t_btree_ii__0_1__11__10 { +using namespace souffle; +struct Type { +static constexpr Relation::arity_type Arity = 2; +using t_tuple = Tuple; +struct t_comparator_0{ + int operator()(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0])) ? -1 : (ramBitCast(a[0]) > ramBitCast(b[0])) ? 1 :((ramBitCast(a[1]) < ramBitCast(b[1])) ? -1 : (ramBitCast(a[1]) > ramBitCast(b[1])) ? 1 :(0)); + } +bool less(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0]))|| ((ramBitCast(a[0]) == ramBitCast(b[0])) && ((ramBitCast(a[1]) < ramBitCast(b[1])))); + } +bool equal(const t_tuple& a, const t_tuple& b) const { +return (ramBitCast(a[0]) == ramBitCast(b[0]))&&(ramBitCast(a[1]) == ramBitCast(b[1])); + } +}; +using t_ind_0 = btree_set; +t_ind_0 ind_0; +using iterator = t_ind_0::iterator; +struct context { +t_ind_0::operation_hints hints_0_lower; +t_ind_0::operation_hints hints_0_upper; +}; +context createContext() { return context(); } +bool insert(const t_tuple& t); +bool insert(const t_tuple& t, context& h); +bool insert(const RamDomain* ramDomain); +bool insert(RamDomain a0,RamDomain a1); +bool contains(const t_tuple& t, context& h) const; +bool contains(const t_tuple& t) const; +std::size_t size() const; +iterator find(const t_tuple& t, context& h) const; +iterator find(const t_tuple& t) const; +range lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const; +range lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */) const; +range lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper) const; +range lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper) const; +bool empty() const; +std::vector> partition() const; +void purge(); +iterator begin() const; +iterator end() const; +void printStatistics(std::ostream& o) const; +}; +} // namespace souffle::t_btree_ii__0_1__11__10 +namespace souffle::t_btree_ii__0_1__11__10 { +using namespace souffle; +using t_ind_0 = Type::t_ind_0; +using iterator = Type::iterator; +using context = Type::context; +bool Type::insert(const t_tuple& t) { +context h; +return insert(t, h); +} +bool Type::insert(const t_tuple& t, context& h) { +if (ind_0.insert(t, h.hints_0_lower)) { +return true; +} else return false; +} +bool Type::insert(const RamDomain* ramDomain) { +RamDomain data[2]; +std::copy(ramDomain, ramDomain + 2, data); +const t_tuple& tuple = reinterpret_cast(data); +context h; +return insert(tuple, h); +} +bool Type::insert(RamDomain a0,RamDomain a1) { +RamDomain data[2] = {a0,a1}; +return insert(data); +} +bool Type::contains(const t_tuple& t, context& h) const { +return ind_0.contains(t, h.hints_0_lower); +} +bool Type::contains(const t_tuple& t) const { +context h; +return contains(t, h); +} +std::size_t Type::size() const { +return ind_0.size(); +} +iterator Type::find(const t_tuple& t, context& h) const { +return ind_0.find(t, h.hints_0_lower); +} +iterator Type::find(const t_tuple& t) const { +context h; +return find(t, h); +} +range Type::lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp == 0) { + auto pos = ind_0.find(lower, h.hints_0_lower); + auto fin = ind_0.end(); + if (pos != fin) {fin = pos; ++fin;} + return make_range(pos, fin); +} +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_11(lower,upper,h); +} +range Type::lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_10(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_10(lower,upper,h); +} +bool Type::empty() const { +return ind_0.empty(); +} +std::vector> Type::partition() const { +return ind_0.getChunks(400); +} +void Type::purge() { +ind_0.clear(); +} +iterator Type::begin() const { +return ind_0.begin(); +} +iterator Type::end() const { +return ind_0.end(); +} +void Type::printStatistics(std::ostream& o) const { +o << " arity 2 direct b-tree index 0 lex-order [0,1]\n"; +ind_0.printStats(o); +} +} // namespace souffle::t_btree_ii__0_1__11__10 +namespace souffle::t_btree_iii__0_1_2__111 { +using namespace souffle; +struct Type { +static constexpr Relation::arity_type Arity = 3; +using t_tuple = Tuple; +struct t_comparator_0{ + int operator()(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0])) ? -1 : (ramBitCast(a[0]) > ramBitCast(b[0])) ? 1 :((ramBitCast(a[1]) < ramBitCast(b[1])) ? -1 : (ramBitCast(a[1]) > ramBitCast(b[1])) ? 1 :((ramBitCast(a[2]) < ramBitCast(b[2])) ? -1 : (ramBitCast(a[2]) > ramBitCast(b[2])) ? 1 :(0))); + } +bool less(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0]))|| ((ramBitCast(a[0]) == ramBitCast(b[0])) && ((ramBitCast(a[1]) < ramBitCast(b[1]))|| ((ramBitCast(a[1]) == ramBitCast(b[1])) && ((ramBitCast(a[2]) < ramBitCast(b[2])))))); + } +bool equal(const t_tuple& a, const t_tuple& b) const { +return (ramBitCast(a[0]) == ramBitCast(b[0]))&&(ramBitCast(a[1]) == ramBitCast(b[1]))&&(ramBitCast(a[2]) == ramBitCast(b[2])); + } +}; +using t_ind_0 = btree_set; +t_ind_0 ind_0; +using iterator = t_ind_0::iterator; +struct context { +t_ind_0::operation_hints hints_0_lower; +t_ind_0::operation_hints hints_0_upper; +}; +context createContext() { return context(); } +bool insert(const t_tuple& t); +bool insert(const t_tuple& t, context& h); +bool insert(const RamDomain* ramDomain); +bool insert(RamDomain a0,RamDomain a1,RamDomain a2); +bool contains(const t_tuple& t, context& h) const; +bool contains(const t_tuple& t) const; +std::size_t size() const; +iterator find(const t_tuple& t, context& h) const; +iterator find(const t_tuple& t) const; +range lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const; +range lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const; +range lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const; +bool empty() const; +std::vector> partition() const; +void purge(); +iterator begin() const; +iterator end() const; +void printStatistics(std::ostream& o) const; +}; +} // namespace souffle::t_btree_iii__0_1_2__111 +namespace souffle::t_btree_iii__0_1_2__111 { +using namespace souffle; +using t_ind_0 = Type::t_ind_0; +using iterator = Type::iterator; +using context = Type::context; +bool Type::insert(const t_tuple& t) { +context h; +return insert(t, h); +} +bool Type::insert(const t_tuple& t, context& h) { +if (ind_0.insert(t, h.hints_0_lower)) { +return true; +} else return false; +} +bool Type::insert(const RamDomain* ramDomain) { +RamDomain data[3]; +std::copy(ramDomain, ramDomain + 3, data); +const t_tuple& tuple = reinterpret_cast(data); +context h; +return insert(tuple, h); +} +bool Type::insert(RamDomain a0,RamDomain a1,RamDomain a2) { +RamDomain data[3] = {a0,a1,a2}; +return insert(data); +} +bool Type::contains(const t_tuple& t, context& h) const { +return ind_0.contains(t, h.hints_0_lower); +} +bool Type::contains(const t_tuple& t) const { +context h; +return contains(t, h); +} +std::size_t Type::size() const { +return ind_0.size(); +} +iterator Type::find(const t_tuple& t, context& h) const { +return ind_0.find(t, h.hints_0_lower); +} +iterator Type::find(const t_tuple& t) const { +context h; +return find(t, h); +} +range Type::lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp == 0) { + auto pos = ind_0.find(lower, h.hints_0_lower); + auto fin = ind_0.end(); + if (pos != fin) {fin = pos; ++fin;} + return make_range(pos, fin); +} +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_111(lower,upper,h); +} +bool Type::empty() const { +return ind_0.empty(); +} +std::vector> Type::partition() const { +return ind_0.getChunks(400); +} +void Type::purge() { +ind_0.clear(); +} +iterator Type::begin() const { +return ind_0.begin(); +} +iterator Type::end() const { +return ind_0.end(); +} +void Type::printStatistics(std::ostream& o) const { +o << " arity 3 direct b-tree index 0 lex-order [0,1,2]\n"; +ind_0.printStats(o); +} +} // namespace souffle::t_btree_iii__0_1_2__111 +namespace souffle::t_btree_iii__0_1_2__110__111 { +using namespace souffle; +struct Type { +static constexpr Relation::arity_type Arity = 3; +using t_tuple = Tuple; +struct t_comparator_0{ + int operator()(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0])) ? -1 : (ramBitCast(a[0]) > ramBitCast(b[0])) ? 1 :((ramBitCast(a[1]) < ramBitCast(b[1])) ? -1 : (ramBitCast(a[1]) > ramBitCast(b[1])) ? 1 :((ramBitCast(a[2]) < ramBitCast(b[2])) ? -1 : (ramBitCast(a[2]) > ramBitCast(b[2])) ? 1 :(0))); + } +bool less(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0]))|| ((ramBitCast(a[0]) == ramBitCast(b[0])) && ((ramBitCast(a[1]) < ramBitCast(b[1]))|| ((ramBitCast(a[1]) == ramBitCast(b[1])) && ((ramBitCast(a[2]) < ramBitCast(b[2])))))); + } +bool equal(const t_tuple& a, const t_tuple& b) const { +return (ramBitCast(a[0]) == ramBitCast(b[0]))&&(ramBitCast(a[1]) == ramBitCast(b[1]))&&(ramBitCast(a[2]) == ramBitCast(b[2])); + } +}; +using t_ind_0 = btree_set; +t_ind_0 ind_0; +using iterator = t_ind_0::iterator; +struct context { +t_ind_0::operation_hints hints_0_lower; +t_ind_0::operation_hints hints_0_upper; +}; +context createContext() { return context(); } +bool insert(const t_tuple& t); +bool insert(const t_tuple& t, context& h); +bool insert(const RamDomain* ramDomain); +bool insert(RamDomain a0,RamDomain a1,RamDomain a2); +bool contains(const t_tuple& t, context& h) const; +bool contains(const t_tuple& t) const; +std::size_t size() const; +iterator find(const t_tuple& t, context& h) const; +iterator find(const t_tuple& t) const; +range lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const; +range lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const; +range lowerUpperRange_110(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_110(const t_tuple& lower, const t_tuple& upper) const; +range lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const; +bool empty() const; +std::vector> partition() const; +void purge(); +iterator begin() const; +iterator end() const; +void printStatistics(std::ostream& o) const; +}; +} // namespace souffle::t_btree_iii__0_1_2__110__111 +namespace souffle::t_btree_iii__0_1_2__110__111 { +using namespace souffle; +using t_ind_0 = Type::t_ind_0; +using iterator = Type::iterator; +using context = Type::context; +bool Type::insert(const t_tuple& t) { +context h; +return insert(t, h); +} +bool Type::insert(const t_tuple& t, context& h) { +if (ind_0.insert(t, h.hints_0_lower)) { +return true; +} else return false; +} +bool Type::insert(const RamDomain* ramDomain) { +RamDomain data[3]; +std::copy(ramDomain, ramDomain + 3, data); +const t_tuple& tuple = reinterpret_cast(data); +context h; +return insert(tuple, h); +} +bool Type::insert(RamDomain a0,RamDomain a1,RamDomain a2) { +RamDomain data[3] = {a0,a1,a2}; +return insert(data); +} +bool Type::contains(const t_tuple& t, context& h) const { +return ind_0.contains(t, h.hints_0_lower); +} +bool Type::contains(const t_tuple& t) const { +context h; +return contains(t, h); +} +std::size_t Type::size() const { +return ind_0.size(); +} +iterator Type::find(const t_tuple& t, context& h) const { +return ind_0.find(t, h.hints_0_lower); +} +iterator Type::find(const t_tuple& t) const { +context h; +return find(t, h); +} +range Type::lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_000(const t_tuple& /* lower */, const t_tuple& /* upper */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_110(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_110(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_110(lower,upper,h); +} +range Type::lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp == 0) { + auto pos = ind_0.find(lower, h.hints_0_lower); + auto fin = ind_0.end(); + if (pos != fin) {fin = pos; ++fin;} + return make_range(pos, fin); +} +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_111(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_111(lower,upper,h); +} +bool Type::empty() const { +return ind_0.empty(); +} +std::vector> Type::partition() const { +return ind_0.getChunks(400); +} +void Type::purge() { +ind_0.clear(); +} +iterator Type::begin() const { +return ind_0.begin(); +} +iterator Type::end() const { +return ind_0.end(); +} +void Type::printStatistics(std::ostream& o) const { +o << " arity 3 direct b-tree index 0 lex-order [0,1,2]\n"; +ind_0.printStats(o); +} +} // namespace souffle::t_btree_iii__0_1_2__110__111 +namespace souffle::t_btree_ii__0_1__11 { +using namespace souffle; +struct Type { +static constexpr Relation::arity_type Arity = 2; +using t_tuple = Tuple; +struct t_comparator_0{ + int operator()(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0])) ? -1 : (ramBitCast(a[0]) > ramBitCast(b[0])) ? 1 :((ramBitCast(a[1]) < ramBitCast(b[1])) ? -1 : (ramBitCast(a[1]) > ramBitCast(b[1])) ? 1 :(0)); + } +bool less(const t_tuple& a, const t_tuple& b) const { + return (ramBitCast(a[0]) < ramBitCast(b[0]))|| ((ramBitCast(a[0]) == ramBitCast(b[0])) && ((ramBitCast(a[1]) < ramBitCast(b[1])))); + } +bool equal(const t_tuple& a, const t_tuple& b) const { +return (ramBitCast(a[0]) == ramBitCast(b[0]))&&(ramBitCast(a[1]) == ramBitCast(b[1])); + } +}; +using t_ind_0 = btree_set; +t_ind_0 ind_0; +using iterator = t_ind_0::iterator; +struct context { +t_ind_0::operation_hints hints_0_lower; +t_ind_0::operation_hints hints_0_upper; +}; +context createContext() { return context(); } +bool insert(const t_tuple& t); +bool insert(const t_tuple& t, context& h); +bool insert(const RamDomain* ramDomain); +bool insert(RamDomain a0,RamDomain a1); +bool contains(const t_tuple& t, context& h) const; +bool contains(const t_tuple& t) const; +std::size_t size() const; +iterator find(const t_tuple& t, context& h) const; +iterator find(const t_tuple& t) const; +range lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const; +range lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */) const; +range lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper, context& h) const; +range lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper) const; +bool empty() const; +std::vector> partition() const; +void purge(); +iterator begin() const; +iterator end() const; +void printStatistics(std::ostream& o) const; +}; +} // namespace souffle::t_btree_ii__0_1__11 +namespace souffle::t_btree_ii__0_1__11 { +using namespace souffle; +using t_ind_0 = Type::t_ind_0; +using iterator = Type::iterator; +using context = Type::context; +bool Type::insert(const t_tuple& t) { +context h; +return insert(t, h); +} +bool Type::insert(const t_tuple& t, context& h) { +if (ind_0.insert(t, h.hints_0_lower)) { +return true; +} else return false; +} +bool Type::insert(const RamDomain* ramDomain) { +RamDomain data[2]; +std::copy(ramDomain, ramDomain + 2, data); +const t_tuple& tuple = reinterpret_cast(data); +context h; +return insert(tuple, h); +} +bool Type::insert(RamDomain a0,RamDomain a1) { +RamDomain data[2] = {a0,a1}; +return insert(data); +} +bool Type::contains(const t_tuple& t, context& h) const { +return ind_0.contains(t, h.hints_0_lower); +} +bool Type::contains(const t_tuple& t) const { +context h; +return contains(t, h); +} +std::size_t Type::size() const { +return ind_0.size(); +} +iterator Type::find(const t_tuple& t, context& h) const { +return ind_0.find(t, h.hints_0_lower); +} +iterator Type::find(const t_tuple& t) const { +context h; +return find(t, h); +} +range Type::lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */, context& /* h */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_00(const t_tuple& /* lower */, const t_tuple& /* upper */) const { +return range(ind_0.begin(),ind_0.end()); +} +range Type::lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper, context& h) const { +t_comparator_0 comparator; +int cmp = comparator(lower, upper); +if (cmp == 0) { + auto pos = ind_0.find(lower, h.hints_0_lower); + auto fin = ind_0.end(); + if (pos != fin) {fin = pos; ++fin;} + return make_range(pos, fin); +} +if (cmp > 0) { + return make_range(ind_0.end(), ind_0.end()); +} +return make_range(ind_0.lower_bound(lower, h.hints_0_lower), ind_0.upper_bound(upper, h.hints_0_upper)); +} +range Type::lowerUpperRange_11(const t_tuple& lower, const t_tuple& upper) const { +context h; +return lowerUpperRange_11(lower,upper,h); +} +bool Type::empty() const { +return ind_0.empty(); +} +std::vector> Type::partition() const { +return ind_0.getChunks(400); +} +void Type::purge() { +ind_0.clear(); +} +iterator Type::begin() const { +return ind_0.begin(); +} +iterator Type::end() const { +return ind_0.end(); +} +void Type::printStatistics(std::ostream& o) const { +o << " arity 2 direct b-tree index 0 lex-order [0,1]\n"; +ind_0.printStats(o); +} +} // namespace souffle::t_btree_ii__0_1__11 +namespace souffle { +using namespace souffle; +class Stratum_assign_e0d78e44f4df6411 { +public: + Stratum_assign_e0d78e44f4df6411(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iiii__0_1_2_3__1110__1111__1100::Type& rel_assign_e4bb6e0824a16a37); +void run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret); +private: +SymbolTable& symTable; +RecordTable& recordTable; +ConcurrentCache& regexCache; +bool& pruneImdtRels; +bool& performIO; +SignalHandler*& signalHandler; +std::atomic& iter; +std::atomic& ctr; +std::string& inputDirectory; +std::string& outputDirectory; +t_btree_iiii__0_1_2_3__1110__1111__1100::Type* rel_assign_e4bb6e0824a16a37; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Stratum_assign_e0d78e44f4df6411::Stratum_assign_e0d78e44f4df6411(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iiii__0_1_2_3__1110__1111__1100::Type& rel_assign_e4bb6e0824a16a37): +symTable(symTable), +recordTable(recordTable), +regexCache(regexCache), +pruneImdtRels(pruneImdtRels), +performIO(performIO), +signalHandler(signalHandler), +iter(iter), +ctr(ctr), +inputDirectory(inputDirectory), +outputDirectory(outputDirectory), +rel_assign_e4bb6e0824a16a37(&rel_assign_e4bb6e0824a16a37){ +} + +void Stratum_assign_e0d78e44f4df6411::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ +if (performIO) { +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\ts\tr\tv"},{"auxArity","0"},{"fact-dir","."},{"name","assign"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"params\": [\"f\", \"s\", \"r\", \"v\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 4, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\", \"+:Value\"]}}"}}); +if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_assign_e4bb6e0824a16a37); +} catch (std::exception& e) {std::cerr << "Error loading assign data: " << e.what() << '\n'; +exit(1); +} +} +} + +} // namespace souffle + +namespace souffle { +using namespace souffle; +class Stratum_bind_8b0da46e2379b6cd { +public: + Stratum_bind_8b0da46e2379b6cd(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iiiii__0_1_2_3_4__11111__11110::Type& rel_bind_c9210fdc63280a40); +void run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret); +private: +SymbolTable& symTable; +RecordTable& recordTable; +ConcurrentCache& regexCache; +bool& pruneImdtRels; +bool& performIO; +SignalHandler*& signalHandler; +std::atomic& iter; +std::atomic& ctr; +std::string& inputDirectory; +std::string& outputDirectory; +t_btree_iiiii__0_1_2_3_4__11111__11110::Type* rel_bind_c9210fdc63280a40; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Stratum_bind_8b0da46e2379b6cd::Stratum_bind_8b0da46e2379b6cd(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iiiii__0_1_2_3_4__11111__11110::Type& rel_bind_c9210fdc63280a40): +symTable(symTable), +recordTable(recordTable), +regexCache(regexCache), +pruneImdtRels(pruneImdtRels), +performIO(performIO), +signalHandler(signalHandler), +iter(iter), +ctr(ctr), +inputDirectory(inputDirectory), +outputDirectory(outputDirectory), +rel_bind_c9210fdc63280a40(&rel_bind_c9210fdc63280a40){ +} + +void Stratum_bind_8b0da46e2379b6cd::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ +if (performIO) { +try {std::map directiveMap({{"IO","file"},{"attributeNames","caller\ts\tr\tcallee\tparam"},{"auxArity","0"},{"fact-dir","."},{"name","bind"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 5, \"params\": [\"caller\", \"s\", \"r\", \"callee\", \"param\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 5, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\", \"s:Function\", \"s:symbol\"]}}"}}); +if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_bind_c9210fdc63280a40); +} catch (std::exception& e) {std::cerr << "Error loading bind data: " << e.what() << '\n'; +exit(1); +} +} +} + +} // namespace souffle + +namespace souffle { +using namespace souffle; +class Stratum_call_104fac07831e2229 { +public: + Stratum_call_104fac07831e2229(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iii__2_0_1__001__111::Type& rel_call_ee1d8972d66cc25f); +void run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret); +private: +SymbolTable& symTable; +RecordTable& recordTable; +ConcurrentCache& regexCache; +bool& pruneImdtRels; +bool& performIO; +SignalHandler*& signalHandler; +std::atomic& iter; +std::atomic& ctr; +std::string& inputDirectory; +std::string& outputDirectory; +t_btree_iii__2_0_1__001__111::Type* rel_call_ee1d8972d66cc25f; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Stratum_call_104fac07831e2229::Stratum_call_104fac07831e2229(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iii__2_0_1__001__111::Type& rel_call_ee1d8972d66cc25f): +symTable(symTable), +recordTable(recordTable), +regexCache(regexCache), +pruneImdtRels(pruneImdtRels), +performIO(performIO), +signalHandler(signalHandler), +iter(iter), +ctr(ctr), +inputDirectory(inputDirectory), +outputDirectory(outputDirectory), +rel_call_ee1d8972d66cc25f(&rel_call_ee1d8972d66cc25f){ +} + +void Stratum_call_104fac07831e2229::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ +if (performIO) { +try {std::map directiveMap({{"IO","file"},{"attributeNames","caller\ts\tcallee"},{"auxArity","0"},{"fact-dir","."},{"name","call"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"caller\", \"s\", \"callee\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:Function\", \"i:Statement\", \"s:Function\"]}}"}}); +if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_call_ee1d8972d66cc25f); +} catch (std::exception& e) {std::cerr << "Error loading call data: " << e.what() << '\n'; +exit(1); +} +} +} + +} // namespace souffle + +namespace souffle { +using namespace souffle; +class Stratum_cf_edge_c2ae152829fd6f1f { +public: + Stratum_cf_edge_c2ae152829fd6f1f(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iii__0_1_2__111::Type& rel_cf_edge_4931a04c8c74bb72); +void run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret); +private: +SymbolTable& symTable; +RecordTable& recordTable; +ConcurrentCache& regexCache; +bool& pruneImdtRels; +bool& performIO; +SignalHandler*& signalHandler; +std::atomic& iter; +std::atomic& ctr; +std::string& inputDirectory; +std::string& outputDirectory; +t_btree_iii__0_1_2__111::Type* rel_cf_edge_4931a04c8c74bb72; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Stratum_cf_edge_c2ae152829fd6f1f::Stratum_cf_edge_c2ae152829fd6f1f(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iii__0_1_2__111::Type& rel_cf_edge_4931a04c8c74bb72): +symTable(symTable), +recordTable(recordTable), +regexCache(regexCache), +pruneImdtRels(pruneImdtRels), +performIO(performIO), +signalHandler(signalHandler), +iter(iter), +ctr(ctr), +inputDirectory(inputDirectory), +outputDirectory(outputDirectory), +rel_cf_edge_4931a04c8c74bb72(&rel_cf_edge_4931a04c8c74bb72){ +} + +void Stratum_cf_edge_c2ae152829fd6f1f::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ +if (performIO) { +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\ts1\ts2"},{"auxArity","0"},{"fact-dir","."},{"name","cf_edge"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"f\", \"s1\", \"s2\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:Function\", \"i:Statement\", \"i:Statement\"]}}"}}); +if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_cf_edge_4931a04c8c74bb72); +} catch (std::exception& e) {std::cerr << "Error loading cf_edge data: " << e.what() << '\n'; +exit(1); +} +} +} + +} // namespace souffle + +namespace souffle { +using namespace souffle; +class Stratum_live_vars_in_a363f2025538826a { +public: + Stratum_live_vars_in_a363f2025538826a(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iii__0_1_2__110__111::Type& rel_delta_live_vars_in_fccc4ee6df066f63,t_btree_iii__0_1_2__111::Type& rel_delta_live_vars_out_acc66913cea62d16,t_btree_iii__0_1_2__110__111::Type& rel_new_live_vars_in_0b01be53183b2351,t_btree_iii__0_1_2__111::Type& rel_new_live_vars_out_2d78073638bb3740,t_btree_iiii__0_1_2_3__1110__1111__1100::Type& rel_assign_e4bb6e0824a16a37,t_btree_iii__0_1_2__111::Type& rel_cf_edge_4931a04c8c74bb72,t_btree_iii__0_1_2__111::Type& rel_live_vars_in_0b002b95687eda95,t_btree_iii__0_1_2__110__111::Type& rel_live_vars_out_f94306e028b67aa4,t_btree_iii__0_1_2__111::Type& rel_use_e955e932f22dad4d); +void run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret); +private: +SymbolTable& symTable; +RecordTable& recordTable; +ConcurrentCache& regexCache; +bool& pruneImdtRels; +bool& performIO; +SignalHandler*& signalHandler; +std::atomic& iter; +std::atomic& ctr; +std::string& inputDirectory; +std::string& outputDirectory; +t_btree_iii__0_1_2__110__111::Type* rel_delta_live_vars_in_fccc4ee6df066f63; +t_btree_iii__0_1_2__111::Type* rel_delta_live_vars_out_acc66913cea62d16; +t_btree_iii__0_1_2__110__111::Type* rel_new_live_vars_in_0b01be53183b2351; +t_btree_iii__0_1_2__111::Type* rel_new_live_vars_out_2d78073638bb3740; +t_btree_iiii__0_1_2_3__1110__1111__1100::Type* rel_assign_e4bb6e0824a16a37; +t_btree_iii__0_1_2__111::Type* rel_cf_edge_4931a04c8c74bb72; +t_btree_iii__0_1_2__111::Type* rel_live_vars_in_0b002b95687eda95; +t_btree_iii__0_1_2__110__111::Type* rel_live_vars_out_f94306e028b67aa4; +t_btree_iii__0_1_2__111::Type* rel_use_e955e932f22dad4d; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Stratum_live_vars_in_a363f2025538826a::Stratum_live_vars_in_a363f2025538826a(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iii__0_1_2__110__111::Type& rel_delta_live_vars_in_fccc4ee6df066f63,t_btree_iii__0_1_2__111::Type& rel_delta_live_vars_out_acc66913cea62d16,t_btree_iii__0_1_2__110__111::Type& rel_new_live_vars_in_0b01be53183b2351,t_btree_iii__0_1_2__111::Type& rel_new_live_vars_out_2d78073638bb3740,t_btree_iiii__0_1_2_3__1110__1111__1100::Type& rel_assign_e4bb6e0824a16a37,t_btree_iii__0_1_2__111::Type& rel_cf_edge_4931a04c8c74bb72,t_btree_iii__0_1_2__111::Type& rel_live_vars_in_0b002b95687eda95,t_btree_iii__0_1_2__110__111::Type& rel_live_vars_out_f94306e028b67aa4,t_btree_iii__0_1_2__111::Type& rel_use_e955e932f22dad4d): +symTable(symTable), +recordTable(recordTable), +regexCache(regexCache), +pruneImdtRels(pruneImdtRels), +performIO(performIO), +signalHandler(signalHandler), +iter(iter), +ctr(ctr), +inputDirectory(inputDirectory), +outputDirectory(outputDirectory), +rel_delta_live_vars_in_fccc4ee6df066f63(&rel_delta_live_vars_in_fccc4ee6df066f63), +rel_delta_live_vars_out_acc66913cea62d16(&rel_delta_live_vars_out_acc66913cea62d16), +rel_new_live_vars_in_0b01be53183b2351(&rel_new_live_vars_in_0b01be53183b2351), +rel_new_live_vars_out_2d78073638bb3740(&rel_new_live_vars_out_2d78073638bb3740), +rel_assign_e4bb6e0824a16a37(&rel_assign_e4bb6e0824a16a37), +rel_cf_edge_4931a04c8c74bb72(&rel_cf_edge_4931a04c8c74bb72), +rel_live_vars_in_0b002b95687eda95(&rel_live_vars_in_0b002b95687eda95), +rel_live_vars_out_f94306e028b67aa4(&rel_live_vars_out_f94306e028b67aa4), +rel_use_e955e932f22dad4d(&rel_use_e955e932f22dad4d){ +} + +void Stratum_live_vars_in_a363f2025538826a::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ +signalHandler->setMsg(R"_(live_vars_in(f,s,r) :- + use(f,s,r). +in file dataflow.dl [50:1-50:39])_"); +if(!(rel_use_e955e932f22dad4d->empty())) { +[&](){ +CREATE_OP_CONTEXT(rel_live_vars_in_0b002b95687eda95_op_ctxt,rel_live_vars_in_0b002b95687eda95->createContext()); +CREATE_OP_CONTEXT(rel_use_e955e932f22dad4d_op_ctxt,rel_use_e955e932f22dad4d->createContext()); +for(const auto& env0 : *rel_use_e955e932f22dad4d) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env0[2])}}; +rel_live_vars_in_0b002b95687eda95->insert(tuple,READ_OP_CONTEXT(rel_live_vars_in_0b002b95687eda95_op_ctxt)); +} +} +();} +[&](){ +CREATE_OP_CONTEXT(rel_delta_live_vars_in_fccc4ee6df066f63_op_ctxt,rel_delta_live_vars_in_fccc4ee6df066f63->createContext()); +CREATE_OP_CONTEXT(rel_live_vars_in_0b002b95687eda95_op_ctxt,rel_live_vars_in_0b002b95687eda95->createContext()); +for(const auto& env0 : *rel_live_vars_in_0b002b95687eda95) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env0[2])}}; +rel_delta_live_vars_in_fccc4ee6df066f63->insert(tuple,READ_OP_CONTEXT(rel_delta_live_vars_in_fccc4ee6df066f63_op_ctxt)); +} +} +();[&](){ +CREATE_OP_CONTEXT(rel_delta_live_vars_out_acc66913cea62d16_op_ctxt,rel_delta_live_vars_out_acc66913cea62d16->createContext()); +CREATE_OP_CONTEXT(rel_live_vars_out_f94306e028b67aa4_op_ctxt,rel_live_vars_out_f94306e028b67aa4->createContext()); +for(const auto& env0 : *rel_live_vars_out_f94306e028b67aa4) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env0[2])}}; +rel_delta_live_vars_out_acc66913cea62d16->insert(tuple,READ_OP_CONTEXT(rel_delta_live_vars_out_acc66913cea62d16_op_ctxt)); +} +} +();auto loop_counter = RamUnsigned(1); +iter = 0; +for(;;) { +signalHandler->setMsg(R"_(live_vars_in(f,s,r) :- + !assign(f,s,r,_), + live_vars_out(f,s,r). +in file dataflow.dl [52:1-52:70])_"); +if(!(rel_delta_live_vars_out_acc66913cea62d16->empty())) { +[&](){ +CREATE_OP_CONTEXT(rel_delta_live_vars_out_acc66913cea62d16_op_ctxt,rel_delta_live_vars_out_acc66913cea62d16->createContext()); +CREATE_OP_CONTEXT(rel_new_live_vars_in_0b01be53183b2351_op_ctxt,rel_new_live_vars_in_0b01be53183b2351->createContext()); +CREATE_OP_CONTEXT(rel_assign_e4bb6e0824a16a37_op_ctxt,rel_assign_e4bb6e0824a16a37->createContext()); +CREATE_OP_CONTEXT(rel_live_vars_in_0b002b95687eda95_op_ctxt,rel_live_vars_in_0b002b95687eda95->createContext()); +for(const auto& env0 : *rel_delta_live_vars_out_acc66913cea62d16) { +if( !(rel_live_vars_in_0b002b95687eda95->contains(Tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env0[2])}},READ_OP_CONTEXT(rel_live_vars_in_0b002b95687eda95_op_ctxt))) && !(!rel_assign_e4bb6e0824a16a37->lowerUpperRange_1110(Tuple{{ramBitCast(env0[0]), ramBitCast(env0[1]), ramBitCast(env0[2]), ramBitCast(MIN_RAM_SIGNED)}},Tuple{{ramBitCast(env0[0]), ramBitCast(env0[1]), ramBitCast(env0[2]), ramBitCast(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_assign_e4bb6e0824a16a37_op_ctxt)).empty())) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env0[2])}}; +rel_new_live_vars_in_0b01be53183b2351->insert(tuple,READ_OP_CONTEXT(rel_new_live_vars_in_0b01be53183b2351_op_ctxt)); +} +} +} +();} +signalHandler->setMsg(R"_(live_vars_out(f,s1,r) :- + cf_edge(f,s1,s2), + live_vars_in(f,s2,r). +in file dataflow.dl [56:1-56:71])_"); +if(!(rel_cf_edge_4931a04c8c74bb72->empty()) && !(rel_delta_live_vars_in_fccc4ee6df066f63->empty())) { +[&](){ +CREATE_OP_CONTEXT(rel_delta_live_vars_in_fccc4ee6df066f63_op_ctxt,rel_delta_live_vars_in_fccc4ee6df066f63->createContext()); +CREATE_OP_CONTEXT(rel_new_live_vars_out_2d78073638bb3740_op_ctxt,rel_new_live_vars_out_2d78073638bb3740->createContext()); +CREATE_OP_CONTEXT(rel_cf_edge_4931a04c8c74bb72_op_ctxt,rel_cf_edge_4931a04c8c74bb72->createContext()); +CREATE_OP_CONTEXT(rel_live_vars_out_f94306e028b67aa4_op_ctxt,rel_live_vars_out_f94306e028b67aa4->createContext()); +for(const auto& env0 : *rel_cf_edge_4931a04c8c74bb72) { +auto range = rel_delta_live_vars_in_fccc4ee6df066f63->lowerUpperRange_110(Tuple{{ramBitCast(env0[0]), ramBitCast(env0[2]), ramBitCast(MIN_RAM_SIGNED)}},Tuple{{ramBitCast(env0[0]), ramBitCast(env0[2]), ramBitCast(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_delta_live_vars_in_fccc4ee6df066f63_op_ctxt)); +for(const auto& env1 : range) { +if( !(rel_live_vars_out_f94306e028b67aa4->contains(Tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env1[2])}},READ_OP_CONTEXT(rel_live_vars_out_f94306e028b67aa4_op_ctxt)))) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env1[2])}}; +rel_new_live_vars_out_2d78073638bb3740->insert(tuple,READ_OP_CONTEXT(rel_new_live_vars_out_2d78073638bb3740_op_ctxt)); +} +} +} +} +();} +if(rel_new_live_vars_in_0b01be53183b2351->empty() && rel_new_live_vars_out_2d78073638bb3740->empty()) break; +[&](){ +CREATE_OP_CONTEXT(rel_new_live_vars_in_0b01be53183b2351_op_ctxt,rel_new_live_vars_in_0b01be53183b2351->createContext()); +CREATE_OP_CONTEXT(rel_live_vars_in_0b002b95687eda95_op_ctxt,rel_live_vars_in_0b002b95687eda95->createContext()); +for(const auto& env0 : *rel_new_live_vars_in_0b01be53183b2351) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env0[2])}}; +rel_live_vars_in_0b002b95687eda95->insert(tuple,READ_OP_CONTEXT(rel_live_vars_in_0b002b95687eda95_op_ctxt)); +} +} +();std::swap(rel_delta_live_vars_in_fccc4ee6df066f63, rel_new_live_vars_in_0b01be53183b2351); +rel_new_live_vars_in_0b01be53183b2351->purge(); +[&](){ +CREATE_OP_CONTEXT(rel_new_live_vars_out_2d78073638bb3740_op_ctxt,rel_new_live_vars_out_2d78073638bb3740->createContext()); +CREATE_OP_CONTEXT(rel_live_vars_out_f94306e028b67aa4_op_ctxt,rel_live_vars_out_f94306e028b67aa4->createContext()); +for(const auto& env0 : *rel_new_live_vars_out_2d78073638bb3740) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1]),ramBitCast(env0[2])}}; +rel_live_vars_out_f94306e028b67aa4->insert(tuple,READ_OP_CONTEXT(rel_live_vars_out_f94306e028b67aa4_op_ctxt)); +} +} +();std::swap(rel_delta_live_vars_out_acc66913cea62d16, rel_new_live_vars_out_2d78073638bb3740); +rel_new_live_vars_out_2d78073638bb3740->purge(); +loop_counter = (ramBitCast(loop_counter) + ramBitCast(RamUnsigned(1))); +iter++; +} +iter = 0; +rel_delta_live_vars_in_fccc4ee6df066f63->purge(); +rel_new_live_vars_in_0b01be53183b2351->purge(); +rel_delta_live_vars_out_acc66913cea62d16->purge(); +rel_new_live_vars_out_2d78073638bb3740->purge(); +if (pruneImdtRels) rel_cf_edge_4931a04c8c74bb72->purge(); +if (pruneImdtRels) rel_live_vars_in_0b002b95687eda95->purge(); +if (pruneImdtRels) rel_use_e955e932f22dad4d->purge(); +} + +} // namespace souffle + +namespace souffle { +using namespace souffle; +class Stratum_might_collect_beadc513d07ff032 { +public: + Stratum_might_collect_beadc513d07ff032(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_ii__0_1__11__10::Type& rel_delta_might_collect_d651f71586aafe59,t_btree_ii__0_1__11__10::Type& rel_new_might_collect_5d48ef45a97e4618,t_btree_iii__2_0_1__001__111::Type& rel_call_ee1d8972d66cc25f,t_btree_ii__0_1__11__10::Type& rel_might_collect_ef1d0b06d36e4ddc); +void run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret); +private: +SymbolTable& symTable; +RecordTable& recordTable; +ConcurrentCache& regexCache; +bool& pruneImdtRels; +bool& performIO; +SignalHandler*& signalHandler; +std::atomic& iter; +std::atomic& ctr; +std::string& inputDirectory; +std::string& outputDirectory; +t_btree_ii__0_1__11__10::Type* rel_delta_might_collect_d651f71586aafe59; +t_btree_ii__0_1__11__10::Type* rel_new_might_collect_5d48ef45a97e4618; +t_btree_iii__2_0_1__001__111::Type* rel_call_ee1d8972d66cc25f; +t_btree_ii__0_1__11__10::Type* rel_might_collect_ef1d0b06d36e4ddc; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Stratum_might_collect_beadc513d07ff032::Stratum_might_collect_beadc513d07ff032(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_ii__0_1__11__10::Type& rel_delta_might_collect_d651f71586aafe59,t_btree_ii__0_1__11__10::Type& rel_new_might_collect_5d48ef45a97e4618,t_btree_iii__2_0_1__001__111::Type& rel_call_ee1d8972d66cc25f,t_btree_ii__0_1__11__10::Type& rel_might_collect_ef1d0b06d36e4ddc): +symTable(symTable), +recordTable(recordTable), +regexCache(regexCache), +pruneImdtRels(pruneImdtRels), +performIO(performIO), +signalHandler(signalHandler), +iter(iter), +ctr(ctr), +inputDirectory(inputDirectory), +outputDirectory(outputDirectory), +rel_delta_might_collect_d651f71586aafe59(&rel_delta_might_collect_d651f71586aafe59), +rel_new_might_collect_5d48ef45a97e4618(&rel_new_might_collect_5d48ef45a97e4618), +rel_call_ee1d8972d66cc25f(&rel_call_ee1d8972d66cc25f), +rel_might_collect_ef1d0b06d36e4ddc(&rel_might_collect_ef1d0b06d36e4ddc){ +} + +void Stratum_might_collect_beadc513d07ff032::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ +signalHandler->setMsg(R"_(might_collect("mylib.MaybeCollect",0). +in file call-graph.dl [14:1-14:40])_"); +[&](){ +CREATE_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt,rel_might_collect_ef1d0b06d36e4ddc->createContext()); +Tuple tuple{{ramBitCast(RamSigned(0)),ramBitCast(RamSigned(0))}}; +rel_might_collect_ef1d0b06d36e4ddc->insert(tuple,READ_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt)); +} +();signalHandler->setMsg(R"_(might_collect(f,s) :- + call(f,s,"mylib.MaybeCollect"). +in file call-graph.dl [15:1-15:57])_"); +if(!(rel_call_ee1d8972d66cc25f->empty())) { +[&](){ +CREATE_OP_CONTEXT(rel_call_ee1d8972d66cc25f_op_ctxt,rel_call_ee1d8972d66cc25f->createContext()); +CREATE_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt,rel_might_collect_ef1d0b06d36e4ddc->createContext()); +auto range = rel_call_ee1d8972d66cc25f->lowerUpperRange_001(Tuple{{ramBitCast(MIN_RAM_SIGNED), ramBitCast(MIN_RAM_SIGNED), ramBitCast(RamSigned(0))}},Tuple{{ramBitCast(MAX_RAM_SIGNED), ramBitCast(MAX_RAM_SIGNED), ramBitCast(RamSigned(0))}},READ_OP_CONTEXT(rel_call_ee1d8972d66cc25f_op_ctxt)); +for(const auto& env0 : range) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1])}}; +rel_might_collect_ef1d0b06d36e4ddc->insert(tuple,READ_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt)); +} +} +();} +[&](){ +CREATE_OP_CONTEXT(rel_delta_might_collect_d651f71586aafe59_op_ctxt,rel_delta_might_collect_d651f71586aafe59->createContext()); +CREATE_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt,rel_might_collect_ef1d0b06d36e4ddc->createContext()); +for(const auto& env0 : *rel_might_collect_ef1d0b06d36e4ddc) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1])}}; +rel_delta_might_collect_d651f71586aafe59->insert(tuple,READ_OP_CONTEXT(rel_delta_might_collect_d651f71586aafe59_op_ctxt)); +} +} +();auto loop_counter = RamUnsigned(1); +iter = 0; +for(;;) { +signalHandler->setMsg(R"_(might_collect(f,s) :- + call(f,s,g), + might_collect(g,_). +in file call-graph.dl [16:1-16:59])_"); +if(!(rel_call_ee1d8972d66cc25f->empty()) && !(rel_delta_might_collect_d651f71586aafe59->empty())) { +[&](){ +CREATE_OP_CONTEXT(rel_delta_might_collect_d651f71586aafe59_op_ctxt,rel_delta_might_collect_d651f71586aafe59->createContext()); +CREATE_OP_CONTEXT(rel_new_might_collect_5d48ef45a97e4618_op_ctxt,rel_new_might_collect_5d48ef45a97e4618->createContext()); +CREATE_OP_CONTEXT(rel_call_ee1d8972d66cc25f_op_ctxt,rel_call_ee1d8972d66cc25f->createContext()); +CREATE_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt,rel_might_collect_ef1d0b06d36e4ddc->createContext()); +for(const auto& env0 : *rel_call_ee1d8972d66cc25f) { +if( !rel_delta_might_collect_d651f71586aafe59->lowerUpperRange_10(Tuple{{ramBitCast(env0[2]), ramBitCast(MIN_RAM_SIGNED)}},Tuple{{ramBitCast(env0[2]), ramBitCast(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_delta_might_collect_d651f71586aafe59_op_ctxt)).empty() && !(rel_might_collect_ef1d0b06d36e4ddc->contains(Tuple{{ramBitCast(env0[0]),ramBitCast(env0[1])}},READ_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt)))) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1])}}; +rel_new_might_collect_5d48ef45a97e4618->insert(tuple,READ_OP_CONTEXT(rel_new_might_collect_5d48ef45a97e4618_op_ctxt)); +} +} +} +();} +if(rel_new_might_collect_5d48ef45a97e4618->empty()) break; +[&](){ +CREATE_OP_CONTEXT(rel_new_might_collect_5d48ef45a97e4618_op_ctxt,rel_new_might_collect_5d48ef45a97e4618->createContext()); +CREATE_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt,rel_might_collect_ef1d0b06d36e4ddc->createContext()); +for(const auto& env0 : *rel_new_might_collect_5d48ef45a97e4618) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env0[1])}}; +rel_might_collect_ef1d0b06d36e4ddc->insert(tuple,READ_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt)); +} +} +();std::swap(rel_delta_might_collect_d651f71586aafe59, rel_new_might_collect_5d48ef45a97e4618); +rel_new_might_collect_5d48ef45a97e4618->purge(); +loop_counter = (ramBitCast(loop_counter) + ramBitCast(RamUnsigned(1))); +iter++; +} +iter = 0; +rel_delta_might_collect_d651f71586aafe59->purge(); +rel_new_might_collect_5d48ef45a97e4618->purge(); +if (performIO) { +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\ts"},{"auxArity","0"},{"name","might_collect"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"f\", \"s\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:Function\", \"i:Statement\"]}}"}}); +if (outputDirectory == "-"){directiveMap["IO"] = "stdout"; directiveMap["headers"] = "true";} +else if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;} +IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_might_collect_ef1d0b06d36e4ddc); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +} +} + +} // namespace souffle + +namespace souffle { +using namespace souffle; +class Stratum_stack_root_vars_4df5b9c3cd2e7586 { +public: + Stratum_stack_root_vars_4df5b9c3cd2e7586(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iiii__0_1_2_3__1110__1111__1100::Type& rel_assign_e4bb6e0824a16a37,t_btree_iiiii__0_1_2_3_4__11111__11110::Type& rel_bind_c9210fdc63280a40,t_btree_iii__2_0_1__001__111::Type& rel_call_ee1d8972d66cc25f,t_btree_iii__0_1_2__110__111::Type& rel_live_vars_out_f94306e028b67aa4,t_btree_ii__0_1__11__10::Type& rel_might_collect_ef1d0b06d36e4ddc,t_btree_ii__0_1__11::Type& rel_stack_root_vars_a138611bd47fd3ff); +void run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret); +private: +SymbolTable& symTable; +RecordTable& recordTable; +ConcurrentCache& regexCache; +bool& pruneImdtRels; +bool& performIO; +SignalHandler*& signalHandler; +std::atomic& iter; +std::atomic& ctr; +std::string& inputDirectory; +std::string& outputDirectory; +t_btree_iiii__0_1_2_3__1110__1111__1100::Type* rel_assign_e4bb6e0824a16a37; +t_btree_iiiii__0_1_2_3_4__11111__11110::Type* rel_bind_c9210fdc63280a40; +t_btree_iii__2_0_1__001__111::Type* rel_call_ee1d8972d66cc25f; +t_btree_iii__0_1_2__110__111::Type* rel_live_vars_out_f94306e028b67aa4; +t_btree_ii__0_1__11__10::Type* rel_might_collect_ef1d0b06d36e4ddc; +t_btree_ii__0_1__11::Type* rel_stack_root_vars_a138611bd47fd3ff; +std::vector regexes; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Stratum_stack_root_vars_4df5b9c3cd2e7586::Stratum_stack_root_vars_4df5b9c3cd2e7586(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iiii__0_1_2_3__1110__1111__1100::Type& rel_assign_e4bb6e0824a16a37,t_btree_iiiii__0_1_2_3_4__11111__11110::Type& rel_bind_c9210fdc63280a40,t_btree_iii__2_0_1__001__111::Type& rel_call_ee1d8972d66cc25f,t_btree_iii__0_1_2__110__111::Type& rel_live_vars_out_f94306e028b67aa4,t_btree_ii__0_1__11__10::Type& rel_might_collect_ef1d0b06d36e4ddc,t_btree_ii__0_1__11::Type& rel_stack_root_vars_a138611bd47fd3ff): +symTable(symTable), +recordTable(recordTable), +regexCache(regexCache), +pruneImdtRels(pruneImdtRels), +performIO(performIO), +signalHandler(signalHandler), +iter(iter), +ctr(ctr), +inputDirectory(inputDirectory), +outputDirectory(outputDirectory), +rel_assign_e4bb6e0824a16a37(&rel_assign_e4bb6e0824a16a37), +rel_bind_c9210fdc63280a40(&rel_bind_c9210fdc63280a40), +rel_call_ee1d8972d66cc25f(&rel_call_ee1d8972d66cc25f), +rel_live_vars_out_f94306e028b67aa4(&rel_live_vars_out_f94306e028b67aa4), +rel_might_collect_ef1d0b06d36e4ddc(&rel_might_collect_ef1d0b06d36e4ddc), +rel_stack_root_vars_a138611bd47fd3ff(&rel_stack_root_vars_a138611bd47fd3ff), +regexes({ + std::regex(".*ctx_.*__init__"), +}){ +} + +void Stratum_stack_root_vars_4df5b9c3cd2e7586::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ +signalHandler->setMsg(R"_(stack_root_vars(f,r) :- + call(f,s,g), + might_collect(g,_), + !bind(f,s,r,g,_), + live_vars_out(f,s,r). +in file dataflow.dl [60:1-60:107])_"); +if(!(rel_might_collect_ef1d0b06d36e4ddc->empty()) && !(rel_live_vars_out_f94306e028b67aa4->empty()) && !(rel_call_ee1d8972d66cc25f->empty())) { +[&](){ +CREATE_OP_CONTEXT(rel_bind_c9210fdc63280a40_op_ctxt,rel_bind_c9210fdc63280a40->createContext()); +CREATE_OP_CONTEXT(rel_call_ee1d8972d66cc25f_op_ctxt,rel_call_ee1d8972d66cc25f->createContext()); +CREATE_OP_CONTEXT(rel_live_vars_out_f94306e028b67aa4_op_ctxt,rel_live_vars_out_f94306e028b67aa4->createContext()); +CREATE_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt,rel_might_collect_ef1d0b06d36e4ddc->createContext()); +CREATE_OP_CONTEXT(rel_stack_root_vars_a138611bd47fd3ff_op_ctxt,rel_stack_root_vars_a138611bd47fd3ff->createContext()); +for(const auto& env0 : *rel_call_ee1d8972d66cc25f) { +if( !rel_might_collect_ef1d0b06d36e4ddc->lowerUpperRange_10(Tuple{{ramBitCast(env0[2]), ramBitCast(MIN_RAM_SIGNED)}},Tuple{{ramBitCast(env0[2]), ramBitCast(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt)).empty()) { +auto range = rel_live_vars_out_f94306e028b67aa4->lowerUpperRange_110(Tuple{{ramBitCast(env0[0]), ramBitCast(env0[1]), ramBitCast(MIN_RAM_SIGNED)}},Tuple{{ramBitCast(env0[0]), ramBitCast(env0[1]), ramBitCast(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_live_vars_out_f94306e028b67aa4_op_ctxt)); +for(const auto& env1 : range) { +if( !(!rel_bind_c9210fdc63280a40->lowerUpperRange_11110(Tuple{{ramBitCast(env0[0]), ramBitCast(env0[1]), ramBitCast(env1[2]), ramBitCast(env0[2]), ramBitCast(MIN_RAM_SIGNED)}},Tuple{{ramBitCast(env0[0]), ramBitCast(env0[1]), ramBitCast(env1[2]), ramBitCast(env0[2]), ramBitCast(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_bind_c9210fdc63280a40_op_ctxt)).empty())) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(env1[2])}}; +rel_stack_root_vars_a138611bd47fd3ff->insert(tuple,READ_OP_CONTEXT(rel_stack_root_vars_a138611bd47fd3ff_op_ctxt)); +} +} +} +} +} +();} +signalHandler->setMsg(R"_(stack_root_vars(f,$LocalVariable(f, v)) :- + might_collect(f,_), + assign(f,0,$LocalVariable(f, v),$Empty()). +in file dataflow.dl [64:1-64:111])_"); +if(!(rel_might_collect_ef1d0b06d36e4ddc->empty()) && !(rel_assign_e4bb6e0824a16a37->empty())) { +[&](){ +CREATE_OP_CONTEXT(rel_assign_e4bb6e0824a16a37_op_ctxt,rel_assign_e4bb6e0824a16a37->createContext()); +CREATE_OP_CONTEXT(rel_might_collect_ef1d0b06d36e4ddc_op_ctxt,rel_might_collect_ef1d0b06d36e4ddc->createContext()); +CREATE_OP_CONTEXT(rel_stack_root_vars_a138611bd47fd3ff_op_ctxt,rel_stack_root_vars_a138611bd47fd3ff->createContext()); +for(const auto& env0 : *rel_might_collect_ef1d0b06d36e4ddc) { +auto range = rel_assign_e4bb6e0824a16a37->lowerUpperRange_1100(Tuple{{ramBitCast(env0[0]), ramBitCast(RamSigned(0)), ramBitCast(MIN_RAM_SIGNED), ramBitCast(MIN_RAM_SIGNED)}},Tuple{{ramBitCast(env0[0]), ramBitCast(RamSigned(0)), ramBitCast(MAX_RAM_SIGNED), ramBitCast(MAX_RAM_SIGNED)}},READ_OP_CONTEXT(rel_assign_e4bb6e0824a16a37_op_ctxt)); +for(const auto& env1 : range) { +RamDomain const ref = env1[2]; +if (ref == 0) continue; +const RamDomain *env2 = recordTable.unpack(ref,2); +{ +if( (ramBitCast(env2[0]) == ramBitCast(RamSigned(0)))) { +RamDomain const ref = env2[1]; +if (ref == 0) continue; +const RamDomain *env3 = recordTable.unpack(ref,2); +{ +if( (ramBitCast(env0[0]) == ramBitCast(env3[0]))) { +RamDomain const ref = env1[3]; +if (ref == 0) continue; +const RamDomain *env4 = recordTable.unpack(ref,2); +{ +if( (ramBitCast(env4[0]) == ramBitCast(RamSigned(0)))) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(pack(recordTable,Tuple{{ramBitCast(ramBitCast(RamSigned(0))),ramBitCast(ramBitCast(pack(recordTable,Tuple{{ramBitCast(ramBitCast(env0[0])),ramBitCast(ramBitCast(env3[1]))}} +)))}} +))}}; +rel_stack_root_vars_a138611bd47fd3ff->insert(tuple,READ_OP_CONTEXT(rel_stack_root_vars_a138611bd47fd3ff_op_ctxt)); +} +} +} +} +} +} +} +} +} +();} +signalHandler->setMsg(R"_(stack_root_vars(f,$ObjectMember("self", m)) :- + match(".*ctx_.*__init__", f), + assign(f,_,$ObjectMember("self", m),_). +in file dataflow.dl [67:1-67:121])_"); +if(!(rel_assign_e4bb6e0824a16a37->empty())) { +[&](){ +CREATE_OP_CONTEXT(rel_assign_e4bb6e0824a16a37_op_ctxt,rel_assign_e4bb6e0824a16a37->createContext()); +CREATE_OP_CONTEXT(rel_stack_root_vars_a138611bd47fd3ff_op_ctxt,rel_stack_root_vars_a138611bd47fd3ff->createContext()); +for(const auto& env0 : *rel_assign_e4bb6e0824a16a37) { +if( std::regex_match(symTable.decode(env0[0]), regexes.at(0))) { +RamDomain const ref = env0[2]; +if (ref == 0) continue; +const RamDomain *env1 = recordTable.unpack(ref,2); +{ +if( (ramBitCast(env1[0]) == ramBitCast(RamSigned(1)))) { +RamDomain const ref = env1[1]; +if (ref == 0) continue; +const RamDomain *env2 = recordTable.unpack(ref,2); +{ +if( (ramBitCast(env2[0]) == ramBitCast(RamSigned(1)))) { +Tuple tuple{{ramBitCast(env0[0]),ramBitCast(pack(recordTable,Tuple{{ramBitCast(ramBitCast(RamSigned(1))),ramBitCast(ramBitCast(pack(recordTable,Tuple{{ramBitCast(ramBitCast(RamSigned(1))),ramBitCast(ramBitCast(env2[1]))}} +)))}} +))}}; +rel_stack_root_vars_a138611bd47fd3ff->insert(tuple,READ_OP_CONTEXT(rel_stack_root_vars_a138611bd47fd3ff_op_ctxt)); +} +} +} +} +} +} +} +();} +if (performIO) { +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\tr"},{"auxArity","0"},{"delimeter","\t"},{"filename","stack_root_vars.tsv"},{"name","stack_root_vars"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"f\", \"r\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:Function\", \"+:Reference\"]}}"}}); +if (outputDirectory == "-"){directiveMap["IO"] = "stdout"; directiveMap["headers"] = "true";} +else if (!outputDirectory.empty()) {directiveMap["output-dir"] = outputDirectory;} +IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_stack_root_vars_a138611bd47fd3ff); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +} +if (pruneImdtRels) rel_assign_e4bb6e0824a16a37->purge(); +if (pruneImdtRels) rel_bind_c9210fdc63280a40->purge(); +if (pruneImdtRels) rel_call_ee1d8972d66cc25f->purge(); +if (pruneImdtRels) rel_live_vars_out_f94306e028b67aa4->purge(); +} + +} // namespace souffle + +namespace souffle { +using namespace souffle; +class Stratum_use_f38e4ba456a0cc9a { +public: + Stratum_use_f38e4ba456a0cc9a(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iii__0_1_2__111::Type& rel_use_e955e932f22dad4d); +void run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret); +private: +SymbolTable& symTable; +RecordTable& recordTable; +ConcurrentCache& regexCache; +bool& pruneImdtRels; +bool& performIO; +SignalHandler*& signalHandler; +std::atomic& iter; +std::atomic& ctr; +std::string& inputDirectory; +std::string& outputDirectory; +t_btree_iii__0_1_2__111::Type* rel_use_e955e932f22dad4d; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Stratum_use_f38e4ba456a0cc9a::Stratum_use_f38e4ba456a0cc9a(SymbolTable& symTable,RecordTable& recordTable,ConcurrentCache& regexCache,bool& pruneImdtRels,bool& performIO,SignalHandler*& signalHandler,std::atomic& iter,std::atomic& ctr,std::string& inputDirectory,std::string& outputDirectory,t_btree_iii__0_1_2__111::Type& rel_use_e955e932f22dad4d): +symTable(symTable), +recordTable(recordTable), +regexCache(regexCache), +pruneImdtRels(pruneImdtRels), +performIO(performIO), +signalHandler(signalHandler), +iter(iter), +ctr(ctr), +inputDirectory(inputDirectory), +outputDirectory(outputDirectory), +rel_use_e955e932f22dad4d(&rel_use_e955e932f22dad4d){ +} + +void Stratum_use_f38e4ba456a0cc9a::run([[maybe_unused]] const std::vector& args,[[maybe_unused]] std::vector& ret){ +if (performIO) { +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\ts\tr"},{"auxArity","0"},{"fact-dir","."},{"name","use"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"f\", \"s\", \"r\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\"]}}"}}); +if (!inputDirectory.empty()) {directiveMap["fact-dir"] = inputDirectory;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_use_e955e932f22dad4d); +} catch (std::exception& e) {std::cerr << "Error loading use data: " << e.what() << '\n'; +exit(1); +} +} +} + +} // namespace souffle + +namespace souffle { +using namespace souffle; +class Sf__: public SouffleProgram { +public: + Sf__(); + ~Sf__(); +void run(); +void runAll(std::string inputDirectoryArg = "",std::string outputDirectoryArg = "",bool performIOArg = true,bool pruneImdtRelsArg = true); +void printAll([[maybe_unused]] std::string outputDirectoryArg = ""); +void loadAll([[maybe_unused]] std::string inputDirectoryArg = ""); +void dumpInputs(); +void dumpOutputs(); +SymbolTable& getSymbolTable(); +RecordTable& getRecordTable(); +void setNumThreads(std::size_t numThreadsValue); +void executeSubroutine(std::string name,const std::vector& args,std::vector& ret); +private: +void runFunction(std::string inputDirectoryArg,std::string outputDirectoryArg,bool performIOArg,bool pruneImdtRelsArg); +SymbolTableImpl symTable; +SpecializedRecordTable<0,2> recordTable; +ConcurrentCache regexCache; +Own rel_assign_e4bb6e0824a16a37; +souffle::RelationWrapper wrapper_rel_assign_e4bb6e0824a16a37; +Own rel_bind_c9210fdc63280a40; +souffle::RelationWrapper wrapper_rel_bind_c9210fdc63280a40; +Own rel_call_ee1d8972d66cc25f; +souffle::RelationWrapper wrapper_rel_call_ee1d8972d66cc25f; +Own rel_might_collect_ef1d0b06d36e4ddc; +souffle::RelationWrapper wrapper_rel_might_collect_ef1d0b06d36e4ddc; +Own rel_delta_might_collect_d651f71586aafe59; +Own rel_new_might_collect_5d48ef45a97e4618; +Own rel_cf_edge_4931a04c8c74bb72; +souffle::RelationWrapper wrapper_rel_cf_edge_4931a04c8c74bb72; +Own rel_use_e955e932f22dad4d; +souffle::RelationWrapper wrapper_rel_use_e955e932f22dad4d; +Own rel_live_vars_in_0b002b95687eda95; +souffle::RelationWrapper wrapper_rel_live_vars_in_0b002b95687eda95; +Own rel_delta_live_vars_in_fccc4ee6df066f63; +Own rel_new_live_vars_in_0b01be53183b2351; +Own rel_live_vars_out_f94306e028b67aa4; +souffle::RelationWrapper wrapper_rel_live_vars_out_f94306e028b67aa4; +Own rel_delta_live_vars_out_acc66913cea62d16; +Own rel_new_live_vars_out_2d78073638bb3740; +Own rel_stack_root_vars_a138611bd47fd3ff; +souffle::RelationWrapper wrapper_rel_stack_root_vars_a138611bd47fd3ff; +Stratum_assign_e0d78e44f4df6411 stratum_assign_f550d366a9215d2a; +Stratum_bind_8b0da46e2379b6cd stratum_bind_1968829e9243d389; +Stratum_call_104fac07831e2229 stratum_call_587d2d7effb5d130; +Stratum_cf_edge_c2ae152829fd6f1f stratum_cf_edge_4017fef287699967; +Stratum_live_vars_in_a363f2025538826a stratum_live_vars_in_c3dc49a4823a7f1e; +Stratum_might_collect_beadc513d07ff032 stratum_might_collect_cc50af26f53a71ac; +Stratum_stack_root_vars_4df5b9c3cd2e7586 stratum_stack_root_vars_49e4f510c537163e; +Stratum_use_f38e4ba456a0cc9a stratum_use_2e20cb5441769259; +std::string inputDirectory; +std::string outputDirectory; +SignalHandler* signalHandler{SignalHandler::instance()}; +std::atomic ctr{}; +std::atomic iter{}; +}; +} // namespace souffle +namespace souffle { +using namespace souffle; + Sf__::Sf__(): +symTable({ + R"_(mylib.MaybeCollect)_", + R"_(self)_", + R"_(.*ctx_.*__init__)_", +}), +recordTable(), +regexCache(), +rel_assign_e4bb6e0824a16a37(mk()), +wrapper_rel_assign_e4bb6e0824a16a37(0, *rel_assign_e4bb6e0824a16a37, *this, "assign", std::array{{"s:Function","i:Statement","+:Reference","+:Value"}}, std::array{{"f","s","r","v"}}, 0), +rel_bind_c9210fdc63280a40(mk()), +wrapper_rel_bind_c9210fdc63280a40(1, *rel_bind_c9210fdc63280a40, *this, "bind", std::array{{"s:Function","i:Statement","+:Reference","s:Function","s:symbol"}}, std::array{{"caller","s","r","callee","param"}}, 0), +rel_call_ee1d8972d66cc25f(mk()), +wrapper_rel_call_ee1d8972d66cc25f(2, *rel_call_ee1d8972d66cc25f, *this, "call", std::array{{"s:Function","i:Statement","s:Function"}}, std::array{{"caller","s","callee"}}, 0), +rel_might_collect_ef1d0b06d36e4ddc(mk()), +wrapper_rel_might_collect_ef1d0b06d36e4ddc(3, *rel_might_collect_ef1d0b06d36e4ddc, *this, "might_collect", std::array{{"s:Function","i:Statement"}}, std::array{{"f","s"}}, 0), +rel_delta_might_collect_d651f71586aafe59(mk()), +rel_new_might_collect_5d48ef45a97e4618(mk()), +rel_cf_edge_4931a04c8c74bb72(mk()), +wrapper_rel_cf_edge_4931a04c8c74bb72(4, *rel_cf_edge_4931a04c8c74bb72, *this, "cf_edge", std::array{{"s:Function","i:Statement","i:Statement"}}, std::array{{"f","s1","s2"}}, 0), +rel_use_e955e932f22dad4d(mk()), +wrapper_rel_use_e955e932f22dad4d(5, *rel_use_e955e932f22dad4d, *this, "use", std::array{{"s:Function","i:Statement","+:Reference"}}, std::array{{"f","s","r"}}, 0), +rel_live_vars_in_0b002b95687eda95(mk()), +wrapper_rel_live_vars_in_0b002b95687eda95(6, *rel_live_vars_in_0b002b95687eda95, *this, "live_vars_in", std::array{{"s:Function","i:Statement","+:Reference"}}, std::array{{"f","s","r"}}, 0), +rel_delta_live_vars_in_fccc4ee6df066f63(mk()), +rel_new_live_vars_in_0b01be53183b2351(mk()), +rel_live_vars_out_f94306e028b67aa4(mk()), +wrapper_rel_live_vars_out_f94306e028b67aa4(7, *rel_live_vars_out_f94306e028b67aa4, *this, "live_vars_out", std::array{{"s:Function","i:Statement","+:Reference"}}, std::array{{"f","s","r"}}, 0), +rel_delta_live_vars_out_acc66913cea62d16(mk()), +rel_new_live_vars_out_2d78073638bb3740(mk()), +rel_stack_root_vars_a138611bd47fd3ff(mk()), +wrapper_rel_stack_root_vars_a138611bd47fd3ff(8, *rel_stack_root_vars_a138611bd47fd3ff, *this, "stack_root_vars", std::array{{"s:Function","+:Reference"}}, std::array{{"f","r"}}, 0), +stratum_assign_f550d366a9215d2a(symTable,recordTable,regexCache,pruneImdtRels,performIO,signalHandler,iter,ctr,inputDirectory,outputDirectory,*rel_assign_e4bb6e0824a16a37), +stratum_bind_1968829e9243d389(symTable,recordTable,regexCache,pruneImdtRels,performIO,signalHandler,iter,ctr,inputDirectory,outputDirectory,*rel_bind_c9210fdc63280a40), +stratum_call_587d2d7effb5d130(symTable,recordTable,regexCache,pruneImdtRels,performIO,signalHandler,iter,ctr,inputDirectory,outputDirectory,*rel_call_ee1d8972d66cc25f), +stratum_cf_edge_4017fef287699967(symTable,recordTable,regexCache,pruneImdtRels,performIO,signalHandler,iter,ctr,inputDirectory,outputDirectory,*rel_cf_edge_4931a04c8c74bb72), +stratum_live_vars_in_c3dc49a4823a7f1e(symTable,recordTable,regexCache,pruneImdtRels,performIO,signalHandler,iter,ctr,inputDirectory,outputDirectory,*rel_delta_live_vars_in_fccc4ee6df066f63,*rel_delta_live_vars_out_acc66913cea62d16,*rel_new_live_vars_in_0b01be53183b2351,*rel_new_live_vars_out_2d78073638bb3740,*rel_assign_e4bb6e0824a16a37,*rel_cf_edge_4931a04c8c74bb72,*rel_live_vars_in_0b002b95687eda95,*rel_live_vars_out_f94306e028b67aa4,*rel_use_e955e932f22dad4d), +stratum_might_collect_cc50af26f53a71ac(symTable,recordTable,regexCache,pruneImdtRels,performIO,signalHandler,iter,ctr,inputDirectory,outputDirectory,*rel_delta_might_collect_d651f71586aafe59,*rel_new_might_collect_5d48ef45a97e4618,*rel_call_ee1d8972d66cc25f,*rel_might_collect_ef1d0b06d36e4ddc), +stratum_stack_root_vars_49e4f510c537163e(symTable,recordTable,regexCache,pruneImdtRels,performIO,signalHandler,iter,ctr,inputDirectory,outputDirectory,*rel_assign_e4bb6e0824a16a37,*rel_bind_c9210fdc63280a40,*rel_call_ee1d8972d66cc25f,*rel_live_vars_out_f94306e028b67aa4,*rel_might_collect_ef1d0b06d36e4ddc,*rel_stack_root_vars_a138611bd47fd3ff), +stratum_use_2e20cb5441769259(symTable,recordTable,regexCache,pruneImdtRels,performIO,signalHandler,iter,ctr,inputDirectory,outputDirectory,*rel_use_e955e932f22dad4d){ +addRelation("assign", wrapper_rel_assign_e4bb6e0824a16a37, true, false); +addRelation("bind", wrapper_rel_bind_c9210fdc63280a40, true, false); +addRelation("call", wrapper_rel_call_ee1d8972d66cc25f, true, false); +addRelation("might_collect", wrapper_rel_might_collect_ef1d0b06d36e4ddc, false, true); +addRelation("cf_edge", wrapper_rel_cf_edge_4931a04c8c74bb72, true, false); +addRelation("use", wrapper_rel_use_e955e932f22dad4d, true, false); +addRelation("live_vars_in", wrapper_rel_live_vars_in_0b002b95687eda95, false, false); +addRelation("live_vars_out", wrapper_rel_live_vars_out_f94306e028b67aa4, false, false); +addRelation("stack_root_vars", wrapper_rel_stack_root_vars_a138611bd47fd3ff, false, true); +} + + Sf__::~Sf__(){ +} + +void Sf__::runFunction(std::string inputDirectoryArg,std::string outputDirectoryArg,bool performIOArg,bool pruneImdtRelsArg){ + + this->inputDirectory = std::move(inputDirectoryArg); + this->outputDirectory = std::move(outputDirectoryArg); + this->performIO = performIOArg; + this->pruneImdtRels = pruneImdtRelsArg; + + // set default threads (in embedded mode) + // if this is not set, and omp is used, the default omp setting of number of cores is used. +#if defined(_OPENMP) + if (0 < getNumThreads()) { omp_set_num_threads(static_cast(getNumThreads())); } +#endif + + signalHandler->set(); +// -- query evaluation -- +{ + std::vector args, ret; +stratum_assign_f550d366a9215d2a.run(args, ret); +} +{ + std::vector args, ret; +stratum_bind_1968829e9243d389.run(args, ret); +} +{ + std::vector args, ret; +stratum_call_587d2d7effb5d130.run(args, ret); +} +{ + std::vector args, ret; +stratum_might_collect_cc50af26f53a71ac.run(args, ret); +} +{ + std::vector args, ret; +stratum_cf_edge_4017fef287699967.run(args, ret); +} +{ + std::vector args, ret; +stratum_use_2e20cb5441769259.run(args, ret); +} +{ + std::vector args, ret; +stratum_live_vars_in_c3dc49a4823a7f1e.run(args, ret); +} +{ + std::vector args, ret; +stratum_stack_root_vars_49e4f510c537163e.run(args, ret); +} + +// -- relation hint statistics -- +signalHandler->reset(); +} + +void Sf__::run(){ +runFunction("", "", false, false); +} + +void Sf__::runAll(std::string inputDirectoryArg,std::string outputDirectoryArg,bool performIOArg,bool pruneImdtRelsArg){ +runFunction(inputDirectoryArg, outputDirectoryArg, performIOArg, pruneImdtRelsArg); +} + +void Sf__::printAll([[maybe_unused]] std::string outputDirectoryArg){ +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\ts"},{"auxArity","0"},{"name","might_collect"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"f\", \"s\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:Function\", \"i:Statement\"]}}"}}); +if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;} +IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_might_collect_ef1d0b06d36e4ddc); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\tr"},{"auxArity","0"},{"delimeter","\t"},{"filename","stack_root_vars.tsv"},{"name","stack_root_vars"},{"operation","output"},{"output-dir","."},{"params","{\"records\": {}, \"relation\": {\"arity\": 2, \"params\": [\"f\", \"r\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 2, \"types\": [\"s:Function\", \"+:Reference\"]}}"}}); +if (!outputDirectoryArg.empty()) {directiveMap["output-dir"] = outputDirectoryArg;} +IOSystem::getInstance().getWriter(directiveMap, symTable, recordTable)->writeAll(*rel_stack_root_vars_a138611bd47fd3ff); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +} + +void Sf__::loadAll([[maybe_unused]] std::string inputDirectoryArg){ +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\ts\tr\tv"},{"auxArity","0"},{"fact-dir","."},{"name","assign"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 4, \"params\": [\"f\", \"s\", \"r\", \"v\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 4, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\", \"+:Value\"]}}"}}); +if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_assign_e4bb6e0824a16a37); +} catch (std::exception& e) {std::cerr << "Error loading assign data: " << e.what() << '\n'; +exit(1); +} +try {std::map directiveMap({{"IO","file"},{"attributeNames","caller\ts\tr\tcallee\tparam"},{"auxArity","0"},{"fact-dir","."},{"name","bind"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 5, \"params\": [\"caller\", \"s\", \"r\", \"callee\", \"param\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 5, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\", \"s:Function\", \"s:symbol\"]}}"}}); +if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_bind_c9210fdc63280a40); +} catch (std::exception& e) {std::cerr << "Error loading bind data: " << e.what() << '\n'; +exit(1); +} +try {std::map directiveMap({{"IO","file"},{"attributeNames","caller\ts\tcallee"},{"auxArity","0"},{"fact-dir","."},{"name","call"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"caller\", \"s\", \"callee\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:Function\", \"i:Statement\", \"s:Function\"]}}"}}); +if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_call_ee1d8972d66cc25f); +} catch (std::exception& e) {std::cerr << "Error loading call data: " << e.what() << '\n'; +exit(1); +} +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\ts1\ts2"},{"auxArity","0"},{"fact-dir","."},{"name","cf_edge"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"f\", \"s1\", \"s2\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:Function\", \"i:Statement\", \"i:Statement\"]}}"}}); +if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_cf_edge_4931a04c8c74bb72); +} catch (std::exception& e) {std::cerr << "Error loading cf_edge data: " << e.what() << '\n'; +exit(1); +} +try {std::map directiveMap({{"IO","file"},{"attributeNames","f\ts\tr"},{"auxArity","0"},{"fact-dir","."},{"name","use"},{"operation","input"},{"params","{\"records\": {}, \"relation\": {\"arity\": 3, \"params\": [\"f\", \"s\", \"r\"]}}"},{"types","{\"ADTs\": {\"+:Reference\": {\"arity\": 2, \"branches\": [{\"name\": \"LocalVariable\", \"types\": [\"s:Function\", \"s:symbol\"]}, {\"name\": \"ObjectMember\", \"types\": [\"s:symbol\", \"s:symbol\"]}], \"enum\": false}, \"+:Value\": {\"arity\": 3, \"branches\": [{\"name\": \"Empty\", \"types\": []}, {\"name\": \"HeapObject\", \"types\": [\"s:symbol\"]}, {\"name\": \"Ref\", \"types\": [\"+:Reference\"]}], \"enum\": false}}, \"records\": {}, \"relation\": {\"arity\": 3, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\"]}}"}}); +if (!inputDirectoryArg.empty()) {directiveMap["fact-dir"] = inputDirectoryArg;} +IOSystem::getInstance().getReader(directiveMap, symTable, recordTable)->readAll(*rel_use_e955e932f22dad4d); +} catch (std::exception& e) {std::cerr << "Error loading use data: " << e.what() << '\n'; +exit(1); +} +} + +void Sf__::dumpInputs(){ +try {std::map rwOperation; +rwOperation["IO"] = "stdout"; +rwOperation["name"] = "assign"; +rwOperation["types"] = "{\"relation\": {\"arity\": 4, \"auxArity\": 0, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\", \"+:Value\"]}}"; +IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_assign_e4bb6e0824a16a37); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +try {std::map rwOperation; +rwOperation["IO"] = "stdout"; +rwOperation["name"] = "bind"; +rwOperation["types"] = "{\"relation\": {\"arity\": 5, \"auxArity\": 0, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\", \"s:Function\", \"s:symbol\"]}}"; +IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_bind_c9210fdc63280a40); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +try {std::map rwOperation; +rwOperation["IO"] = "stdout"; +rwOperation["name"] = "call"; +rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"s:Function\", \"i:Statement\", \"s:Function\"]}}"; +IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_call_ee1d8972d66cc25f); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +try {std::map rwOperation; +rwOperation["IO"] = "stdout"; +rwOperation["name"] = "cf_edge"; +rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"s:Function\", \"i:Statement\", \"i:Statement\"]}}"; +IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_cf_edge_4931a04c8c74bb72); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +try {std::map rwOperation; +rwOperation["IO"] = "stdout"; +rwOperation["name"] = "use"; +rwOperation["types"] = "{\"relation\": {\"arity\": 3, \"auxArity\": 0, \"types\": [\"s:Function\", \"i:Statement\", \"+:Reference\"]}}"; +IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_use_e955e932f22dad4d); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +} + +void Sf__::dumpOutputs(){ +try {std::map rwOperation; +rwOperation["IO"] = "stdout"; +rwOperation["name"] = "might_collect"; +rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:Function\", \"i:Statement\"]}}"; +IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_might_collect_ef1d0b06d36e4ddc); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +try {std::map rwOperation; +rwOperation["IO"] = "stdout"; +rwOperation["name"] = "stack_root_vars"; +rwOperation["types"] = "{\"relation\": {\"arity\": 2, \"auxArity\": 0, \"types\": [\"s:Function\", \"+:Reference\"]}}"; +IOSystem::getInstance().getWriter(rwOperation, symTable, recordTable)->writeAll(*rel_stack_root_vars_a138611bd47fd3ff); +} catch (std::exception& e) {std::cerr << e.what();exit(1);} +} + +SymbolTable& Sf__::getSymbolTable(){ +return symTable; +} + +RecordTable& Sf__::getRecordTable(){ +return recordTable; +} + +void Sf__::setNumThreads(std::size_t numThreadsValue){ +SouffleProgram::setNumThreads(numThreadsValue); +symTable.setNumLanes(getNumThreads()); +recordTable.setNumLanes(getNumThreads()); +regexCache.setNumLanes(getNumThreads()); +} + +void Sf__::executeSubroutine(std::string name,const std::vector& args,std::vector& ret){ +if (name == "assign") { +stratum_assign_f550d366a9215d2a.run(args, ret); +return;} +if (name == "bind") { +stratum_bind_1968829e9243d389.run(args, ret); +return;} +if (name == "call") { +stratum_call_587d2d7effb5d130.run(args, ret); +return;} +if (name == "cf_edge") { +stratum_cf_edge_4017fef287699967.run(args, ret); +return;} +if (name == "live_vars_in") { +stratum_live_vars_in_c3dc49a4823a7f1e.run(args, ret); +return;} +if (name == "might_collect") { +stratum_might_collect_cc50af26f53a71ac.run(args, ret); +return;} +if (name == "stack_root_vars") { +stratum_stack_root_vars_49e4f510c537163e.run(args, ret); +return;} +if (name == "use") { +stratum_use_2e20cb5441769259.run(args, ret); +return;} +fatal(("unknown subroutine " + name).c_str()); +} + +} // namespace souffle +namespace souffle { +SouffleProgram *newInstance__(){return new souffle::Sf__;} +SymbolTable *getST__(SouffleProgram *p){return &reinterpret_cast(p)->getSymbolTable();} +} // namespace souffle + +#ifndef __EMBEDDED_SOUFFLE__ +#include "souffle/CompiledOptions.h" +int main(int argc, char** argv) +{ +try{ +souffle::CmdOptions opt(R"(mycpp/datalog/dataflow.dl)", +R"()", +R"()", +false, +R"()", +1); +if (!opt.parse(argc,argv)) return 1; +souffle::Sf__ obj; +#if defined(_OPENMP) +obj.setNumThreads(opt.getNumJobs()); + +#endif +obj.runAll(opt.getInputFileDir(), opt.getOutputFileDir()); +return 0; +} catch(std::exception &e) { souffle::SignalHandler::instance()->error(e.what());} +} +#endif + +namespace souffle { +using namespace souffle; +class factory_Sf__: souffle::ProgramFactory { +public: +souffle::SouffleProgram* newInstance(); + factory_Sf__(); +private: +}; +} // namespace souffle +namespace souffle { +using namespace souffle; +souffle::SouffleProgram* factory_Sf__::newInstance(){ +return new souffle::Sf__(); +} + + factory_Sf__::factory_Sf__(): +souffle::ProgramFactory("_"){ +} + +} // namespace souffle +namespace souffle { + +#ifdef __EMBEDDED_SOUFFLE__ +extern "C" { +souffle::factory_Sf__ __factory_Sf___instance; +} +#endif +} // namespace souffle + diff --git a/prebuilt/ninja/mycpp.mycpp_main/deps.txt b/prebuilt/ninja/mycpp.mycpp_main/deps.txt index d51cdbb296..eda182b56b 100644 --- a/prebuilt/ninja/mycpp.mycpp_main/deps.txt +++ b/prebuilt/ninja/mycpp.mycpp_main/deps.txt @@ -9,3 +9,4 @@ mycpp/mycpp_main.py mycpp/pass_state.py mycpp/util.py mycpp/visitor.py +_bin/datalog/dataflow diff --git a/testdata/control-flow-graph/classes/assign.facts b/testdata/control-flow-graph/classes/assign.facts index 10b5a7b420..fec59b0d2a 100644 --- a/testdata/control-flow-graph/classes/assign.facts +++ b/testdata/control-flow-graph/classes/assign.facts @@ -1,13 +1,68 @@ -examples.classes.Base.__init__ 2 $Member(examples.classes.Base, next) $Variable(n) -examples.classes.BenchmarkSimpleNode 6 $Variable(next_) $Variable(node) -examples.classes.BenchmarkVirtualNodes 10 $Variable(next_) $Variable(node3) -examples.classes.BenchmarkVirtualNodes 12 $Variable(current) $Variable(node3) -examples.classes.ColorOutput.__init__ 1 $Member(examples.classes.ColorOutput, f) $Variable(f) -examples.classes.DerivedI.__init__ 2 $Member(examples.classes.DerivedI, i) $Variable(i) -examples.classes.DerivedSS.__init__ 2 $Member(examples.classes.DerivedSS, t) $Variable(t) -examples.classes.DerivedSS.__init__ 3 $Member(examples.classes.DerivedSS, u) $Variable(u) -examples.classes.Node.__init__ 1 $Member(examples.classes.Node, next) $Variable(n) -examples.classes.Node.__init__ 2 $Member(examples.classes.Node, i) $Variable(i) -examples.classes.PrintLength 1 $Variable(current) $Variable(node) -examples.classes.PrintLength 6 $Variable(current) $Member(examples.classes.Node, next) -examples.classes.PrintLengthBase 5 $Variable(current) $Member(examples.classes.Base, next) +examples.classes.Abstract.TypeString 0 $LocalVariable(examples.classes.Abstract.TypeString, self) $Empty +examples.classes.Abstract.__init__ 0 $LocalVariable(examples.classes.Abstract.__init__, self) $Empty +examples.classes.Base.TypeString 0 $LocalVariable(examples.classes.Base.TypeString, self) $Empty +examples.classes.Base.__init__ 0 $LocalVariable(examples.classes.Base.__init__, self) $Empty +examples.classes.Base.__init__ 0 $LocalVariable(examples.classes.Base.__init__, n) $Empty +examples.classes.Base.__init__ 2 $ObjectMember(self, next) $Ref($LocalVariable(examples.classes.Base.__init__, n)) +examples.classes.BenchmarkSimpleNode 0 $LocalVariable(examples.classes.BenchmarkSimpleNode, n) $Empty +examples.classes.BenchmarkSimpleNode 3 $LocalVariable(examples.classes.BenchmarkSimpleNode, next_) $HeapObject(h13) +examples.classes.BenchmarkSimpleNode 5 $LocalVariable(examples.classes.BenchmarkSimpleNode, node) $HeapObject(h14) +examples.classes.BenchmarkSimpleNode 6 $LocalVariable(examples.classes.BenchmarkSimpleNode, next_) $Ref($LocalVariable(examples.classes.BenchmarkSimpleNode, node)) +examples.classes.BenchmarkVirtualNodes 0 $LocalVariable(examples.classes.BenchmarkVirtualNodes, n) $Empty +examples.classes.BenchmarkVirtualNodes 3 $LocalVariable(examples.classes.BenchmarkVirtualNodes, next_) $HeapObject(h16) +examples.classes.BenchmarkVirtualNodes 5 $LocalVariable(examples.classes.BenchmarkVirtualNodes, node1) $HeapObject(h17) +examples.classes.BenchmarkVirtualNodes 6 $LocalVariable(examples.classes.BenchmarkVirtualNodes, s1) $HeapObject(h18) +examples.classes.BenchmarkVirtualNodes 7 $LocalVariable(examples.classes.BenchmarkVirtualNodes, s2) $HeapObject(h19) +examples.classes.BenchmarkVirtualNodes 8 $LocalVariable(examples.classes.BenchmarkVirtualNodes, node2) $HeapObject(h20) +examples.classes.BenchmarkVirtualNodes 9 $LocalVariable(examples.classes.BenchmarkVirtualNodes, node3) $HeapObject(h21) +examples.classes.BenchmarkVirtualNodes 10 $LocalVariable(examples.classes.BenchmarkVirtualNodes, next_) $Ref($LocalVariable(examples.classes.BenchmarkVirtualNodes, node3)) +examples.classes.BenchmarkVirtualNodes 11 $LocalVariable(examples.classes.BenchmarkVirtualNodes, current) $HeapObject(h22) +examples.classes.BenchmarkVirtualNodes 12 $LocalVariable(examples.classes.BenchmarkVirtualNodes, current) $Ref($LocalVariable(examples.classes.BenchmarkVirtualNodes, node3)) +examples.classes.BenchmarkWriter 0 $LocalVariable(examples.classes.BenchmarkWriter, n) $Empty +examples.classes.BenchmarkWriter 3 $LocalVariable(examples.classes.BenchmarkWriter, f) $HeapObject(h9) +examples.classes.BenchmarkWriter 4 $LocalVariable(examples.classes.BenchmarkWriter, out) $HeapObject(h10) +examples.classes.BenchmarkWriter 5 $LocalVariable(examples.classes.BenchmarkWriter, i) $HeapObject(h11) +examples.classes.ColorOutput.__init__ 0 $LocalVariable(examples.classes.ColorOutput.__init__, self) $Empty +examples.classes.ColorOutput.__init__ 0 $LocalVariable(examples.classes.ColorOutput.__init__, f) $Empty +examples.classes.ColorOutput.__init__ 1 $ObjectMember(self, f) $Ref($LocalVariable(examples.classes.ColorOutput.__init__, f)) +examples.classes.ColorOutput.__init__ 2 $ObjectMember(self, num_chars) $HeapObject(h0) +examples.classes.ColorOutput.write 0 $LocalVariable(examples.classes.ColorOutput.write, self) $Empty +examples.classes.ColorOutput.write 0 $LocalVariable(examples.classes.ColorOutput.write, s) $Empty +examples.classes.DerivedI.Integer 0 $LocalVariable(examples.classes.DerivedI.Integer, self) $Empty +examples.classes.DerivedI.TypeString 0 $LocalVariable(examples.classes.DerivedI.TypeString, self) $Empty +examples.classes.DerivedI.__init__ 0 $LocalVariable(examples.classes.DerivedI.__init__, self) $Empty +examples.classes.DerivedI.__init__ 0 $LocalVariable(examples.classes.DerivedI.__init__, n) $Empty +examples.classes.DerivedI.__init__ 0 $LocalVariable(examples.classes.DerivedI.__init__, i) $Empty +examples.classes.DerivedI.__init__ 2 $ObjectMember(self, i) $Ref($LocalVariable(examples.classes.DerivedI.__init__, i)) +examples.classes.DerivedSS.TypeString 0 $LocalVariable(examples.classes.DerivedSS.TypeString, self) $Empty +examples.classes.DerivedSS.__init__ 0 $LocalVariable(examples.classes.DerivedSS.__init__, self) $Empty +examples.classes.DerivedSS.__init__ 0 $LocalVariable(examples.classes.DerivedSS.__init__, n) $Empty +examples.classes.DerivedSS.__init__ 0 $LocalVariable(examples.classes.DerivedSS.__init__, t) $Empty +examples.classes.DerivedSS.__init__ 0 $LocalVariable(examples.classes.DerivedSS.__init__, u) $Empty +examples.classes.DerivedSS.__init__ 2 $ObjectMember(self, t) $Ref($LocalVariable(examples.classes.DerivedSS.__init__, t)) +examples.classes.DerivedSS.__init__ 3 $ObjectMember(self, u) $Ref($LocalVariable(examples.classes.DerivedSS.__init__, u)) +examples.classes.Node.__init__ 0 $LocalVariable(examples.classes.Node.__init__, self) $Empty +examples.classes.Node.__init__ 0 $LocalVariable(examples.classes.Node.__init__, n) $Empty +examples.classes.Node.__init__ 0 $LocalVariable(examples.classes.Node.__init__, i) $Empty +examples.classes.Node.__init__ 1 $ObjectMember(self, next) $Ref($LocalVariable(examples.classes.Node.__init__, n)) +examples.classes.Node.__init__ 2 $ObjectMember(self, i) $Ref($LocalVariable(examples.classes.Node.__init__, i)) +examples.classes.PrintLength 0 $LocalVariable(examples.classes.PrintLength, node) $Empty +examples.classes.PrintLength 1 $LocalVariable(examples.classes.PrintLength, current) $Ref($LocalVariable(examples.classes.PrintLength, node)) +examples.classes.PrintLength 2 $LocalVariable(examples.classes.PrintLength, linked_list_len) $HeapObject(h12) +examples.classes.PrintLength 6 $LocalVariable(examples.classes.PrintLength, current) $Ref($ObjectMember(current, next)) +examples.classes.PrintLengthBase 0 $LocalVariable(examples.classes.PrintLengthBase, current) $Empty +examples.classes.PrintLengthBase 1 $LocalVariable(examples.classes.PrintLengthBase, linked_list_len) $HeapObject(h15) +examples.classes.PrintLengthBase 5 $LocalVariable(examples.classes.PrintLengthBase, current) $Ref($ObjectMember(current, next)) +examples.classes.TestInheritance 1 $LocalVariable(examples.classes.TestInheritance, b) $HeapObject(h6) +examples.classes.TestInheritance 2 $LocalVariable(examples.classes.TestInheritance, di) $HeapObject(h7) +examples.classes.TestInheritance 3 $LocalVariable(examples.classes.TestInheritance, dss) $HeapObject(h8) +examples.classes.TestMethods 1 $LocalVariable(examples.classes.TestMethods, stdout_) $HeapObject(h4) +examples.classes.TestMethods 2 $LocalVariable(examples.classes.TestMethods, out) $HeapObject(h5) +examples.classes.TextOutput.MutateFields 0 $LocalVariable(examples.classes.TextOutput.MutateFields, self) $Empty +examples.classes.TextOutput.MutateFields 1 $ObjectMember(self, num_chars) $HeapObject(h2) +examples.classes.TextOutput.MutateFields 2 $ObjectMember(self, i) $HeapObject(h3) +examples.classes.TextOutput.PrintFields 0 $LocalVariable(examples.classes.TextOutput.PrintFields, self) $Empty +examples.classes.TextOutput.__init__ 0 $LocalVariable(examples.classes.TextOutput.__init__, self) $Empty +examples.classes.TextOutput.__init__ 0 $LocalVariable(examples.classes.TextOutput.__init__, f) $Empty +examples.classes.TextOutput.__init__ 3 $ObjectMember(self, i) $HeapObject(h1) +examples.classes.f 0 $LocalVariable(examples.classes.f, obj) $Empty diff --git a/testdata/control-flow-graph/classes/define.facts b/testdata/control-flow-graph/classes/define.facts deleted file mode 100644 index 8bc480e1e4..0000000000 --- a/testdata/control-flow-graph/classes/define.facts +++ /dev/null @@ -1,55 +0,0 @@ -examples.classes.Abstract.TypeString 0 $Variable(self) -examples.classes.Abstract.__init__ 0 $Variable(self) -examples.classes.Base.TypeString 0 $Variable(self) -examples.classes.Base.__init__ 0 $Variable(self) -examples.classes.Base.__init__ 0 $Variable(n) -examples.classes.BenchmarkSimpleNode 0 $Variable(n) -examples.classes.BenchmarkSimpleNode 3 $Variable(next_) -examples.classes.BenchmarkSimpleNode 5 $Variable(node) -examples.classes.BenchmarkVirtualNodes 0 $Variable(n) -examples.classes.BenchmarkVirtualNodes 3 $Variable(next_) -examples.classes.BenchmarkVirtualNodes 5 $Variable(node1) -examples.classes.BenchmarkVirtualNodes 6 $Variable(s1) -examples.classes.BenchmarkVirtualNodes 7 $Variable(s2) -examples.classes.BenchmarkVirtualNodes 8 $Variable(node2) -examples.classes.BenchmarkVirtualNodes 9 $Variable(node3) -examples.classes.BenchmarkVirtualNodes 11 $Variable(current) -examples.classes.BenchmarkWriter 0 $Variable(n) -examples.classes.BenchmarkWriter 3 $Variable(f) -examples.classes.BenchmarkWriter 4 $Variable(out) -examples.classes.BenchmarkWriter 5 $Variable(i) -examples.classes.ColorOutput.__init__ 0 $Variable(self) -examples.classes.ColorOutput.__init__ 0 $Variable(f) -examples.classes.ColorOutput.__init__ 2 $Member(examples.classes.ColorOutput, num_chars) -examples.classes.ColorOutput.write 0 $Variable(self) -examples.classes.ColorOutput.write 0 $Variable(s) -examples.classes.DerivedI.Integer 0 $Variable(self) -examples.classes.DerivedI.TypeString 0 $Variable(self) -examples.classes.DerivedI.__init__ 0 $Variable(self) -examples.classes.DerivedI.__init__ 0 $Variable(n) -examples.classes.DerivedI.__init__ 0 $Variable(i) -examples.classes.DerivedSS.TypeString 0 $Variable(self) -examples.classes.DerivedSS.__init__ 0 $Variable(self) -examples.classes.DerivedSS.__init__ 0 $Variable(n) -examples.classes.DerivedSS.__init__ 0 $Variable(t) -examples.classes.DerivedSS.__init__ 0 $Variable(u) -examples.classes.Node.__init__ 0 $Variable(self) -examples.classes.Node.__init__ 0 $Variable(n) -examples.classes.Node.__init__ 0 $Variable(i) -examples.classes.PrintLength 0 $Variable(node) -examples.classes.PrintLength 2 $Variable(linked_list_len) -examples.classes.PrintLengthBase 0 $Variable(current) -examples.classes.PrintLengthBase 1 $Variable(linked_list_len) -examples.classes.TestInheritance 1 $Variable(b) -examples.classes.TestInheritance 2 $Variable(di) -examples.classes.TestInheritance 3 $Variable(dss) -examples.classes.TestMethods 1 $Variable(stdout_) -examples.classes.TestMethods 2 $Variable(out) -examples.classes.TextOutput.MutateFields 0 $Variable(self) -examples.classes.TextOutput.MutateFields 1 $Member(examples.classes.TextOutput, num_chars) -examples.classes.TextOutput.MutateFields 2 $Member(examples.classes.TextOutput, i) -examples.classes.TextOutput.PrintFields 0 $Variable(self) -examples.classes.TextOutput.__init__ 0 $Variable(self) -examples.classes.TextOutput.__init__ 0 $Variable(f) -examples.classes.TextOutput.__init__ 3 $Member(examples.classes.TextOutput, i) -examples.classes.f 0 $Variable(obj) From 7370174fa5a66268fce7adae771b97f862375d12 Mon Sep 17 00:00:00 2001 From: Aidan <46799759+PossiblyAShrub@users.noreply.github.com> Date: Mon, 12 Aug 2024 09:43:18 -0600 Subject: [PATCH 142/506] [builtins] Implement Str.split() (#2048) --- builtin/method_str.py | 46 ++++++++++++++++++++++++++++++++ core/shell.py | 1 + doc/ref/chap-type-method.md | 18 +++++++++++++ spec/ysh-builtin-eval.test.sh | 4 +-- spec/ysh-methods.test.sh | 50 +++++++++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 2 deletions(-) diff --git a/builtin/method_str.py b/builtin/method_str.py index f864caf0b9..5704410727 100644 --- a/builtin/method_str.py +++ b/builtin/method_str.py @@ -477,3 +477,49 @@ def Call(self, rd): return value.Str("".join(parts)) raise AssertionError() + + +class Split(vm._Callable): + + def __init__(self): + # type: () -> None + pass + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + """ + s.split(sep, count=-1) + + Count behaves like in replace() in that: + - `count` < 0 -> ignore + - `count` >= 0 -> there will be at most `count` splits + """ + string = rd.PosStr() + sep = rd.PosStr() + count = mops.BigTruncate(rd.NamedInt("count", -1)) + rd.Done() + + if len(sep) == 0: + raise error.Structured(3, "sep must be non-empty", rd.LeftParenToken()) + + if len(string) == 0: + return value.List([]) + + cursor = 0 + chunks = [] # type: List[value_t] + while cursor < len(string) and count != 0: + next = string.find(sep, cursor) + if next == -1: + break + + chunks.append(value.Str(string[cursor:next])) + cursor = next + len(sep) + count -= 1 + + if cursor == len(string): + # An instance of sep was against the end of the string + chunks.append(value.Str("")) + else: + chunks.append(value.Str(string[cursor:])) + + return value.List(chunks) diff --git a/core/shell.py b/core/shell.py index 31fe05cee6..d0ee8edba5 100644 --- a/core/shell.py +++ b/core/shell.py @@ -740,6 +740,7 @@ def Main( 'trimEnd': method_str.Trim(method_str.END), 'upper': method_str.Upper(), 'lower': method_str.Lower(), + 'split': method_str.Split(), # finds a substring, optional position to start at 'find': None, diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index e9d0416893..08424bcfb0 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -256,6 +256,24 @@ The `%start` or `^` metacharacter will only match when `pos` is zero. (Similar to Python's `re.match()`.) +### split() + +Split a string by a `Str` separator `sep` into a `List` of chunks. + + pp ('a;b;;c'.split(';')) # => ["a", "b", "", "c"] + pp ('a<>b<>c')) # => ["a","b","c ["🌞", "🌞", "🌞"] + +Optionally, provide a `count` to split on `sep` at most `count` times. A +negative `count` will split on all occurrences of `sep`. + + pp ('a;b;;c'.split(';', count=2)) # => ["a", "b", ";c"] + pp ('a;b;;c'.split(';', count=-1)) # => ["a", "b", "", "c"] + +Passing an empty `sep` will result in an error: + + pp test_ ('abc'.split('')) # => Error: Sep cannot be "" + ## List A List contains an ordered sequence of values. diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 0db5ec8741..adbf22b11b 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -109,7 +109,7 @@ foo bar baz #### eval lines with argv bindings proc my-split (;;; block) { while read --raw-line { - var cols = _reply => split() + var cols = split(_reply) eval (block, pos_args=cols) } } @@ -148,7 +148,7 @@ d c local2 proc my-split (;;; block) { while read --raw-line { - var cols = _reply => split() + var cols = split(_reply) eval (block, vars={_line: _reply, _first: cols[0]}) } } diff --git a/spec/ysh-methods.test.sh b/spec/ysh-methods.test.sh index 152bb0b337..61e9d969a0 100644 --- a/spec/ysh-methods.test.sh +++ b/spec/ysh-methods.test.sh @@ -382,6 +382,56 @@ pp test_ (en2fr => keys()) (List) ["hello","friend","cat"] ## END +#### Str => split(sep), non-empty sep +pp test_ ('a,b,c'.split(',')) +pp test_ ('aa'.split('a')) +pp test_ ('a<>b<>c')) +pp test_ ('a;b;;c'.split(';')) +pp test_ (''.split('foo')) +## STDOUT: +(List) ["a","b","c"] +(List) ["","",""] +(List) ["a","b","c split(sep, count), non-empty sep +pp test_ ('a,b,c'.split(',', count=-1)) +pp test_ ('a,b,c'.split(',', count=-2)) # Any negative count means "ignore count" +pp test_ ('aa'.split('a', count=1)) +pp test_ ('a<>b<>c', count=10)) +pp test_ ('a;b;;c'.split(';', count=2)) +pp test_ (''.split('foo', count=3)) +pp test_ ('a,b,c'.split(',', count=0)) +pp test_ (''.split(',', count=0)) +## STDOUT: +(List) ["a","b","c"] +(List) ["a","b","c"] +(List) ["","a"] +(List) ["a","b","c split(), usage errors +try { pp test_ ('abc'.split('')) } # Sep cannot be "" +echo status=$[_error.code] +try { pp test_ ('abc'.split()) } # Sep must be present +echo status=$[_error.code] +## STDOUT: +status=3 +status=3 +## END + +#### Str => split(), non-ascii +pp test_ ('🌞🌝🌞🌝🌞'.split('🌝')) +## STDOUT: +(List) ["🌞","🌞","🌞"] +## END + #### Dict => values() var en2fr = {} setvar en2fr["hello"] = "bonjour" From 700ff0a9d32a886dce746a9a1be92c3508711eb4 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 10 Aug 2024 23:58:06 -0400 Subject: [PATCH 143/506] [ysh] stdin -> io.stdin io is a new value.Obj that will replace _io. It will not be available in pure functions. (We also need to remove read, $SECONDS, etc. from pure functions) We may also have a value.Obj for Dict.keys, etc. - Add spec tests for things we want to deprecate. --- core/shell.py | 19 +++++++++++--- doc/framing.md | 2 +- doc/idioms.md | 2 +- doc/ref/chap-cmd-lang.md | 4 +-- doc/stream-table-process.md | 2 +- doc/ysh-tour.md | 4 +-- spec/TODO-deprecate.test.sh | 43 +++++++++++++++++++++++++++++++ spec/testdata/builtin-trap-int.sh | 7 ++++- spec/testdata/ysh-for-stdin.ysh | 11 ++++---- spec/ysh-for.test.sh | 4 +-- test/bugs.sh | 2 ++ 11 files changed, 80 insertions(+), 20 deletions(-) diff --git a/core/shell.py b/core/shell.py index d0ee8edba5..c0db239f20 100644 --- a/core/shell.py +++ b/core/shell.py @@ -11,7 +11,7 @@ from _devbuild.gen.runtime_asdl import scope_e from _devbuild.gen.syntax_asdl import (loc, source, source_t, IntParamBox, debug_frame, debug_frame_t) -from _devbuild.gen.value_asdl import (value, value_e) +from _devbuild.gen.value_asdl import (value, value_e, value_t, Obj) from core import alloc from core import comp_ui from core import dev @@ -563,7 +563,15 @@ def Main( # PromptEvaluator rendering is needed in non-interactive shells for @P. prompt_ev = prompt.Evaluator(lang, version_str, parse_ctx, mem) global_io = value.IO(cmd_ev, prompt_ev) - global_guts = value.Guts(None) + + io_methods = { + '__mut_eval': value.BuiltinFunc(method_io.Eval(cmd_ev)), + 'captureStdout': value.BuiltinFunc(method_io.CaptureStdout(shell_ex)), + + # TODO: glob, etc. + } # type: Dict[str, value_t] + io_props = {'stdin': value.Stdin} # type: Dict[str, value_t] + io_obj = Obj(io_props, Obj(io_methods, None)) # Wire up circular dependencies. vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, @@ -902,10 +910,13 @@ def Main( _SetGlobalFunc(mem, '_a2sp', func_misc.BashArrayToSparse()) _SetGlobalFunc(mem, '_opsp', func_misc.SparseOp()) + # TODO: remove this mem.SetNamed(location.LName('_io'), global_io, scope_e.GlobalOnly) - mem.SetNamed(location.LName('_guts'), global_guts, scope_e.GlobalOnly) - mem.SetNamed(location.LName('stdin'), value.Stdin, scope_e.GlobalOnly) + # TODO: 'io' can be in the builtin module, and then hidden in functions + mem.SetNamed(location.LName('io'), io_obj, scope_e.GlobalOnly) + + #mem.SetNamed(location.LName('stdin'), value.Stdin, scope_e.GlobalOnly) # # Is the shell interactive? diff --git a/doc/framing.md b/doc/framing.md index 40f1fd6c33..d074a17142 100644 --- a/doc/framing.md +++ b/doc/framing.md @@ -65,7 +65,7 @@ YSH has a simpler idiom: Or you can read all lines: - for line in (stdin) { # buffered + for line in (io.stdin) { # buffered echo line=$line break # remaining bytes may be lost in a buffer } diff --git a/doc/idioms.md b/doc/idioms.md index 479534dc86..57c235648f 100644 --- a/doc/idioms.md +++ b/doc/idioms.md @@ -231,7 +231,7 @@ Yes: Yes: - for line in (stdin) { + for line in (io.stdin) { echo $line } # this reads buffered lines, which is much faster diff --git a/doc/ref/chap-cmd-lang.md b/doc/ref/chap-cmd-lang.md index b7fd6d84d4..823004b987 100644 --- a/doc/ref/chap-cmd-lang.md +++ b/doc/ref/chap-cmd-lang.md @@ -519,11 +519,11 @@ You can also ask for the index: Here's how to iterate over the lines of stdin: - for line in (stdin) { + for line in (io.stdin) { echo $line } -Likewise, you can ask for the index with `for i, line in (stdin) { ...`. +Likewise, you can ask for the index with `for i, line in (io.stdin) { ...`. ### ysh-while diff --git a/doc/stream-table-process.md b/doc/stream-table-process.md index 299375a4ee..635596deb9 100644 --- a/doc/stream-table-process.md +++ b/doc/stream-table-process.md @@ -250,7 +250,7 @@ We're doing **all of these**. - Buffered for loop - YSH is now roughly as fast as Awk! - - `for x in (stdin)` + - `for x in (io.stdin)` - "magic awk loop" diff --git a/doc/ysh-tour.md b/doc/ysh-tour.md index 5a5bc173ed..2bd0694dea 100644 --- a/doc/ysh-tour.md +++ b/doc/ysh-tour.md @@ -439,14 +439,14 @@ You can also request the loop index: To iterate over lines of `stdin`, use: - for line in (stdin) { + for line in (io.stdin) { echo $line } # lines are buffered, so it's much faster than `while read --rawline` Ask for the loop index: - for i, line in (stdin) { + for i, line in (io.stdin) { echo "$i $line" } diff --git a/spec/TODO-deprecate.test.sh b/spec/TODO-deprecate.test.sh index 646e0d052c..0e3205e934 100644 --- a/spec/TODO-deprecate.test.sh +++ b/spec/TODO-deprecate.test.sh @@ -78,3 +78,46 @@ fi ## STDOUT: OIL ## END + + +#### stdin is now io.stdin + +seq 3 | for line in (io.stdin) { + echo $line +} +## STDOUT: +1 +2 +3 +## END + + +#### Old _io builtin + +echo $[_io=>captureStdout(^(echo hi))] + +## STDOUT: +hi +## END + +#### s.upper(), not s => upper() + +echo $['foo' => upper()] + +## STDOUT: +FOO +## END + + +#### Mutating methods must be ->, not => or . + +var mylist = [] +call mylist=>append('foo') +call mylist.append('bar') + +pp test_ (mylist) + +## STDOUT: +(List) ["foo","bar"] +## END + diff --git a/spec/testdata/builtin-trap-int.sh b/spec/testdata/builtin-trap-int.sh index 2f4c68b9fb..520c05e668 100755 --- a/spec/testdata/builtin-trap-int.sh +++ b/spec/testdata/builtin-trap-int.sh @@ -1,5 +1,10 @@ -# Why don't other shells run this trap? It's not a subshell +# ISSUE WITH TEST: & means that trap handler isn't run! +# I guess because the background job gets disconnected from the terminal? +# So it doesn't need SIGINT + +# We need some other way to kill it with SIGINT + $SH -c 'trap "echo int" INT; sleep 0.1' & sleep 0.05 diff --git a/spec/testdata/ysh-for-stdin.ysh b/spec/testdata/ysh-for-stdin.ysh index aab02497af..36c332a743 100644 --- a/spec/testdata/ysh-for-stdin.ysh +++ b/spec/testdata/ysh-for-stdin.ysh @@ -6,13 +6,12 @@ # < *.py README.md > # etc. -seq 3 | for x in (stdin) { +seq 3 | for x in (io.stdin) { echo "-$x-" } echo - -seq 3 | for i, x in (stdin) { +seq 3 | for i, x in (io.stdin) { echo "$i $x" } echo @@ -20,7 +19,7 @@ echo echo 'empty' fopen < /dev/null { - for x in (stdin) { + for x in (io.stdin) { echo "$x" } } @@ -30,7 +29,7 @@ echo echo 'empty2' -for x in (stdin) { +for x in (io.stdin) { echo "$x" } < /dev/null @@ -39,6 +38,6 @@ echo echo 'space' -echo 'hi' | for x in ( stdin ) { +echo 'hi' | for x in ( io.stdin ) { echo "$x" } diff --git a/spec/ysh-for.test.sh b/spec/ysh-for.test.sh index 6588ebd39f..dd6d77d713 100644 --- a/spec/ysh-for.test.sh +++ b/spec/ysh-for.test.sh @@ -152,7 +152,7 @@ for i, file in *.py {README,foo}.md { 3 foo.md ## END -#### for x in (stdin) { +#### for x in (io.stdin) { # to avoid stdin conflict @@ -182,7 +182,7 @@ hi set +o errexit # EISDIR - stdin descriptor is dir -$SH -c 'for x in (stdin) { echo $x }' < / +$SH -c 'for x in (io.stdin) { echo $x }' < / if test $? -ne 0; then echo pass fi diff --git a/test/bugs.sh b/test/bugs.sh index 03880f6131..4b7eb0639d 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -66,6 +66,8 @@ trap-2() { echo "$sh status=$?" } +# ODD RESULTS in spec tests: the handler is NOT run in bash or other shells +# The handler IS run in manual testing spec-sig() { ### Run spec test outside the sh-spec framework From a65e273b54f89d434e6e358df89063ab647d6673 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 14 Aug 2024 13:46:18 -0400 Subject: [PATCH 144/506] [ysh] Implement prototype() and propView() To get the 2 parts of a prototypal Object(). --- builtin/func_misc.py | 26 ++++++++++++++++++++++---- core/shell.py | 3 ++- core/value.asdl | 4 ++-- doc/ref/chap-builtin-func.md | 32 ++++++++++++++++++++++++++++++++ doc/ref/toc-ysh.md | 1 + frontend/typed_args.py | 21 +++++++++++++++++++-- spec/ysh-object.test.sh | 24 +++++++++++++++++++++++- 7 files changed, 101 insertions(+), 10 deletions(-) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index a590955fd9..9273919b1f 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -60,8 +60,7 @@ def Call(self, rd): raise error.TypeErr(prototype, 'Object() expected Obj or Null', rd.BlamePos()) - # Opposite order - return Obj(props, chain) + return Obj(chain, props) class Prototype(vm._Callable): @@ -73,9 +72,28 @@ def __init__(self): def Call(self, rd): # type: (typed_args.Reader) -> value_t + obj = rd.PosObj() + rd.Done() - # TODO - return value.Null + if obj.prototype is None: + return value.Null + + return obj.prototype + + +class PropView(vm._Callable): + """Get a Dict view of an object's properties.""" + + def __init__(self): + # type: () -> None + pass + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + obj = rd.PosObj() + rd.Done() + + return value.Dict(obj.d) class Len(vm._Callable): diff --git a/core/shell.py b/core/shell.py index c0db239f20..98de7ffe27 100644 --- a/core/shell.py +++ b/core/shell.py @@ -571,7 +571,7 @@ def Main( # TODO: glob, etc. } # type: Dict[str, value_t] io_props = {'stdin': value.Stdin} # type: Dict[str, value_t] - io_obj = Obj(io_props, Obj(io_methods, None)) + io_obj = Obj(Obj(None, io_methods), io_props) # Wire up circular dependencies. vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, @@ -868,6 +868,7 @@ def Main( _SetGlobalFunc(mem, 'Object', func_misc.Object()) _SetGlobalFunc(mem, 'prototype', func_misc.Prototype()) + _SetGlobalFunc(mem, 'propView', func_misc.PropView()) # type conversions _SetGlobalFunc(mem, 'bool', func_misc.Bool()) diff --git a/core/value.asdl b/core/value.asdl index 7ff03ef7a8..b8b81af5b9 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -58,8 +58,8 @@ module value No | Yes %RegexMatch - # prototype is for the attribute lookup chain - Obj = (Dict[str, value] d, Obj? prototype) + # Arbitrary objects, where attributes are looked up on the prototype chain. + Obj = (Obj? prototype, Dict[str, value] d) # Commands, words, and expressions from syntax.asdl are evaluated to a VALUE. # value_t instances are stored in state.Mem(). diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index 040c035ecb..8991e78f4d 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -224,6 +224,38 @@ It's usually better to make an approximate comparison: = abs(float1 - float2) < 0.001 (Bool) false +## Obj + +### Object + +Construct an object with a prototype and properties: + + var obj = Object(null, {x: 42}} + +An object with methods: + + func mymethod(self) { return (self.x) } + var cls = Object(null, {mymethod: mymethod}) + var obj = Object(cls, {x: 42}} + +### prototype() + +Get the prototype of an object. May be null: + + ysh$ = prototype(obj) + (Null) null + +### propView() + +Get a Dict that aliases an object's properties. + + ysh andy@hoover:~/git/oilshell/oil$ = propView(obj) + (Dict) {x: 42} + +This means that if the Dict is modified, then the object is too. + +If you want to copy it, use `dict(obj)`. + ## Word ### glob() diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index a0cc022d1e..09a42868d0 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -79,6 +79,7 @@ X [Module] name() filename() [Str] X strcmp() X split() shSplit() [List] join() [Float] floatsEqual() X isinf() X isnan() + [Obj] Object() prototype() propView() [Word] glob() maybe() [Serialize] toJson() fromJson() toJson8() fromJson8() diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 6c0169d836..5d6d2da4fd 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -4,7 +4,7 @@ from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, LiteralBlock, command_t, expr_t, Token) -from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch) +from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, Obj) from core import error from core.error import e_usage from frontend import location @@ -194,7 +194,11 @@ def PosValue(self): self.LeastSpecificLocation()) self.pos_consumed += 1 - return self.pos_args.pop(0) + val = self.pos_args.pop(0) + + # Should be value.Null + assert val is not None + return val def OptionalValue(self): # type: () -> Optional[value_t] @@ -270,6 +274,14 @@ def _ToDict(self, val): raise error.TypeErr(val, 'Arg %d should be a Dict' % self.pos_consumed, self.BlamePos()) + def _ToObj(self, val): + # type: (value_t) -> Obj + if val.tag() == value_e.Obj: + return cast(Obj, val) + + raise error.TypeErr(val, 'Arg %d should be a Obj' % self.pos_consumed, + self.BlamePos()) + def _ToPlace(self, val): # type: (value_t) -> value.Place if val.tag() == value_e.Place: @@ -403,6 +415,11 @@ def PosDict(self): val = self.PosValue() return self._ToDict(val) + def PosObj(self): + # type: () -> Obj + val = self.PosValue() + return self._ToObj(val) + def PosPlace(self): # type: () -> value.Place val = self.PosValue() diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index 89405d1b3f..77d8868610 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -1,5 +1,5 @@ ## our_shell: ysh -## oils_failures_allowed: 3 +## oils_failures_allowed: 2 #### Object() creates prototype chain @@ -52,6 +52,28 @@ pp test_ (prototype(Rect)) pp test_ (prototype(obj)) ## STDOUT: +(Null) null +(Obj) {"area":} +## END + +#### attributes() + +var obj = Object(null, {x: 3, y: 4}) +var props = propView(obj) + +pp test_ (props) + +# object can be mutated +setvar props.x = 99 + +pp test_ (props) + +var e = propView(null) # error + +## status: 3 +## STDOUT: +(Dict) {"x":3,"y":4} +(Dict) {"x":99,"y":4} ## END #### Copy to Dict with dict(), and mutate From afceb111a5fdc1256bcc6ac7ee5c28009945cd06 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 14 Aug 2024 14:44:48 -0400 Subject: [PATCH 145/506] [ysh] Fix typo in type, which caused crash --- ysh/expr_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 66a6591d34..73ddb60f16 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -980,7 +980,7 @@ def _EvalDot(self, node, obj): UP_obj = obj with tagswitch(obj) as case: if case(value_e.Dict): - obj = cast(Obj, UP_obj) + obj = cast(value.Dict, UP_obj) attr_name = node.attr_name # Dict key / normal attribute lookup From 3d7f043d2dc27be4b16bf1cdc97d42aae362122b Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 14 Aug 2024 23:23:47 -0400 Subject: [PATCH 146/506] [builtin/ysh] MOving value.IO to value.Obj This is the first use case for value.Obj. We have io.stdin and io.captureStdout() working. We also want call io->eval(b) to work. --- builtin/method_io.py | 18 ++++----- core/shell.py | 27 ++++++------- core/test_lib.py | 2 +- core/value.asdl | 3 +- frontend/typed_args.py | 13 ------ spec/ysh-method-io.test.sh | 81 ++++++++++++++++++++++++++++++++++++++ ysh/expr_eval.py | 23 +++++++---- 7 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 spec/ysh-method-io.test.sh diff --git a/builtin/method_io.py b/builtin/method_io.py index b7cb56dedf..bae6bfdf8e 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -9,7 +9,7 @@ from mycpp.mylib import log from osh import prompt -from typing import Dict, cast, TYPE_CHECKING +from typing import Dict, TYPE_CHECKING if TYPE_CHECKING: from frontend import typed_args from osh import cmd_eval @@ -28,13 +28,14 @@ class Eval(vm._Callable): The CALLER must handle errors. """ + def __init__(self, cmd_ev): # type: (cmd_eval.CommandEvaluator) -> None self.cmd_ev = cmd_ev def Call(self, rd): # type: (typed_args.Reader) -> value_t - io = rd.PosIO() + unused = rd.PosValue() cmd = rd.PosCommand() rd.Done() # no more args @@ -52,7 +53,7 @@ def __init__(self, shell_ex): def Call(self, rd): # type: (typed_args.Reader) -> value_t - io = rd.PosIO() + unused = rd.PosValue() cmd = rd.PosCommand() rd.Done() # no more args @@ -77,15 +78,15 @@ class PromptVal(vm._Callable): It expands to $ or # when root """ - def __init__(self): - # type: () -> None - pass + def __init__(self, prompt_ev): + # type: (prompt.Evaluator) -> None + self.prompt_ev = prompt_ev def Call(self, rd): # type: (typed_args.Reader) -> value_t # "self" param is guaranteed to succeed - io = rd.PosIO() + unused = rd.PosValue() what = rd.PosStr() rd.Done() # no more args @@ -95,8 +96,7 @@ def Call(self, rd): 'promptVal() expected a single char, got %r' % what, rd.LeftParenToken()) - prompt_ev = cast(prompt.Evaluator, io.prompt_ev) - return value.Str(prompt_ev.PromptVal(what)) + return value.Str(self.prompt_ev.PromptVal(what)) class Time(vm._Callable): diff --git a/core/shell.py b/core/shell.py index 98de7ffe27..cf5eea1c9a 100644 --- a/core/shell.py +++ b/core/shell.py @@ -562,13 +562,21 @@ def Main( # PromptEvaluator rendering is needed in non-interactive shells for @P. prompt_ev = prompt.Evaluator(lang, version_str, parse_ctx, mem) - global_io = value.IO(cmd_ev, prompt_ev) + global_io = value.IO(None) io_methods = { - '__mut_eval': value.BuiltinFunc(method_io.Eval(cmd_ev)), + # The M/ prefix means it's io->eval() + # This + 'M/eval': value.BuiltinFunc(method_io.Eval(cmd_ev)), + + # identical to command sub 'captureStdout': value.BuiltinFunc(method_io.CaptureStdout(shell_ex)), + 'eval': value.BuiltinFunc(method_io.CaptureStdout(shell_ex)), + 'time': value.BuiltinFunc(method_io.Time()), + 'strftime': value.BuiltinFunc(method_io.Strftime()), - # TODO: glob, etc. + # TODO: + 'glob': None, } # type: Dict[str, value_t] io_props = {'stdin': value.Stdin} # type: Dict[str, value_t] io_obj = Obj(Obj(None, io_methods), io_props) @@ -813,19 +821,8 @@ def Main( } methods[value_e.IO] = { - # TODO: io.eval() or io->eval()? - # We are not mutating the object itself - we are mutating the system. - # That is already captured by io, so let's make it io.eval(). - - # io->eval(myblock) is the functional version of eval (myblock) - # Should we also have expr->eval() instead of evalExpr? + 'promptVal': method_io.PromptVal(prompt_ev), 'eval': method_io.Eval(cmd_ev), - - # identical to command sub - 'captureStdout': method_io.CaptureStdout(shell_ex), - 'promptVal': method_io.PromptVal(), - 'time': method_io.Time(), - 'strftime': method_io.Strftime(), } methods[value_e.Place] = { diff --git a/core/test_lib.py b/core/test_lib.py index e9f1045f30..31f2b55dc0 100644 --- a/core/test_lib.py +++ b/core/test_lib.py @@ -292,7 +292,7 @@ def InitCommandEvaluator(parse_ctx=None, assert cmd_ev.mutable_opts is not None, cmd_ev prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem) - global_io = value.IO(cmd_ev, prompt_ev) + global_io = value.IO(None) vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, prompt_ev, global_io, tracer) diff --git a/core/value.asdl b/core/value.asdl index b8b81af5b9..5cccb38132 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -142,7 +142,8 @@ module value # The ability to use operating system functions. Right now some functions # leak, like glob(). - | IO(any cmd_ev, any prompt_ev) + # TODO: Removed 'unused' after ASDL in Python makes value.IO a type + | IO(any unused) # Do we need this? # _guts->heapId() can be used to detect object cycles. diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 5d6d2da4fd..e7553153c6 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -309,14 +309,6 @@ def _ToEggex(self, val): 'Arg %d should be an Eggex' % self.pos_consumed, self.BlamePos()) - def _ToIO(self, val): - # type: (value_t) -> value.IO - if val.tag() == value_e.IO: - return cast(value.IO, val) - - raise error.TypeErr(val, 'Arg %d should be IO' % self.pos_consumed, - self.BlamePos()) - def _ToExpr(self, val): # type: (value_t) -> expr_t if val.tag() == value_e.Expr: @@ -435,11 +427,6 @@ def PosMatch(self): val = self.PosValue() return self._ToMatch(val) - def PosIO(self): - # type: () -> value.IO - val = self.PosValue() - return self._ToIO(val) - def PosCommand(self): # type: () -> command_t val = self.PosValue() diff --git a/spec/ysh-method-io.test.sh b/spec/ysh-method-io.test.sh new file mode 100644 index 0000000000..2361779c74 --- /dev/null +++ b/spec/ysh-method-io.test.sh @@ -0,0 +1,81 @@ +## our_shell: ysh +## oils_failures_allowed: 0 + +#### captureStdout() is like $() + +var c = ^(echo one; echo two) + +var y = io.captureStdout(c) +pp test_ (y) + +## STDOUT: +(Str) "one\ntwo" +## END + +#### captureStdout() failure + +var c = ^(echo one; false; echo two) + +# Hm this prints a message, but no stack trace +# Should make it fail I think + +try { + var x = io.captureStdout(c) +} +# This has {"code": 3} because it's an expression error. Should probably +pp test_ (_error) + +var x = io.captureStdout(c) + +## status: 4 +## STDOUT: +(Dict) {"status":1,"code":4,"message":"captureStdout(): command failed with status 1"} +## END + +#### _io->eval() is like eval builtin + +var c = ^(echo one; echo two) +var status = _io->eval(c) + +# doesn't return anything +echo status=$status + +## STDOUT: +one +two +status=null +## END + +#### _io->eval() with failing command - caller must handle + +var c = ^(echo one; false; echo two) + +try { + call _io->eval(c) +} +pp test_ (_error) + +call _io->eval(c) + +## status: 1 +## STDOUT: +one +(Dict) {"code":1} +one +## END + +#### _io->eval() with exit + +var c = ^(echo one; exit; echo two) + +try { + call _io->eval(c) +} +echo 'we do not get here' +pp test_ (_error) + + +## STDOUT: +one +## END + diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 73ddb60f16..983a59bffc 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -1047,7 +1047,7 @@ def _EvalAttribute(self, node): # Right now => is a synonym for -> # Later we may enforce that => is pure, and -> is for mutation and # I/O. - if case(Id.Expr_RArrow, Id.Expr_RDArrow): + if case(Id.Expr_RArrow): name = node.attr_name # Look up builtin methods type_methods = self.methods.get(o.tag()) @@ -1056,13 +1056,21 @@ def _EvalAttribute(self, node): if vm_callable: func_val = value.BuiltinFunc(vm_callable) return value.BoundFunc(o, func_val) + #return self._EvalRArrow(node, o) - # If the operator is ->, fail because we don't have any - # user-defined methods - if node.op.id == Id.Expr_RArrow: - raise error.TypeErrVerbose( - 'Method %r does not exist on type %s' % - (name, ui.ValType(o)), node.attr) + raise error.TypeErrVerbose( + 'Method %r does not exist on type %s' % + (name, ui.ValType(o)), node.attr) + + elif case(Id.Expr_RDArrow): + name = node.attr_name + # Look up builtin methods + type_methods = self.methods.get(o.tag()) + vm_callable = (type_methods.get(name) + if type_methods is not None else None) + if vm_callable: + func_val = value.BuiltinFunc(vm_callable) + return value.BoundFunc(o, func_val) # Operator is =>, so try function chaining. @@ -1088,6 +1096,7 @@ def _EvalAttribute(self, node): else: raise AssertionError(node.op) + raise AssertionError() def _EvalExpr(self, node): # type: (expr_t) -> value_t From 654bab685edb865f0a70fe1bc788ea84b087ec83 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 15 Aug 2024 11:02:52 -0400 Subject: [PATCH 147/506] [ysh] Implement mutating method lookup on Obj e.g. io->eval() Still need to document it in doc/ref. The => operator becomes the function chaining operator. It can also do method lookups I think? We want to discourage OOP style when there is no polymorphism. --- spec/TODO-deprecate.test.sh | 9 ---- spec/ysh-method-io.test.sh | 14 +++--- spec/ysh-object.test.sh | 20 +++++++- test/ysh-runtime-errors.sh | 30 ++++++++++++ ysh/expr_eval.py | 97 ++++++++++++++++++++++--------------- 5 files changed, 114 insertions(+), 56 deletions(-) diff --git a/spec/TODO-deprecate.test.sh b/spec/TODO-deprecate.test.sh index 0e3205e934..19abc771b3 100644 --- a/spec/TODO-deprecate.test.sh +++ b/spec/TODO-deprecate.test.sh @@ -91,15 +91,6 @@ seq 3 | for line in (io.stdin) { 3 ## END - -#### Old _io builtin - -echo $[_io=>captureStdout(^(echo hi))] - -## STDOUT: -hi -## END - #### s.upper(), not s => upper() echo $['foo' => upper()] diff --git a/spec/ysh-method-io.test.sh b/spec/ysh-method-io.test.sh index 2361779c74..75fe81fc7d 100644 --- a/spec/ysh-method-io.test.sh +++ b/spec/ysh-method-io.test.sh @@ -32,10 +32,10 @@ var x = io.captureStdout(c) (Dict) {"status":1,"code":4,"message":"captureStdout(): command failed with status 1"} ## END -#### _io->eval() is like eval builtin +#### io->eval() is like eval builtin var c = ^(echo one; echo two) -var status = _io->eval(c) +var status = io->eval(c) # doesn't return anything echo status=$status @@ -46,16 +46,16 @@ two status=null ## END -#### _io->eval() with failing command - caller must handle +#### io->eval() with failing command - caller must handle var c = ^(echo one; false; echo two) try { - call _io->eval(c) + call io->eval(c) } pp test_ (_error) -call _io->eval(c) +call io->eval(c) ## status: 1 ## STDOUT: @@ -64,12 +64,12 @@ one one ## END -#### _io->eval() with exit +#### io->eval() with exit var c = ^(echo one; exit; echo two) try { - call _io->eval(c) + call io->eval(c) } echo 'we do not get here' pp test_ (_error) diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index 77d8868610..ab135a2bb7 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -56,7 +56,7 @@ pp test_ (prototype(obj)) (Obj) {"area":} ## END -#### attributes() +#### propView() var obj = Object(null, {x: 3, y: 4}) var props = propView(obj) @@ -76,6 +76,24 @@ var e = propView(null) # error (Dict) {"x":99,"y":4} ## END +#### Mutating method lookup with -> + +func inc(self, n) { + setvar self.i += n +} +var Counter_methods = Object(null, {'M/inc': inc}) + +var c = Object(Counter_methods, {i: 5}) + +echo $[c.i] +call c->inc(3) +echo $[c.i] + +## STDOUT: +5 +8 +## END + #### Copy to Dict with dict(), and mutate var rect = Object(null, {x: 3, y: 4}) diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 7114ca2eb2..997f194856 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -438,6 +438,36 @@ test-func-error-locs() { ' } +test-attr-error-locs() { + _ysh-expr-error '= {}.key' + _ysh-expr-error '= {}->method' + + _ysh-expr-error 'var obj = Object(null, {}); = obj.attr' + _ysh-expr-error 'var obj = Object(null, {}); = obj->method' + +} + +# TODO: +test-error-loc-bugs() { + _ysh-expr-error ' +func id(x) { + return (x) +} + +#pp test_ (id(len(42))) + +# This should point at ( in len, not id( +pp test_ (len(id(42))) + ' + + _ysh-expr-error ' +var methods = {} + +# Should point at methods, not {} +var o = Object(methods, {}) + ' +} + test-var-decl() { _ysh-expr-error 'var x, y = 1, 2, 3' _ysh-expr-error 'setvar x, y = 1, 2, 3' diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 983a59bffc..d55171a242 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -970,21 +970,21 @@ def _ChainedLookup(self, obj, current, attr_name): return None - def _EvalDot(self, node, obj): + def _EvalDot(self, node, val): # type: (Attribute, value_t) -> value_t - """ obj.attr on RHS or LHS + """ foo.attr on RHS or LHS - setvar x = obj.attr - setglobal g[obj.attr] = 42 + setvar x = foo.attr + setglobal g[foo.attr] = 42 """ - UP_obj = obj - with tagswitch(obj) as case: + UP_val = val + with tagswitch(val) as case: if case(value_e.Dict): - obj = cast(value.Dict, UP_obj) + val = cast(value.Dict, UP_val) attr_name = node.attr_name # Dict key / normal attribute lookup - result = obj.d.get(attr_name) + result = val.d.get(attr_name) if result is not None: return result @@ -992,7 +992,7 @@ def _EvalDot(self, node, obj): node.op) elif case(value_e.Obj): - obj = cast(Obj, UP_obj) + obj = cast(Obj, UP_val) attr_name = node.attr_name # Dict key / normal attribute lookup @@ -1006,33 +1006,63 @@ def _EvalDot(self, node, obj): if result is not None: return result - raise error.Expr('Obj attribute %r not found' % attr_name, + raise error.Expr('Attribute %r not found on Obj' % attr_name, node.op) else: # Method lookup on builtin types. # They don't have attributes or prototype chains -- we only # have a flat dict. - type_methods = self.methods.get(obj.tag()) + type_methods = self.methods.get(val.tag()) name = node.attr_name vm_callable = (type_methods.get(name) if type_methods is not None else None) if vm_callable: func_val = value.BuiltinFunc(vm_callable) - return value.BoundFunc(obj, func_val) + return value.BoundFunc(val, func_val) raise error.TypeErrVerbose( - 'Method %r does not exist on builtin type %s' % - (name, ui.ValType(obj)), node.attr) + "Method %r not found on builtin type %s" % + (name, ui.ValType(val)), node.attr) + + raise AssertionError() + + def _EvalRArrow(self, node, val): + # type: (Attribute, value_t) -> value_t + name = node.attr_name + + UP_val = val + with tagswitch(val) as case: + if case(value_e.Obj): + obj = cast(Obj, UP_val) + mut_name = 'M/' + name + + if obj.prototype is not None: + result = self._ChainedLookup(obj, obj.prototype, mut_name) + if result is not None: + return result + + raise error.Expr( + "Mutating method %r not found on Obj" % mut_name, + node.attr) + else: + # Look up methods on builtin types + type_methods = self.methods.get(val.tag()) + vm_callable = (type_methods.get(name) + if type_methods is not None else None) + if vm_callable: + func_val = value.BuiltinFunc(vm_callable) + return value.BoundFunc(val, func_val) + raise error.TypeErrVerbose( + "Method %r not found on builtin type %s" % + (name, ui.ValType(val)), node.attr) raise AssertionError() def _EvalAttribute(self, node): # type: (Attribute) -> value_t - o = self._EvalExpr(node.obj) - UP_o = o - + val = self._EvalExpr(node.obj) with switch(node.op.id) as case: # TODO: # -> add value.Obj rule - mut_mymethod() @@ -1047,30 +1077,22 @@ def _EvalAttribute(self, node): # Right now => is a synonym for -> # Later we may enforce that => is pure, and -> is for mutation and # I/O. - if case(Id.Expr_RArrow): - name = node.attr_name - # Look up builtin methods - type_methods = self.methods.get(o.tag()) - vm_callable = (type_methods.get(name) - if type_methods is not None else None) - if vm_callable: - func_val = value.BuiltinFunc(vm_callable) - return value.BoundFunc(o, func_val) - #return self._EvalRArrow(node, o) - raise error.TypeErrVerbose( - 'Method %r does not exist on type %s' % - (name, ui.ValType(o)), node.attr) + if case(Id.Expr_Dot): # d.key is like d['key'] + return self._EvalDot(node, val) + + elif case(Id.Expr_RArrow): # e.g. mylist->append(42) + return self._EvalRArrow(node, val) - elif case(Id.Expr_RDArrow): + elif case(Id.Expr_RDArrow): # chaining s => split() name = node.attr_name # Look up builtin methods - type_methods = self.methods.get(o.tag()) + type_methods = self.methods.get(val.tag()) vm_callable = (type_methods.get(name) if type_methods is not None else None) if vm_callable: func_val = value.BuiltinFunc(vm_callable) - return value.BoundFunc(o, func_val) + return value.BoundFunc(val, func_val) # Operator is =>, so try function chaining. @@ -1081,19 +1103,16 @@ def _EvalAttribute(self, node): # f() => str() => upper() # Could improve error message: may give "Undefined variable" - val = self._LookupVar(name, node.attr) + val2 = self._LookupVar(name, node.attr) with tagswitch(val) as case2: if case2(value_e.Func, value_e.BuiltinFunc): - return value.BoundFunc(o, val) + return value.BoundFunc(val, val2) else: raise error.TypeErr( - val, 'Fat arrow => expects method or function', + val2, 'Fat arrow => expects method or function', node.attr) - elif case(Id.Expr_Dot): # d.key is like d['key'] - return self._EvalDot(node, o) - else: raise AssertionError(node.op) raise AssertionError() From 788fb4e13d309be18fe9e4bdf1293337a2ed0787 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 15 Aug 2024 11:45:19 -0400 Subject: [PATCH 148/506] [ysh] Fix typo bug --- ysh/expr_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index d55171a242..d9a82d8456 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -1105,7 +1105,7 @@ def _EvalAttribute(self, node): # Could improve error message: may give "Undefined variable" val2 = self._LookupVar(name, node.attr) - with tagswitch(val) as case2: + with tagswitch(val2) as case2: if case2(value_e.Func, value_e.BuiltinFunc): return value.BoundFunc(val, val2) else: From 236ee38f56b25e1db25f335c846430e7b843a77c Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 15 Aug 2024 11:48:10 -0400 Subject: [PATCH 149/506] [builtin] Migrate io.promptVal() to Obj It's no longer io->promptVal() Document the new behavior of . and -> on value.Obj. They both can look up the prototype chain. The . operator additionally looks for properties. --- core/shell.py | 21 +++++---------- core/value.asdl | 19 ++++--------- core/vm.py | 4 +-- doc/ref/chap-expr-lang.md | 53 +++++++++++++++++++++++++++++++++---- doc/ref/chap-type-method.md | 6 ++--- doc/ref/toc-ysh.md | 4 +-- osh/prompt.py | 4 +-- spec/ysh-object.test.sh | 14 ++++++++++ spec/ysh-prompt.test.sh | 12 ++++----- ysh/expr_eval.py | 30 ++++++++++----------- 10 files changed, 102 insertions(+), 65 deletions(-) diff --git a/core/shell.py b/core/shell.py index cf5eea1c9a..ff65cce25f 100644 --- a/core/shell.py +++ b/core/shell.py @@ -562,20 +562,19 @@ def Main( # PromptEvaluator rendering is needed in non-interactive shells for @P. prompt_ev = prompt.Evaluator(lang, version_str, parse_ctx, mem) - global_io = value.IO(None) io_methods = { + 'promptVal': value.BuiltinFunc(method_io.PromptVal(prompt_ev)), + # The M/ prefix means it's io->eval() - # This 'M/eval': value.BuiltinFunc(method_io.Eval(cmd_ev)), - # identical to command sub + # Identical to command sub 'captureStdout': value.BuiltinFunc(method_io.CaptureStdout(shell_ex)), - 'eval': value.BuiltinFunc(method_io.CaptureStdout(shell_ex)), - 'time': value.BuiltinFunc(method_io.Time()), - 'strftime': value.BuiltinFunc(method_io.Strftime()), # TODO: + 'time': value.BuiltinFunc(method_io.Time()), + 'strftime': value.BuiltinFunc(method_io.Strftime()), 'glob': None, } # type: Dict[str, value_t] io_props = {'stdin': value.Stdin} # type: Dict[str, value_t] @@ -583,7 +582,7 @@ def Main( # Wire up circular dependencies. vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, - prompt_ev, global_io, tracer) + prompt_ev, io_obj, tracer) unsafe_arith = sh_expr_eval.UnsafeArith(mem, exec_opts, mutable_opts, parse_ctx, arith_ev, errfmt) @@ -820,11 +819,6 @@ def Main( 'end': func_eggex.MatchMethod(func_eggex.E, None), } - methods[value_e.IO] = { - 'promptVal': method_io.PromptVal(prompt_ev), - 'eval': method_io.Eval(cmd_ev), - } - methods[value_e.Place] = { # __mut_setValue() @@ -908,9 +902,6 @@ def Main( _SetGlobalFunc(mem, '_a2sp', func_misc.BashArrayToSparse()) _SetGlobalFunc(mem, '_opsp', func_misc.SparseOp()) - # TODO: remove this - mem.SetNamed(location.LName('_io'), global_io, scope_e.GlobalOnly) - # TODO: 'io' can be in the builtin module, and then hidden in functions mem.SetNamed(location.LName('io'), io_obj, scope_e.GlobalOnly) diff --git a/core/value.asdl b/core/value.asdl index 5cccb38132..901c39a6d6 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -90,7 +90,6 @@ module value | Null | Bool(bool b) | Int(BigInt i) - #| Int(int i) | Float(float f) | List(List[value] items) | Dict(Dict[str, value] d) @@ -135,22 +134,12 @@ module value # The frame MUST be lower on the stack at the time of use. | Place(y_lvalue lval, Dict[str, Cell] frame) + # TODO: Remove this, could be value.Obj # for Flags/flag and Flags/arg? # for json read/write ? # Possibly unify Hay and modules/namespaces | Module(Dict[str, value] defs) - # The ability to use operating system functions. Right now some functions - # leak, like glob(). - # TODO: Removed 'unused' after ASDL in Python makes value.IO a type - | IO(any unused) - - # Do we need this? - # _guts->heapId() can be used to detect object cycles. - # It's considered impure; it depends on VM implementation details. The = - # operator and 'pp value' also print the heap ID. - | Guts(any vm) - # callable is vm._Callable. # TODO: ASDL needs some kind of "extern" to declare vm._Callable and # cmd_eval.CommandEvaluator. I think it would just generate a forward @@ -172,11 +161,13 @@ module value List[value] pos_defaults, Dict[str, value] named_defaults, Dict[str, Cell]? module_) + # for i in (1:n) { echo $i } # both ends are required + | Range(int lower, int upper) + + # internal detail - can't be instantied by users # a[3:5] a[:10] a[3:] a[:] # both ends are optional | Slice(IntBox? lower, IntBox? upper) - # for i in (1:n) { echo $i } # both ends are required - | Range(int lower, int upper) } # vim: sw=2 diff --git a/core/vm.py b/core/vm.py index 2a8fb4be8c..d9ba839c03 100644 --- a/core/vm.py +++ b/core/vm.py @@ -5,7 +5,7 @@ from _devbuild.gen.runtime_asdl import (CommandStatus, StatusArray, flow_e, flow_t) from _devbuild.gen.syntax_asdl import Token -from _devbuild.gen.value_asdl import value, value_t +from _devbuild.gen.value_asdl import value, value_t, Obj from core import error from core import pyos from mycpp.mylib import log @@ -123,7 +123,7 @@ def InitCircularDeps( cmd_ev, # type: CommandEvaluator shell_ex, # type: _Executor prompt_ev, # type: prompt.Evaluator - global_io, # type: value.IO + global_io, # type: Obj tracer, # type: dev.Tracer ): # type: (...) -> None diff --git a/doc/ref/chap-expr-lang.md b/doc/ref/chap-expr-lang.md index 68f576ee87..31a331c145 100644 --- a/doc/ref/chap-expr-lang.md +++ b/doc/ref/chap-expr-lang.md @@ -472,9 +472,39 @@ The ternary operator is borrowed from Python: ### ysh-attr -The expression `mydict.key` is short for `mydict['key']`. +The `.` operator performs attribute lookup. -(Like JavaScript, but unlike Python.) +On `Dict` instances, the expression `mydict.key` is short for `mydict['key']` +(like JavaScript, but unlike Python.) + +On `Obj` instances, the expression `obj.attr` does two things, in order: + +1. Searches in the object's properties for a field named `attr`. + - If it exists, return the value literally. +2. Searches up the prototype chain for `attr` + - If it exists, return a **bound method**, which is an (object, function) + pair. + +Later, when the bound method is called, the object is passed as the first +argument to the function, making it a method call. The method can then use the +object's properties. + +Example of first rule: + + func Free(i) { + return (i + 1) + } + var module = Object(null, {Free}) + var x = module.Free(42) # => 43 + +Example of second rule: + + func method(self, i) { + return (self.n + i) + } + var methods = Object(null, {method}) + var obj = Object(methods, {n: 1}) + var x = obj.method(42) # => 43 ### ysh-slice @@ -525,11 +555,24 @@ The thin arrow is for mutating methods: var mylist = ['bar'] call mylist->pop() - + +On `Obj` instances, `obj->mymethod` looks up the prototype chain for a function +named `M/mymethod`. The `M/` prefix signals mutation. + +Example: + + func inc(self, n) { + setvar self.i += n + } + var Counter_methods = Object(null, {'M/inc': inc}) + var c = Object(Counter_methods, {i: 0}) + + call c->inc(5) + echo $[c.i] # => 5 + +It does **not** look in the properties of an object. ### fat-arrow diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 08424bcfb0..55d9a56300 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -546,7 +546,7 @@ Though this runs in the same VM, not a new one. Capture stdout of a command a string. var c = ^(echo hi) - var stdout_str = _io->captureStdout(c) # => "hi" + var stdout_str = _io.captureStdout(c) # => "hi" It's like `$()`, but useful in pure functions. Trailing newlines `\n` are removed. @@ -564,8 +564,8 @@ An API the wraps the `$PS1` language. For example, to simulate `PS1='\w\$ '`: func renderPrompt(io) { var parts = [] - call parts->append(io->promptval('w')) # pass 'w' for \w - call parts->append(io->promptval('$')) # pass '$' for \$ + call parts->append(io.promptval('w')) # pass 'w' for \w + call parts->append(io.promptval('$')) # pass '$' for \$ call parts->append(' ') return (join(parts)) } diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 09a42868d0..d114e4efa4 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -265,11 +265,11 @@ X [External Lang] BEGIN END when (awk) ysh-bitwise ~ & | ^ << >> ysh-ternary '+' if x >= 0 else '-' ysh-index s[0] mylist[3] mydict['key'] - ysh-attr mydict.key + ysh-attr mydict.key mystr.startsWith('x') ysh-slice a[1:-1] s[1:-1] func-call f(x, y; ...named) thin-arrow mylist->pop() - fat-arrow mystr => startsWith('prefix') + fat-arrow mylist => join() => upper() match-ops ~ !~ ~~ !~~ [Eggex] re-literal / d+ ; re-flags ; ERE / re-primitive %zero 'sq' diff --git a/osh/prompt.py b/osh/prompt.py index 27e6d29d2e..08e45723b3 100644 --- a/osh/prompt.py +++ b/osh/prompt.py @@ -9,7 +9,7 @@ from _devbuild.gen.id_kind_asdl import Id, Id_t from _devbuild.gen.syntax_asdl import (loc, command_t, source, CompoundWord) -from _devbuild.gen.value_asdl import (value, value_e, value_t) +from _devbuild.gen.value_asdl import (value, value_e, value_t, Obj) from core import alloc from core import main_loop from core import error @@ -104,7 +104,7 @@ def __init__(self, lang, version_str, parse_ctx, mem): # type: (str, str, ParseContext, Mem) -> None self.word_ev = None # type: word_eval.AbstractWordEvaluator self.expr_ev = None # type: expr_eval.ExprEvaluator - self.global_io = None # type: value.IO + self.global_io = None # type: Obj assert lang in ('osh', 'ysh'), lang self.lang = lang diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index ab135a2bb7..cd7abb6c3b 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -94,6 +94,20 @@ echo $[c.i] 8 ## END +#### Mutating method must be up the prototype chain, not on the object + +func inc(self, n) { + setvar self.i += n +} +var c = Object(null, {'M/inc': inc, i: 0}) + +call c->inc(3) + +## status: 3 +## STDOUT: +## END + + #### Copy to Dict with dict(), and mutate var rect = Object(null, {x: 3, y: 4}) diff --git a/spec/ysh-prompt.test.sh b/spec/ysh-prompt.test.sh index f7298040cb..2d7841b7bb 100644 --- a/spec/ysh-prompt.test.sh +++ b/spec/ysh-prompt.test.sh @@ -4,12 +4,12 @@ shopt -s ysh:upgrade -var x = _io->promptVal('$') +var x = io.promptVal('$') # We're not root, so it should be $ echo x=$x -var x = _io->promptVal('w') +var x = io.promptVal('w') if (x === PWD) { echo pass } else { @@ -24,14 +24,14 @@ pass #### promptVal() with invalid chars # \D{} will be supported with date and time functions -var x = _io->promptVal('D') +var x = io.promptVal('D') echo x=$x # something else -var x = _io->promptVal('/') +var x = io.promptVal('/') echo x=$x -var x = _io->promptVal('ZZ') +var x = io.promptVal('ZZ') echo x=$x ## status: 3 @@ -60,7 +60,7 @@ cat >yshrc <<'EOF' func renderPrompt(io) { var parts = [] call parts->append('hi') - call parts->append(io->promptVal('$')) + call parts->append(io.promptVal('$')) call parts->append(' ') return (join(parts)) } diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index d9a82d8456..c1b4730a55 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -1042,11 +1042,17 @@ def _EvalRArrow(self, node, val): if result is not None: return result + # TODO: we could have different errors for: + # - no prototype + # - found in the properties, not in the prototype chain (not + # sure if this error is common.) raise error.Expr( - "Mutating method %r not found on Obj" % mut_name, + "Mutating method %r not found on Obj prototype chain" % mut_name, node.attr) else: # Look up methods on builtin types + # TODO: These should also be called M/append, M/erase, etc. + type_methods = self.methods.get(val.tag()) vm_callable = (type_methods.get(name) if type_methods is not None else None) @@ -1064,20 +1070,6 @@ def _EvalAttribute(self, node): val = self._EvalExpr(node.obj) with switch(node.op.id) as case: - # TODO: - # -> add value.Obj rule - mut_mymethod() - # then change value.List to have __mut_append()? - # this means you can no longer do call foo => end(), which we want - # - # => eventually remove method lookup - it's only the chaining - # operator - # s => upper() => strip() might be OK though - # versus s.upper().strip() - - # Right now => is a synonym for -> - # Later we may enforce that => is pure, and -> is for mutation and - # I/O. - if case(Id.Expr_Dot): # d.key is like d['key'] return self._EvalDot(node, val) @@ -1086,7 +1078,13 @@ def _EvalAttribute(self, node): elif case(Id.Expr_RDArrow): # chaining s => split() name = node.attr_name - # Look up builtin methods + + # Look up builtin methods, e.g. + # s => strip() is like s.strip() + # Note: + # m => group(1) is worse than m.group(1) + # This is not a transformation, but more like an attribute + type_methods = self.methods.get(val.tag()) vm_callable = (type_methods.get(name) if type_methods is not None else None) From 93b4c493d4741afd363713a5290006786cba9027 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 15 Aug 2024 13:44:29 -0400 Subject: [PATCH 150/506] [test/unit] Fix test --- core/test_lib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/test_lib.py b/core/test_lib.py index 31f2b55dc0..f3c31afcc3 100644 --- a/core/test_lib.py +++ b/core/test_lib.py @@ -15,7 +15,7 @@ from _devbuild.gen.option_asdl import builtin_i, option_i from _devbuild.gen.runtime_asdl import cmd_value, scope_e from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token -from _devbuild.gen.value_asdl import value +from _devbuild.gen.value_asdl import value, Obj from asdl import pybase from builtin import assign_osh from builtin import completion_osh @@ -292,7 +292,7 @@ def InitCommandEvaluator(parse_ctx=None, assert cmd_ev.mutable_opts is not None, cmd_ev prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem) - global_io = value.IO(None) + global_io = Obj(None, None) vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, prompt_ev, global_io, tracer) From 3f9e8209d36823ef8ab334f9dd551dd41599a156 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 15 Aug 2024 13:45:12 -0400 Subject: [PATCH 151/506] [test/lint] Fix build --- core/vm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm.py b/core/vm.py index d9ba839c03..f90d34ed65 100644 --- a/core/vm.py +++ b/core/vm.py @@ -5,7 +5,7 @@ from _devbuild.gen.runtime_asdl import (CommandStatus, StatusArray, flow_e, flow_t) from _devbuild.gen.syntax_asdl import Token -from _devbuild.gen.value_asdl import value, value_t, Obj +from _devbuild.gen.value_asdl import value_t, Obj from core import error from core import pyos from mycpp.mylib import log From 474de14ac6c924a42a28a0bc54e64cd189be70d0 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 15 Aug 2024 14:12:08 -0400 Subject: [PATCH 152/506] [ysh breaking] Enforce that -> is always mutating method lookup Even on builtin types. Now we no longer have the "relaxed" semantics of -> and => being interchangeable. --- Idea for hiding the 'io'. name. This is a bit tricky since doing it at every function call is overkill. Ideally, we want to do it only at the proc/func boundary. That can't be at ctx_Expr either, because we want echo @[io.glob('*.py')] to work. --- core/shell.py | 21 ++++++++++----------- spec/ysh-builtins.test.sh | 2 +- spec/ysh-convert.test.sh | 2 +- spec/ysh-expr-compare.test.sh | 2 +- spec/ysh-regex-api.test.sh | 4 ++-- spec/ysh-stdlib-args.test.sh | 2 +- stdlib/ysh/args.ysh | 2 +- ysh/expr_eval.py | 9 ++++----- ysh/func_proc.py | 3 +++ 9 files changed, 24 insertions(+), 23 deletions(-) diff --git a/core/shell.py b/core/shell.py index ff65cce25f..bbe2558a49 100644 --- a/core/shell.py +++ b/core/shell.py @@ -776,8 +776,7 @@ def Main( 'fullMatch': None, } methods[value_e.Dict] = { - # TODO: __mut_erase - 'erase': method_dict.Erase(), + 'M/erase': method_dict.Erase(), # Dict.get() # Dict.keys() @@ -793,19 +792,19 @@ def Main( # # call d->inc('mycounter') # call d->inc('mycounter', 3) - 'inc': None, + 'M/inc': None, # call d->accum('mygroup', 'value') - 'accum': None, + 'M/accum': None, } methods[value_e.List] = { # TODO: __mut_{reverse,append,extend,pop,insert,remove} - 'reverse': method_list.Reverse(), - 'append': method_list.Append(), - 'extend': method_list.Extend(), - 'pop': method_list.Pop(), - 'insert': None, # insert object before index - 'remove': None, # insert object before index + 'M/reverse': method_list.Reverse(), + 'M/append': method_list.Append(), + 'M/extend': method_list.Extend(), + 'M/pop': method_list.Pop(), + 'M/insert': None, # insert object before index + 'M/remove': None, # insert object before index 'indexOf': method_list.IndexOf(), # return first index of value, or -1 # Python list() has index(), which raises ValueError # But this is consistent with Str->find(), and doesn't @@ -823,7 +822,7 @@ def Main( # __mut_setValue() # instead of setplace keyword - 'setValue': method_other.SetValue(mem), + 'M/setValue': method_other.SetValue(mem), } methods[value_e.Command] = { diff --git a/spec/ysh-builtins.test.sh b/spec/ysh-builtins.test.sh index 5877c04658..875cc04d0d 100644 --- a/spec/ysh-builtins.test.sh +++ b/spec/ysh-builtins.test.sh @@ -550,7 +550,7 @@ func f() { echo $[type(f)] echo $[type(len)] -echo $[type('foo'->startsWith)] +echo $[type('foo'=>startsWith)] echo $[type('foo'=>join)] # Type error happens later echo $[type(1..3)] ## STDOUT: diff --git a/spec/ysh-convert.test.sh b/spec/ysh-convert.test.sh index a9b4b63ab7..6e42bf2cee 100644 --- a/spec/ysh-convert.test.sh +++ b/spec/ysh-convert.test.sh @@ -10,7 +10,7 @@ echo "$[bool([])]" echo "$[bool({})]" echo "$[bool(null)]" echo "$[bool(len)]" -echo "$[bool('foo'->startsWith)]" +echo "$[bool('foo'=>startsWith)]" echo "$[bool(1..3)]" ## STDOUT: true diff --git a/spec/ysh-expr-compare.test.sh b/spec/ysh-expr-compare.test.sh index a9294f2784..a33a8fe38f 100644 --- a/spec/ysh-expr-compare.test.sh +++ b/spec/ysh-expr-compare.test.sh @@ -362,7 +362,7 @@ var unimpl = [ myexpr, # Expr ^(echo hello), # Block f, # Func - mydict->keys, # BoundFunc + mydict=>keys, # BoundFunc # These cannot be constructed # - Proc # - Slice diff --git a/spec/ysh-regex-api.test.sh b/spec/ysh-regex-api.test.sh index 728da506a5..334b643d54 100644 --- a/spec/ysh-regex-api.test.sh +++ b/spec/ysh-regex-api.test.sh @@ -273,7 +273,7 @@ g1 0 2 hi g2 2 3 5 ## END -#### Str->leftMatch() can implement lexer pattern +#### Str=>leftMatch() can implement lexer pattern shopt -s ysh:upgrade @@ -286,7 +286,7 @@ proc show-tokens (s) { while (true) { echo "pos=$pos" - var m = s->leftMatch(lexer, pos=pos) + var m = s=>leftMatch(lexer, pos=pos) if (not m) { break } diff --git a/spec/ysh-stdlib-args.test.sh b/spec/ysh-stdlib-args.test.sh index 57b3e90350..ba3d14ffcb 100644 --- a/spec/ysh-stdlib-args.test.sh +++ b/spec/ysh-stdlib-args.test.sh @@ -172,7 +172,7 @@ print(result) ''' for args in (argsCases) { - var args_str = args->join(" ") + var args_str = args=>join(" ") echo "---------- $args_str ----------" echo "\$ bin/ysh example.sh $args_str" pp test_ (parseArgs(spec, args)) diff --git a/stdlib/ysh/args.ysh b/stdlib/ysh/args.ysh index ed81950e29..9143d15fff 100644 --- a/stdlib/ysh/args.ysh +++ b/stdlib/ysh/args.ysh @@ -143,7 +143,7 @@ func parseArgs(spec, argv) { var found while (i < argc) { var arg = argv[i] - if (arg->startsWith('-')) { + if (arg.startsWith('-')) { setvar found = false for flag in (spec.flags) { diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index c1b4730a55..1fbd8cde47 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -1029,13 +1029,12 @@ def _EvalDot(self, node, val): def _EvalRArrow(self, node, val): # type: (Attribute, value_t) -> value_t - name = node.attr_name + mut_name = 'M/' + node.attr_name UP_val = val with tagswitch(val) as case: if case(value_e.Obj): obj = cast(Obj, UP_val) - mut_name = 'M/' + name if obj.prototype is not None: result = self._ChainedLookup(obj, obj.prototype, mut_name) @@ -1054,15 +1053,15 @@ def _EvalRArrow(self, node, val): # TODO: These should also be called M/append, M/erase, etc. type_methods = self.methods.get(val.tag()) - vm_callable = (type_methods.get(name) + vm_callable = (type_methods.get(mut_name) if type_methods is not None else None) if vm_callable: func_val = value.BuiltinFunc(vm_callable) return value.BoundFunc(val, func_val) raise error.TypeErrVerbose( - "Method %r not found on builtin type %s" % - (name, ui.ValType(val)), node.attr) + "Mutating method %r not found on builtin type %s" % + (mut_name, ui.ValType(val)), node.attr) raise AssertionError() def _EvalAttribute(self, node): diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 27d35c1da8..86c061df48 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -558,6 +558,9 @@ def CallUserFunc( # type: (...) -> value_t # Push a new stack frame + + # TODO: ctx_Eval() can replace io with DummyIO type! It can possibly + # implement __getattr__ and __get_mutating__? with state.ctx_FuncCall(mem, func): _BindFuncArgs(func, rd, mem) From 4b0ff4e9b19548c8d1b9726c9b42287317379f90 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 16 Aug 2024 14:19:25 -0400 Subject: [PATCH 153/506] [ysh/testdata] Fix example --- ysh/testdata/expr-sub.ysh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ysh/testdata/expr-sub.ysh b/ysh/testdata/expr-sub.ysh index d0af177d99..64a317bac3 100644 --- a/ysh/testdata/expr-sub.ysh +++ b/ysh/testdata/expr-sub.ysh @@ -41,7 +41,7 @@ simple-demo() { echo ' Notes:' echo ' - the Dict->reverse() method is from Python.' echo - write -- @[mydict->keys()] + write -- @[mydict=>keys()] echo # But this is a syntax error From 641c6110494e692759e3e569a77483b0127473a3 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 16 Aug 2024 14:20:35 -0400 Subject: [PATCH 154/506] [test/spec] Remove deprecated cases --- doc/ref/toc-ysh.md | 2 +- spec/TODO-deprecate.test.sh | 28 +--------------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index d114e4efa4..382a98d3b4 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -48,7 +48,7 @@ error handling, and more. search() leftMatch() [List] List/append() pop() extend() indexOf() X insert() X remove() reverse() - [Dict] keys() values() X get() erase() + [Dict] keys() values() get() erase() X inc() X accum() [Range] [Eggex] diff --git a/spec/TODO-deprecate.test.sh b/spec/TODO-deprecate.test.sh index 19abc771b3..05811fc35a 100644 --- a/spec/TODO-deprecate.test.sh +++ b/spec/TODO-deprecate.test.sh @@ -79,36 +79,10 @@ fi OIL ## END - -#### stdin is now io.stdin - -seq 3 | for line in (io.stdin) { - echo $line -} -## STDOUT: -1 -2 -3 -## END - -#### s.upper(), not s => upper() +#### s.upper(), not s => upper() (might keep this) echo $['foo' => upper()] ## STDOUT: FOO ## END - - -#### Mutating methods must be ->, not => or . - -var mylist = [] -call mylist=>append('foo') -call mylist.append('bar') - -pp test_ (mylist) - -## STDOUT: -(List) ["foo","bar"] -## END - From ec3d1473bc0659e0ce3549bde64dac0ab3793e38 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 16 Aug 2024 15:04:12 -0400 Subject: [PATCH 155/506] [test/spec refactor] Move deprecation file to ysh suite --- ...{TODO-deprecate.test.sh => ysh-TODO-deprecate.test.sh} | 0 test/spec.sh | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename spec/{TODO-deprecate.test.sh => ysh-TODO-deprecate.test.sh} (100%) diff --git a/spec/TODO-deprecate.test.sh b/spec/ysh-TODO-deprecate.test.sh similarity index 100% rename from spec/TODO-deprecate.test.sh rename to spec/ysh-TODO-deprecate.test.sh diff --git a/test/spec.sh b/test/spec.sh index 731d082832..3e9d5a2d77 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -91,10 +91,6 @@ osh-bugs() { run-file osh-bugs "$@" } -TODO-deprecate() { - run-file TODO-deprecate "$@" -} - blog1() { sh-spec spec/blog1.test.sh \ ${REF_SHELLS[@]} $ZSH $OSH_LIST "$@" @@ -674,6 +670,10 @@ hay-meta() { # YSH # +ysh-TODO-deprecate() { + run-file ysh-TODO-deprecate "$@" +} + ysh-convert() { run-file ysh-convert "$@" } From ecd4104befb64b1d38dd97523b417b6521861b8f Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 19 Aug 2024 17:13:31 -0400 Subject: [PATCH 156/506] [test/spec] Migrate a few files to the new style --- spec/if_.test.sh | 3 +-- spec/quote.test.sh | 3 ++- spec/subshell.test.sh | 2 +- test/spec.sh | 9 +++------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/spec/if_.test.sh b/spec/if_.test.sh index 9986545a78..c8468fd37d 100644 --- a/spec/if_.test.sh +++ b/spec/if_.test.sh @@ -1,5 +1,4 @@ -# -# Test the if statement +## compare_shells: dash bash mksh zsh #### If if true; then diff --git a/spec/quote.test.sh b/spec/quote.test.sh index c5ba4aa1e8..e789f625bc 100644 --- a/spec/quote.test.sh +++ b/spec/quote.test.sh @@ -1,4 +1,5 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 +## compare_shells: dash bash mksh ash #### Unquoted words echo unquoted words diff --git a/spec/subshell.test.sh b/spec/subshell.test.sh index b02931d447..5857b151d5 100644 --- a/spec/subshell.test.sh +++ b/spec/subshell.test.sh @@ -1,4 +1,4 @@ -# spec/subshell +## compare_shells: dash bash mksh #### Subshell exit code ( false; ) diff --git a/test/spec.sh b/test/spec.sh index 3e9d5a2d77..1d63c05839 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -148,13 +148,11 @@ background() { } subshell() { - sh-spec spec/subshell.test.sh \ - ${REF_SHELLS[@]} $OSH_LIST "$@" + run-file subshell "$@" } quote() { - sh-spec spec/quote.test.sh \ - ${REF_SHELLS[@]} $BUSYBOX_ASH $OSH_LIST "$@" + run-file quote "$@" } unicode() { @@ -170,8 +168,7 @@ case_() { } if_() { - sh-spec spec/if_.test.sh \ - ${REF_SHELLS[@]} $ZSH $OSH_LIST "$@" + run-file if_ "$@" } builtin-misc() { From e8e5e2a06b513825a8b24d694edb452d334c6ea1 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 19 Aug 2024 18:38:52 -0400 Subject: [PATCH 157/506] [test/spec] Migrate a few more files --- spec/builtin-dirs.test.sh | 3 +++ spec/builtin-times.test.sh | 1 + spec/command-parsing.test.sh | 3 ++- spec/func-parsing.test.sh | 1 + test/spec.sh | 10 ++++------ 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/spec/builtin-dirs.test.sh b/spec/builtin-dirs.test.sh index db1ed60a33..f9a51920b7 100644 --- a/spec/builtin-dirs.test.sh +++ b/spec/builtin-dirs.test.sh @@ -1,3 +1,6 @@ +## compare_shells: bash zsh + +# dash and mksh don't implement 'dirs' #### pushd/popd set -o errexit diff --git a/spec/builtin-times.test.sh b/spec/builtin-times.test.sh index 783937aabc..f3452e94b8 100644 --- a/spec/builtin-times.test.sh +++ b/spec/builtin-times.test.sh @@ -1,3 +1,4 @@ +## compare_shells: bash zsh #### times shows two formatted lines output=$(times) diff --git a/spec/command-parsing.test.sh b/spec/command-parsing.test.sh index dad1576fc2..62c4d663ae 100644 --- a/spec/command-parsing.test.sh +++ b/spec/command-parsing.test.sh @@ -1,4 +1,5 @@ -# +## compare_shells: dash bash mksh + # Some nonsensical combinations which can all be detected at PARSE TIME. # All shells allow these, but right now OSH disallowed. # TODO: Run the parser on your whole corpus, and then if there are no errors, diff --git a/spec/func-parsing.test.sh b/spec/func-parsing.test.sh index b30c71053e..aff3831815 100644 --- a/spec/func-parsing.test.sh +++ b/spec/func-parsing.test.sh @@ -1,3 +1,4 @@ +## compare_shells: dash bash mksh #### Incomplete Function ## code: foo() diff --git a/test/spec.sh b/test/spec.sh index 1d63c05839..0a5108244b 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -216,10 +216,8 @@ builtin-history() { run-file builtin-history "$@" } -# dash and mksh don't implement 'dirs' builtin-dirs() { - sh-spec spec/builtin-dirs.test.sh \ - $BASH $ZSH $OSH_LIST "$@" + run-file builtin-dirs "$@" } builtin-vars() { @@ -277,15 +275,15 @@ builtin-special() { } builtin-times() { - sh-spec spec/builtin-times.test.sh $BASH $ZSH $OSH_LIST "$@" + run-file builtin-times "$@" } command-parsing() { - sh-spec spec/command-parsing.test.sh ${REF_SHELLS[@]} $OSH_LIST "$@" + run-file command-parsing "$@" } func-parsing() { - sh-spec spec/func-parsing.test.sh ${REF_SHELLS[@]} $OSH_LIST "$@" + run-file func-parsing "$@" } sh-func() { From 1e8120ee14c2e79ff0d7ccf0c146551f585b9573 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 Aug 2024 12:06:45 -0400 Subject: [PATCH 158/506] [core] Minor cleanup of signal handling - Make it clear that SIGINT is a special case in both AddUserTrap and RemoveUserTrap - Consistently throw OSError on sigaction() failure, rather than assert() - Rename some functions --- builtin/trap_osh.py | 18 +++++++++--------- core/process.py | 23 ++++++++++++----------- core/pyos.py | 4 +++- core/shell.py | 2 +- cpp/core.cc | 16 ++++++++++++---- cpp/core.h | 2 +- cpp/obj_layout_test.cc | 3 ++- 7 files changed, 40 insertions(+), 28 deletions(-) diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index 7fef379dda..49698d4480 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -75,13 +75,15 @@ def RemoveUserHook(self, hook_name): def AddUserTrap(self, sig_num, handler): # type: (int, command_t) -> None - """E.g. - - SIGUSR1. - """ + """ e.g. SIGUSR1 """ self.traps[sig_num] = handler - if sig_num == SIGWINCH: + if sig_num == SIGINT: + # Don't disturb the runtime signal handlers: + # 1. from CPython + # 2. pyos::InitSignalSafe() calls RegisterSignalInterest(SIGINT) + pass + elif sig_num == SIGWINCH: self.signal_safe.SetSigWinchCode(SIGWINCH) else: pyos.RegisterSignalInterest(sig_num) @@ -92,14 +94,12 @@ def RemoveUserTrap(self, sig_num): mylib.dict_erase(self.traps, sig_num) if sig_num == SIGINT: - # Don't disturb the runtime signal handlers: - # 1. from CPython - # 2. pyos::InitSignalSafe() calls RegisterSignalInterest(SIGINT) + # Same reason as above pass elif sig_num == SIGWINCH: self.signal_safe.SetSigWinchCode(pyos.UNTRAPPED_SIGWINCH) else: - pyos.Sigaction(sig_num, SIG_DFL) + pyos.sigaction(sig_num, SIG_DFL) def GetPendingTraps(self): # type: () -> Optional[List[command_t]] diff --git a/core/process.py b/core/process.py index a8ffba6463..a62884fc40 100644 --- a/core/process.py +++ b/core/process.py @@ -70,6 +70,7 @@ from _devbuild.gen.syntax_asdl import command_t from builtin import trap_osh from core import optview + from core import pyos from core.util import _DebugFile from osh.cmd_eval import CommandEvaluator @@ -107,21 +108,21 @@ def __exit__(self, type, value, traceback): self.f.close() -def InitInteractiveShell(): - # type: () -> None +def InitInteractiveShell(signal_safe): + # type: (pyos.SignalSafe) -> None """Called when initializing an interactive shell.""" # The shell itself should ignore Ctrl-\. - pyos.Sigaction(SIGQUIT, SIG_IGN) + pyos.sigaction(SIGQUIT, SIG_IGN) # This prevents Ctrl-Z from suspending OSH in interactive mode. - pyos.Sigaction(SIGTSTP, SIG_IGN) + pyos.sigaction(SIGTSTP, SIG_IGN) # More signals from # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html # (but not SIGCHLD) - pyos.Sigaction(SIGTTOU, SIG_IGN) - pyos.Sigaction(SIGTTIN, SIG_IGN) + pyos.sigaction(SIGTTOU, SIG_IGN) + pyos.sigaction(SIGTTIN, SIG_IGN) # Register a callback to receive terminal width changes. # NOTE: In line_input.c, we turned off rl_catch_sigwinch. @@ -1065,23 +1066,23 @@ def StartProcess(self, why): # shouldn't have this. # https://docs.python.org/2/library/signal.html # See Python/pythonrun.c. - pyos.Sigaction(SIGPIPE, SIG_DFL) + pyos.sigaction(SIGPIPE, SIG_DFL) # Respond to Ctrl-\ (core dump) - pyos.Sigaction(SIGQUIT, SIG_DFL) + pyos.sigaction(SIGQUIT, SIG_DFL) # Only standalone children should get Ctrl-Z. Pipelines remain in the # foreground because suspending them is difficult with our 'lastpipe' # semantics. pid = posix.getpid() if posix.getpgid(0) == pid and self.parent_pipeline is None: - pyos.Sigaction(SIGTSTP, SIG_DFL) + pyos.sigaction(SIGTSTP, SIG_DFL) # More signals from # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html # (but not SIGCHLD) - pyos.Sigaction(SIGTTOU, SIG_DFL) - pyos.Sigaction(SIGTTIN, SIG_DFL) + pyos.sigaction(SIGTTOU, SIG_DFL) + pyos.sigaction(SIGTTIN, SIG_DFL) self.tracer.OnNewProcess(pid) # clear foreground pipeline for subshells diff --git a/core/pyos.py b/core/pyos.py index 8a6f1a53b0..45907b8223 100644 --- a/core/pyos.py +++ b/core/pyos.py @@ -376,9 +376,11 @@ def InitSignalSafe(): return gSignalSafe -def Sigaction(sig_num, handler): +def sigaction(sig_num, handler): # type: (int, Any) -> None """Register a signal handler.""" + # SIGINT must be registered through SignalSafe + assert sig_num != signal.SIGINT signal.signal(sig_num, handler) diff --git a/core/shell.py b/core/shell.py index bbe2558a49..c1a3925224 100644 --- a/core/shell.py +++ b/core/shell.py @@ -1063,7 +1063,7 @@ def Main( display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state, debug_f) - process.InitInteractiveShell() # Set signal handlers + process.InitInteractiveShell(signal_safe) # Set signal handlers # The interactive shell leads a process group which controls the terminal. # It MUST give up the terminal afterward, otherwise we get SIGTTIN / diff --git a/cpp/core.cc b/cpp/core.cc index c08e065d25..5dfa7f466a 100644 --- a/cpp/core.cc +++ b/cpp/core.cc @@ -272,7 +272,13 @@ SignalSafe* InitSignalSafe() { return gSignalSafe; } -void Sigaction(int sig_num, void (*handler)(int)) { +// Note that the Python implementation of pyos.sigaction() calls +// signal.signal(), which calls PyOS_setsig(), which calls sigaction() #ifdef +// HAVE_SIGACTION. +void sigaction(int sig_num, void (*handler)(int)) { + // SIGINT must be registered through SignalSafe + DCHECK(sig_num != SIGINT); + struct sigaction act = {}; act.sa_handler = handler; if (sigaction(sig_num, &act, nullptr) != 0) { @@ -280,15 +286,17 @@ void Sigaction(int sig_num, void (*handler)(int)) { } } -static void signal_handler(int sig_num) { +static void OurSignalHandler(int sig_num) { assert(gSignalSafe != nullptr); gSignalSafe->UpdateFromSignalHandler(sig_num); } void RegisterSignalInterest(int sig_num) { struct sigaction act = {}; - act.sa_handler = signal_handler; - assert(sigaction(sig_num, &act, nullptr) == 0); + act.sa_handler = OurSignalHandler; + if (sigaction(sig_num, &act, nullptr) != 0) { + throw Alloc(errno); + } } Tuple2* MakeDirCacheKey(BigStr* path) { diff --git a/cpp/core.h b/cpp/core.h index ca16fe4356..08632dbd78 100644 --- a/cpp/core.h +++ b/cpp/core.h @@ -232,7 +232,7 @@ extern SignalSafe* gSignalSafe; // Allocate global and return it. SignalSafe* InitSignalSafe(); -void Sigaction(int sig_num, void (*handler)(int)); +void sigaction(int sig_num, void (*handler)(int)); void RegisterSignalInterest(int sig_num); diff --git a/cpp/obj_layout_test.cc b/cpp/obj_layout_test.cc index f17d55e7d4..100ca5394c 100644 --- a/cpp/obj_layout_test.cc +++ b/cpp/obj_layout_test.cc @@ -23,7 +23,8 @@ TEST sizeof_syntax() { // Reordered to be 16 bytes log("sizeof(runtime_asdl::Cell) = %d", sizeof(runtime_asdl::Cell)); // now 32 bytes, down from 56 - log("sizeof(runtime_asdl::cmd_value::Argv) = %d", sizeof(runtime_asdl::cmd_value::Argv)); + log("sizeof(runtime_asdl::cmd_value::Argv) = %d", + sizeof(runtime_asdl::cmd_value::Argv)); // 24 bytes: std::vector log("sizeof(List) = %d", sizeof(List)); From f5d29435edb9e129a05655342ed161ca84b8db48 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 Aug 2024 12:20:21 -0400 Subject: [PATCH 159/506] [cpp] Fix unit tests Also try to set up structure for fixing the trap INT bug Samuel reported Not successful so far. --- builtin/trap_osh.py | 16 +++++++++++----- core/pyos.py | 16 ++++++++++++++-- cpp/core.cc | 3 ++- cpp/core.h | 17 +++++++++++++++-- cpp/core_test.cc | 4 ++-- 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index 49698d4480..f80dd6597a 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -82,7 +82,7 @@ def AddUserTrap(self, sig_num, handler): # Don't disturb the runtime signal handlers: # 1. from CPython # 2. pyos::InitSignalSafe() calls RegisterSignalInterest(SIGINT) - pass + self.signal_safe.SetSigIntTrapped(True) elif sig_num == SIGWINCH: self.signal_safe.SetSigWinchCode(SIGWINCH) else: @@ -94,17 +94,23 @@ def RemoveUserTrap(self, sig_num): mylib.dict_erase(self.traps, sig_num) if sig_num == SIGINT: - # Same reason as above - pass + self.signal_safe.SetSigIntTrapped(False) elif sig_num == SIGWINCH: self.signal_safe.SetSigWinchCode(pyos.UNTRAPPED_SIGWINCH) else: + # TODO: In process.InitInteractiveShell(), 4 signals are set to + # SIG_IGN, not SIG_DFL: + # + # SIGQUIT SIGTSTP SIGTTOU SIGTTIN + # + # Should we restore them? It's rare that you type 'trap' in + # interactive shells, but it might be more correct. See what other + # shells do. pyos.sigaction(sig_num, SIG_DFL) def GetPendingTraps(self): # type: () -> Optional[List[command_t]] - """Transfer ownership of the current queue of pending trap handlers to - the caller.""" + """Transfer ownership of queue of pending trap handlers to caller.""" signals = self.signal_safe.TakePendingSignals() if 0: log('*** GetPendingTraps') diff --git a/core/pyos.py b/core/pyos.py index 45907b8223..fceed9303b 100644 --- a/core/pyos.py +++ b/core/pyos.py @@ -319,6 +319,14 @@ def PollSigInt(self): self.received_sigint = False return result + def SetSigIntTrapped(self, b): + # type: (bool) -> None + """Set a flag to tell us whether sigint is trapped by the user. + + Only needed in C++ + """ + pass + def SetSigWinchCode(self, code): # type: (int) -> None """Depending on whether or not SIGWINCH is trapped by a user, it is @@ -378,9 +386,13 @@ def InitSignalSafe(): def sigaction(sig_num, handler): # type: (int, Any) -> None - """Register a signal handler.""" - # SIGINT must be registered through SignalSafe + """ + Handle a signal with SIG_DFL or SIG_IGN, not our own signal handler. + """ + + # SIGINT and SIGWINCH must be registered through SignalSafe assert sig_num != signal.SIGINT + assert sig_num != signal.SIGWINCH signal.signal(sig_num, handler) diff --git a/cpp/core.cc b/cpp/core.cc index 5dfa7f466a..22fc0ab53e 100644 --- a/cpp/core.cc +++ b/cpp/core.cc @@ -276,8 +276,9 @@ SignalSafe* InitSignalSafe() { // signal.signal(), which calls PyOS_setsig(), which calls sigaction() #ifdef // HAVE_SIGACTION. void sigaction(int sig_num, void (*handler)(int)) { - // SIGINT must be registered through SignalSafe + // SIGINT and SIGWINCH must be registered through SignalSafe DCHECK(sig_num != SIGINT); + DCHECK(sig_num != SIGWINCH); struct sigaction act = {}; act.sa_handler = handler; diff --git a/cpp/core.h b/cpp/core.h index 08632dbd78..4770cfbb34 100644 --- a/cpp/core.h +++ b/cpp/core.h @@ -171,14 +171,26 @@ class SignalSafe { #endif } - // Main thread wants to know if SIGINT was received since the last time - // PollSigInt was called. + void SetSigIntTrapped(bool b) { + sigint_trapped_ = b; + } + + // Used by pyos.WaitPid, Read, ReadByte. bool PollSigInt() { bool result = received_sigint_; received_sigint_ = false; return result; } +#if 0 + // Used by osh/cmd_eval.py. Main loop wants to know if SIGINT was received + // since the last time PollSigInt was called. + bool PollUntrappedSigInt() { + bool received = PollSigInt(); // clears a flag + return received && sigint_trapped_; + } +#endif + // Main thread tells us whether SIGWINCH is trapped. void SetSigWinchCode(int code) { sigwinch_code_ = code; @@ -221,6 +233,7 @@ class SignalSafe { #endif // Not sufficient: volatile sig_atomic_t last_sig_num_; + bool sigint_trapped_; int received_sigint_; int received_sigwinch_; int sigwinch_code_; diff --git a/cpp/core_test.cc b/cpp/core_test.cc index c149b14669..03a807704f 100644 --- a/cpp/core_test.cc +++ b/cpp/core_test.cc @@ -266,7 +266,7 @@ TEST signal_test() { signal_safe->ReuseEmptyList(q); } - pyos::Sigaction(SIGUSR1, SIG_IGN); + pyos::sigaction(SIGUSR1, SIG_IGN); kill(mypid, SIGUSR1); { List* q = signal_safe->TakePendingSignals(); @@ -274,7 +274,7 @@ TEST signal_test() { ASSERT(len(q) == 0); signal_safe->ReuseEmptyList(q); } - pyos::Sigaction(SIGUSR2, SIG_IGN); + pyos::sigaction(SIGUSR2, SIG_IGN); pyos::RegisterSignalInterest(SIGWINCH); From 3e1fd034022846153cfbc57236831935bc931d2c Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 Aug 2024 12:56:34 -0400 Subject: [PATCH 160/506] [builtin/trap] Revert part of refactoring It inadvertently made a spec test pass: spec/builtin-trap.test.sh, case 20 I will restore the fix once it's clear why it happened! Ah I think this is because we removed a call to: pyos.RegisterSignalInterest(SIGINT) which calls signal.signal(SIGINT) which affects KeyboardInterrupt. This is related to the trap INT bug, but I'm not sure what the real fix is. OSH matches other shells now, besides mksh. But I thought mksh behavior was OK! --- builtin/trap_osh.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index f80dd6597a..5fcdb1a725 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -78,12 +78,15 @@ def AddUserTrap(self, sig_num, handler): """ e.g. SIGUSR1 """ self.traps[sig_num] = handler - if sig_num == SIGINT: - # Don't disturb the runtime signal handlers: - # 1. from CPython - # 2. pyos::InitSignalSafe() calls RegisterSignalInterest(SIGINT) - self.signal_safe.SetSigIntTrapped(True) - elif sig_num == SIGWINCH: + #if sig_num == SIGINT: + # Don't disturb the underlying runtime's SIGINT handllers + # 1. CPython has one for KeyboardInterrupt + # 2. mycpp runtime simulates KeyboardInterrupt: + # pyos::InitSignalSafe() calls RegisterSignalInterest(SIGINT), + # then we PollSigInt() in the osh/cmd_eval.py main loop + #self.signal_safe.SetSigIntTrapped(True) + # pass + if sig_num == SIGWINCH: self.signal_safe.SetSigWinchCode(SIGWINCH) else: pyos.RegisterSignalInterest(sig_num) @@ -94,7 +97,8 @@ def RemoveUserTrap(self, sig_num): mylib.dict_erase(self.traps, sig_num) if sig_num == SIGINT: - self.signal_safe.SetSigIntTrapped(False) + #self.signal_safe.SetSigIntTrapped(False) + pass elif sig_num == SIGWINCH: self.signal_safe.SetSigWinchCode(pyos.UNTRAPPED_SIGWINCH) else: From 373cc2af5caeb7083bfd79cb06b4c27e283c2036 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 20 Aug 2024 14:31:45 -0400 Subject: [PATCH 161/506] [cpp] Only throw KeyboardInterrupt when SIGINT is not trapped This matches CPython behavior. This fixes the bug Samuel reported on #oil-discuss-public > trap INT not running. test/bugs.sh trap-1 test/bugs.sh trap-2 test/bugs.sh sigint-parent-child However I think there is still an issue with the EXIT trap. --- builtin/trap_osh.py | 9 ++++----- core/pyos.py | 14 +++++++++++++- cpp/core.cc | 6 +++--- cpp/core.h | 4 +--- cpp/core_test.cc | 32 ++++++++++++++++++++++++++++++++ osh/cmd_eval.py | 2 +- test/bugs.sh | 24 ++++++++++++++++++++++-- test/signal-state.sh | 8 +++++++- 8 files changed, 83 insertions(+), 16 deletions(-) diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index 5fcdb1a725..eb415b34e5 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -78,15 +78,14 @@ def AddUserTrap(self, sig_num, handler): """ e.g. SIGUSR1 """ self.traps[sig_num] = handler - #if sig_num == SIGINT: + if sig_num == SIGINT: # Don't disturb the underlying runtime's SIGINT handllers # 1. CPython has one for KeyboardInterrupt # 2. mycpp runtime simulates KeyboardInterrupt: # pyos::InitSignalSafe() calls RegisterSignalInterest(SIGINT), # then we PollSigInt() in the osh/cmd_eval.py main loop - #self.signal_safe.SetSigIntTrapped(True) - # pass - if sig_num == SIGWINCH: + self.signal_safe.SetSigIntTrapped(True) + elif sig_num == SIGWINCH: self.signal_safe.SetSigWinchCode(SIGWINCH) else: pyos.RegisterSignalInterest(sig_num) @@ -97,7 +96,7 @@ def RemoveUserTrap(self, sig_num): mylib.dict_erase(self.traps, sig_num) if sig_num == SIGINT: - #self.signal_safe.SetSigIntTrapped(False) + self.signal_safe.SetSigIntTrapped(False) pass elif sig_num == SIGWINCH: self.signal_safe.SetSigWinchCode(pyos.UNTRAPPED_SIGWINCH) diff --git a/core/pyos.py b/core/pyos.py index fceed9303b..8c97ceb44b 100644 --- a/core/pyos.py +++ b/core/pyos.py @@ -312,6 +312,13 @@ def LastSignal(self): """Return the number of the last signal that fired.""" return self.last_sig_num + def PollUntrappedSigInt(self): + # type: () -> bool + """Has SIGINT received since the last time PollSigInt() was called?""" + result = self.received_sigint + self.received_sigint = False + return result + def PollSigInt(self): # type: () -> bool """Has SIGINT received since the last time PollSigInt() was called?""" @@ -353,7 +360,7 @@ def TakePendingSignals(self): # exclusivity should be maintained by the atomic nature of pointer # assignment (i.e. word-sized writes) on most modern platforms. # The replacement run list is allocated before the swap, so it can be - # interuppted at any point without consequence. + # interrupted at any point without consequence. # This means the signal handler always has exclusive access to # `self.pending_signals`. In the worst case the signal handler might write to # `new_queue` and the corresponding trap handler won't get executed @@ -381,6 +388,9 @@ def InitSignalSafe(): """Set global instance so the signal handler can access it.""" global gSignalSafe gSignalSafe = SignalSafe() + + RegisterSignalInterest(signal.SIGINT) + return gSignalSafe @@ -399,6 +409,8 @@ def sigaction(sig_num, handler): def RegisterSignalInterest(sig_num): # type: (int) -> None """Have the kernel notify the main loop about the given signal.""" + #log('RegisterSignalInterest %d', sig_num) + assert gSignalSafe is not None signal.signal(sig_num, gSignalSafe.UpdateFromSignalHandler) diff --git a/cpp/core.cc b/cpp/core.cc index 22fc0ab53e..fc846b954e 100644 --- a/cpp/core.cc +++ b/cpp/core.cc @@ -34,7 +34,7 @@ Tuple2 WaitPid(int waitpid_options) { int status; int result = ::waitpid(-1, &status, WUNTRACED | waitpid_options); if (result < 0) { - if (errno == EINTR && gSignalSafe->PollSigInt()) { + if (errno == EINTR && gSignalSafe->PollUntrappedSigInt()) { throw Alloc(); } return Tuple2(-1, errno); @@ -47,7 +47,7 @@ Tuple2 Read(int fd, int n, List* chunks) { int length = ::read(fd, s->data(), n); if (length < 0) { - if (errno == EINTR && gSignalSafe->PollSigInt()) { + if (errno == EINTR && gSignalSafe->PollUntrappedSigInt()) { throw Alloc(); } return Tuple2(-1, errno); @@ -67,7 +67,7 @@ Tuple2 ReadByte(int fd) { unsigned char buf[1]; ssize_t n = read(fd, &buf, 1); if (n < 0) { // read error - if (errno == EINTR && gSignalSafe->PollSigInt()) { + if (errno == EINTR && gSignalSafe->PollUntrappedSigInt()) { throw Alloc(); } return Tuple2(-1, errno); diff --git a/cpp/core.h b/cpp/core.h index 4770cfbb34..7e1576efc3 100644 --- a/cpp/core.h +++ b/cpp/core.h @@ -182,14 +182,12 @@ class SignalSafe { return result; } -#if 0 // Used by osh/cmd_eval.py. Main loop wants to know if SIGINT was received // since the last time PollSigInt was called. bool PollUntrappedSigInt() { bool received = PollSigInt(); // clears a flag - return received && sigint_trapped_; + return received && !sigint_trapped_; } -#endif // Main thread tells us whether SIGWINCH is trapped. void SetSigWinchCode(int code) { diff --git a/cpp/core_test.cc b/cpp/core_test.cc index 03a807704f..602ef976e5 100644 --- a/cpp/core_test.cc +++ b/cpp/core_test.cc @@ -5,6 +5,7 @@ #include // SIG*, kill() #include // stat #include // uname +#include // waitpid #include // getpid(), getuid(), environ #include "cpp/embedded_file.h" @@ -384,6 +385,35 @@ TEST asan_global_leak_test() { PASS(); } +// manual demo +TEST waitpid_demo() { + pyos::InitSignalSafe(); + pyos::RegisterSignalInterest(SIGINT); + + int result = fork(); + if (result < 0) { + FAIL(); + } else if (result == 0) { + // child + + log("sleeping in child, pid = %d", getpid()); + char* argv[] = {"sleep", "5", nullptr}; + char* env[] = {nullptr}; + int e = execvpe("sleep", argv, env); + log("execve failed %d", e); + + } else { + // parent + + int wstatus; + log("waiting in parent"); + int result = ::waitpid(-1, &wstatus, 0); + log("waitpid = %d, status = %d", result, wstatus); + } + + PASS(); +} + GREATEST_MAIN_DEFS(); int main(int argc, char** argv) { @@ -410,6 +440,8 @@ int main(int argc, char** argv) { RUN_TEST(dir_cache_key_test); RUN_TEST(asan_global_leak_test); + // RUN_TEST(waitpid_demo); + gHeap.CleanProcessExit(); GREATEST_MAIN_END(); /* display results */ diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index bdb40f6136..63abb9bb55 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1796,7 +1796,7 @@ def _Execute(self, node): # We only need this somewhat hacky check in osh-cpp since python's runtime # handles SIGINT for us in osh. if mylib.CPP: - if self.signal_safe.PollSigInt(): + if self.signal_safe.PollUntrappedSigInt(): raise KeyboardInterrupt() # Manual GC point before every statement diff --git a/test/bugs.sh b/test/bugs.sh index 4b7eb0639d..2612504033 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -50,7 +50,7 @@ trap-1() { set +o errexit # This fails to run the trap - $sh -x -c 'echo pid=$$; trap "echo int" INT; sleep 5' + $sh -x -c 'echo shell=$$; trap "echo int" INT; sleep 5' echo "$sh status=$?" } @@ -61,11 +61,31 @@ trap-2() { set +o errexit # This runs it - $sh -x -c 'echo pid=$$; trap "echo int" INT; sleep 5; echo last' + $sh -x -c 'echo shell=$$; trap "echo int" INT; sleep 5; echo last' echo "$sh status=$?" } +# Does Ctrl-C cause both signal handlers to run? Yes. +sigint-parent-child() { + local sh=${1:-bin/osh} + + cat > _tmp/sigint.py < Date: Wed, 21 Aug 2024 16:21:45 -0400 Subject: [PATCH 162/506] [osh-py] Revert change to unconditionally register SIGINT The last change broke some spec/stateful cases. We have to sort out the handling of KeyboardInterrupt in C++ and Python a bit more. --- core/pyos.py | 6 +++++- spec/builtin-trap.test.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/pyos.py b/core/pyos.py index 8c97ceb44b..1d91fd6795 100644 --- a/core/pyos.py +++ b/core/pyos.py @@ -389,7 +389,11 @@ def InitSignalSafe(): global gSignalSafe gSignalSafe = SignalSafe() - RegisterSignalInterest(signal.SIGINT) + # Note: we only need this in C++ because of the way Python's signal module + # works? See + # - demo/cpython/keyboard_interrupt.py + # - pyos::InitSignalSafe() + #RegisterSignalInterest(signal.SIGINT) return gSignalSafe diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index 216a3e5c7a..bf32a34a4e 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -1,5 +1,5 @@ ## compare_shells: dash bash mksh ash -## oils_failures_allowed: 2 +## oils_failures_allowed: 1 # builtin-trap.test.sh From 27de2dd623570da3836db956259ef385fb7c892e Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 19:17:35 -0400 Subject: [PATCH 163/506] [core/pyos refactor] Update comments, and start porting I think we want the shell process to unconditionally handle SIGINT. This means we don't deal with KeyboardInterrupt - we can have our own error.Interrupt or something. --- core/pyos.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/core/pyos.py b/core/pyos.py index 1d91fd6795..a09e49c7bd 100644 --- a/core/pyos.py +++ b/core/pyos.py @@ -5,6 +5,7 @@ """ from __future__ import print_function +#from errno import EINTR import pwd import resource import signal @@ -60,6 +61,8 @@ def WaitPid(waitpid_options): # - waitpid_options can be WNOHANG pid, status = posix.waitpid(-1, WUNTRACED | waitpid_options) except OSError as e: + #if e.errno == EINTR and gSignalSafe.PollUntrappedSigInt(): + # raise KeyboardInterrupt() return -1, e.errno return pid, status @@ -79,8 +82,9 @@ def __init__(self, err_num): def Read(fd, n, chunks): # type: (int, int, List[str]) -> Tuple[int, int] """C-style wrapper around Python's posix.read() that uses return values - instead of exceptions for errors. We will implement this directly in C++ - and not use exceptions at all. + instead of exceptions for errors. + + We will implement this directly in C++ and not use exceptions at all. It reads n bytes from the given file descriptor and appends it to chunks. @@ -101,8 +105,9 @@ def Read(fd, n, chunks): def ReadByte(fd): # type: (int) -> Tuple[int, int] - """Another low level interface with a return value interface. Used by - _ReadUntilDelim() and _ReadLineSlowly(). + """Low-level interface that returns values rather than raising exceptions. + + Used by _ReadUntilDelim() and _ReadLineSlowly(). Returns: failure: (-1, errno) on failure @@ -286,6 +291,7 @@ def __init__(self): # type: () -> None self.pending_signals = [] # type: List[int] self.last_sig_num = 0 # type: int + self.sigint_trapped = False self.received_sigint = False self.received_sigwinch = False self.sigwinch_code = UNTRAPPED_SIGWINCH @@ -309,37 +315,39 @@ def UpdateFromSignalHandler(self, sig_num, unused_frame): def LastSignal(self): # type: () -> int - """Return the number of the last signal that fired.""" + """Return the number of the last signal received.""" return self.last_sig_num - def PollUntrappedSigInt(self): + def PollSigInt(self): # type: () -> bool """Has SIGINT received since the last time PollSigInt() was called?""" result = self.received_sigint self.received_sigint = False return result - def PollSigInt(self): + def PollUntrappedSigInt(self): # type: () -> bool """Has SIGINT received since the last time PollSigInt() was called?""" - result = self.received_sigint - self.received_sigint = False - return result + received = self.PollSigInt() + return received and not self.sigint_trapped + + if 0: + + def SigIntTrapped(self): + # type: () -> bool + return self.sigint_trapped def SetSigIntTrapped(self, b): # type: (bool) -> None - """Set a flag to tell us whether sigint is trapped by the user. - - Only needed in C++ - """ - pass + """Set a flag to tell us whether sigint is trapped by the user.""" + self.sigint_trapped = b def SetSigWinchCode(self, code): # type: (int) -> None """Depending on whether or not SIGWINCH is trapped by a user, it is expected to report a different code to `wait`. - SetSigwinchCode() lets us set which code is reported. + SetSigWinchCode() lets us set which code is reported. """ self.sigwinch_code = code @@ -353,6 +361,8 @@ def PollSigWinch(self): def TakePendingSignals(self): # type: () -> List[int] + """Transfer ownership of queue of pending signals to caller.""" + # A note on signal-safety here. The main loop might be calling this function # at the same time a signal is firing and appending to # `self.pending_signals`. We can forgoe using a lock here From b4eaaf6abfdedba0e032c72bc01498dc6fb8ce13 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 20:05:02 -0400 Subject: [PATCH 164/506] [osh-py] SIGINT handler is always on, as in C++ This moves the two implementations closer together. There are still some divergences in spec/builtin-trap to fix. I added a hack to restore the default SIGINT handler while running CPython's raw_input(). This could be changed to a fork of raw_input(). It only affects the dev build though, so it's not high priority. In addition to spec tests and spec/stateful, I manually tested that we fix the "trap INT not running" bug from Samuel in both C++ and Python. --- core/pyos.py | 23 +++++++++++++++++------ core/shell.py | 3 +++ demo/cpython/keyboard_interrupt.py | 4 +++- frontend/reader.py | 22 +++++++++++++++++++++- osh/cmd_eval.py | 3 ++- spec/builtin-trap.test.sh | 2 +- 6 files changed, 47 insertions(+), 10 deletions(-) diff --git a/core/pyos.py b/core/pyos.py index a09e49c7bd..555fa7960d 100644 --- a/core/pyos.py +++ b/core/pyos.py @@ -5,7 +5,7 @@ """ from __future__ import print_function -#from errno import EINTR +from errno import EINTR import pwd import resource import signal @@ -61,8 +61,8 @@ def WaitPid(waitpid_options): # - waitpid_options can be WNOHANG pid, status = posix.waitpid(-1, WUNTRACED | waitpid_options) except OSError as e: - #if e.errno == EINTR and gSignalSafe.PollUntrappedSigInt(): - # raise KeyboardInterrupt() + if e.errno == EINTR and gSignalSafe.PollUntrappedSigInt(): + raise KeyboardInterrupt() return -1, e.errno return pid, status @@ -95,6 +95,8 @@ def Read(fd, n, chunks): try: chunk = posix.read(fd, n) except OSError as e: + if e.errno == EINTR and gSignalSafe.PollUntrappedSigInt(): + raise KeyboardInterrupt() return -1, e.errno else: length = len(chunk) @@ -116,6 +118,8 @@ def ReadByte(fd): try: b = posix.read(fd, 1) except OSError as e: + if e.errno == EINTR and gSignalSafe.PollUntrappedSigInt(): + raise KeyboardInterrupt() return -1, e.errno else: if len(b): @@ -392,6 +396,8 @@ def ReuseEmptyList(self, empty_list): gSignalSafe = None # type: SignalSafe +gOrigSigIntHandler = None # type: Any + def InitSignalSafe(): # type: () -> SignalSafe @@ -399,11 +405,16 @@ def InitSignalSafe(): global gSignalSafe gSignalSafe = SignalSafe() - # Note: we only need this in C++ because of the way Python's signal module - # works? See + # See # - demo/cpython/keyboard_interrupt.py # - pyos::InitSignalSafe() - #RegisterSignalInterest(signal.SIGINT) + + # In C++, we do + # RegisterSignalInterest(signal.SIGINT) + + global gOrigSigIntHandler + gOrigSigIntHandler = signal.signal(signal.SIGINT, + gSignalSafe.UpdateFromSignalHandler) return gSignalSafe diff --git a/core/shell.py b/core/shell.py index c1a3925224..2609c1d2b7 100644 --- a/core/shell.py +++ b/core/shell.py @@ -473,6 +473,9 @@ def Main( multi_trace) fd_state.tracer = tracer # circular dep + # RegisterSignalInterest should return old sigint handler + # then InteractiveLineReader can use it + # InteractiveLineReader signal_safe = pyos.InitSignalSafe() trap_state = trap_osh.TrapState(signal_safe) diff --git a/demo/cpython/keyboard_interrupt.py b/demo/cpython/keyboard_interrupt.py index afcaa2bfdd..edcbf3f6d9 100755 --- a/demo/cpython/keyboard_interrupt.py +++ b/demo/cpython/keyboard_interrupt.py @@ -22,7 +22,9 @@ def main(argv): # This suppresses KeyboardInterrupt. You can still do Ctrl-\ or check a flag # and throw your own exception. - signal.signal(signal.SIGINT, SigInt) + old = signal.signal(signal.SIGINT, SigInt) + # We may want to restore the old handler! + print(old) while True: print('----') diff --git a/frontend/reader.py b/frontend/reader.py index fcdbad85f2..a03c0d7298 100644 --- a/frontend/reader.py +++ b/frontend/reader.py @@ -210,6 +210,26 @@ def Reset(self): """Called after command execution.""" self.render_ps1 = True + def _ReadlinePromptInput(self): + # type: () -> str + if mylib.CPP: + line = self.line_input.prompt_input(self.prompt_str) + else: + # Hack to restore CPython's signal handling behavior while + # raw_input() is called. + # + # A cleaner way to do this would be to fork CPython's raw_input() + # so it handles EINTR. It's called in frontend/pyreadline.py + import signal + from core import pyos + + tmp = signal.signal(signal.SIGINT, pyos.gOrigSigIntHandler) + try: + line = self.line_input.prompt_input(self.prompt_str) + finally: + signal.signal(signal.SIGINT, tmp) + return line + def _GetLine(self): # type: () -> Optional[str] @@ -229,7 +249,7 @@ def _GetLine(self): not mylib.Stdin().isatty()): line = _PlainPromptInput(self.prompt_str) else: - line = self.line_input.prompt_input(self.prompt_str) + line = self._ReadlinePromptInput() except EOFError: print('^D') # bash prints 'exit'; mksh prints ^D. diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 63abb9bb55..c68cf3ecc4 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1795,7 +1795,8 @@ def _Execute(self, node): # We only need this somewhat hacky check in osh-cpp since python's runtime # handles SIGINT for us in osh. - if mylib.CPP: + #if mylib.CPP: + if 1: if self.signal_safe.PollUntrappedSigInt(): raise KeyboardInterrupt() diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index bf32a34a4e..2cf25dba0d 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -1,5 +1,5 @@ ## compare_shells: dash bash mksh ash -## oils_failures_allowed: 1 +## oils_failures_allowed: 3 # builtin-trap.test.sh From 7304da719d60872dbc49ff7dbd4727f82093a0c9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 22:02:20 -0400 Subject: [PATCH 165/506] [core] Fix missing trap EXIT after SIGINT The batch shell now handles KeyboardInterrupt, as the interactive shell does. --- bin/oils_for_unix.py | 3 +++ core/shell.py | 3 +++ spec/builtin-trap.test.sh | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/oils_for_unix.py b/bin/oils_for_unix.py index 41cb39c39b..c4300a85dc 100755 --- a/bin/oils_for_unix.py +++ b/bin/oils_for_unix.py @@ -175,6 +175,9 @@ def main(argv): return 2 except KeyboardInterrupt: + # The interactive shell and the batch shell both handle + # KeyboardInterrupt themselves. + # This is a catch-all for --tool and so forth. print('') return 130 # 128 + 2 diff --git a/core/shell.py b/core/shell.py index 2609c1d2b7..cee29984a4 100644 --- a/core/shell.py +++ b/core/shell.py @@ -1171,6 +1171,9 @@ def Main( cmd_flags=cmd_eval.IsMainProgram) except util.UserExit as e: status = e.status + except KeyboardInterrupt: + # The interactive shell handles this in main_loop.Interactive + status = 130 # 128 + 2 mut_status = IntParamBox(status) cmd_ev.RunTrapsOnExit(mut_status) diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index 2cf25dba0d..216a3e5c7a 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -1,5 +1,5 @@ ## compare_shells: dash bash mksh ash -## oils_failures_allowed: 3 +## oils_failures_allowed: 2 # builtin-trap.test.sh From e5834f819ced353c77c849494fdff7f9dc2d98ca Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 22:23:43 -0400 Subject: [PATCH 166/506] [spec/builtin-trap] It is OK to run the SIGINT trap Like mksh does sometimes --- spec/builtin-trap.test.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index 216a3e5c7a..c460dffc75 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -1,13 +1,13 @@ ## compare_shells: dash bash mksh ash -## oils_failures_allowed: 2 +## oils_failures_allowed: 1 # builtin-trap.test.sh #### trap accepts/ignores -- trap -- 'echo hi' EXIT -echo done +echo ok ## STDOUT: -done +ok hi ## END @@ -297,6 +297,14 @@ status=0 mksh ## END +# Not sure why other shells differ here, but running the trap is consistent +# with interactive cases in test/bugs.sh + +## OK osh STDOUT: +int +status=0 +## END + #### trap EXIT, sleep, SIGINT: non-interactively $REPO_ROOT/spec/testdata/builtin-trap-exit.sh From 245e99ac2404b76d4fcd6cf5a4a432350a94aaab Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 22:34:19 -0400 Subject: [PATCH 167/506] [spec/builtin-trap] Adjust timing There are some osh-py failures in the CI, but only in the cpp-spec job. I can't reproduce them locally, so they may be timing related. --- spec/testdata/builtin-trap-exit.sh | 2 +- spec/testdata/builtin-trap-int.sh | 2 +- spec/testdata/builtin-trap-usr1.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/testdata/builtin-trap-exit.sh b/spec/testdata/builtin-trap-exit.sh index 6ef722c03d..779fa1b066 100755 --- a/spec/testdata/builtin-trap-exit.sh +++ b/spec/testdata/builtin-trap-exit.sh @@ -2,7 +2,7 @@ # Why don't other shells run this trap? It's not a subshell $SH -c 'trap "echo on exit" EXIT; sleep 0.1' & -sleep 0.05 +sleep 0.02 # Note: this is SIGINT, for the KeyboardInterrupt problem $(command -v kill) -INT $! diff --git a/spec/testdata/builtin-trap-int.sh b/spec/testdata/builtin-trap-int.sh index 520c05e668..13585ff00a 100755 --- a/spec/testdata/builtin-trap-int.sh +++ b/spec/testdata/builtin-trap-int.sh @@ -7,7 +7,7 @@ $SH -c 'trap "echo int" INT; sleep 0.1' & -sleep 0.05 +sleep 0.02 $(command -v kill) -INT $! diff --git a/spec/testdata/builtin-trap-usr1.sh b/spec/testdata/builtin-trap-usr1.sh index 070f52653e..af39f3bd96 100755 --- a/spec/testdata/builtin-trap-usr1.sh +++ b/spec/testdata/builtin-trap-usr1.sh @@ -3,7 +3,7 @@ $SH -c 'trap "echo usr1" USR1; sleep 0.1' & #$SH -c 'trap "echo int" INT; sleep 0.1' & -sleep 0.05 +sleep 0.02 $(which kill) -USR1 $! From f7dba489b0156bc2086ef8c5610f503b6337315d Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 22:37:21 -0400 Subject: [PATCH 168/506] [spec/builtin-trap] Move 'kill' out of the way Try to reduce timing issues. --- spec/testdata/builtin-trap-exit.sh | 5 +++-- spec/testdata/builtin-trap-int.sh | 6 ++++-- spec/testdata/builtin-trap-usr1.sh | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/spec/testdata/builtin-trap-exit.sh b/spec/testdata/builtin-trap-exit.sh index 779fa1b066..66aeed0dcd 100755 --- a/spec/testdata/builtin-trap-exit.sh +++ b/spec/testdata/builtin-trap-exit.sh @@ -1,11 +1,12 @@ +kill=$(command -v kill) # Why don't other shells run this trap? It's not a subshell $SH -c 'trap "echo on exit" EXIT; sleep 0.1' & -sleep 0.02 +sleep 0.05 # Note: this is SIGINT, for the KeyboardInterrupt problem -$(command -v kill) -INT $! +$kill -INT $! wait diff --git a/spec/testdata/builtin-trap-int.sh b/spec/testdata/builtin-trap-int.sh index 13585ff00a..1c03f63873 100755 --- a/spec/testdata/builtin-trap-int.sh +++ b/spec/testdata/builtin-trap-int.sh @@ -5,11 +5,13 @@ # We need some other way to kill it with SIGINT +kill=$(command -v kill) + $SH -c 'trap "echo int" INT; sleep 0.1' & -sleep 0.02 +sleep 0.05 -$(command -v kill) -INT $! +$kill -INT $! wait diff --git a/spec/testdata/builtin-trap-usr1.sh b/spec/testdata/builtin-trap-usr1.sh index af39f3bd96..b7e31b67ea 100755 --- a/spec/testdata/builtin-trap-usr1.sh +++ b/spec/testdata/builtin-trap-usr1.sh @@ -1,11 +1,11 @@ -# Why don't other shells run this trap? It's not a subshell +kill=$(command -v kill) + $SH -c 'trap "echo usr1" USR1; sleep 0.1' & -#$SH -c 'trap "echo int" INT; sleep 0.1' & -sleep 0.02 +sleep 0.05 -$(which kill) -USR1 $! +$kill -USR1 $! wait From 24ce8b2ee8de635a0c1c27ce7c7a2e809735709f Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 22:47:28 -0400 Subject: [PATCH 169/506] [spec/builtin-trap] Adjust timing again, so it doesn't fail in CI The previous values were flaky on Github Actions --- spec/testdata/builtin-trap-exit.sh | 4 ++-- spec/testdata/builtin-trap-int.sh | 4 ++-- spec/testdata/builtin-trap-usr1.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spec/testdata/builtin-trap-exit.sh b/spec/testdata/builtin-trap-exit.sh index 66aeed0dcd..44f2c55e61 100755 --- a/spec/testdata/builtin-trap-exit.sh +++ b/spec/testdata/builtin-trap-exit.sh @@ -1,9 +1,9 @@ kill=$(command -v kill) # Why don't other shells run this trap? It's not a subshell -$SH -c 'trap "echo on exit" EXIT; sleep 0.1' & +$SH -c 'trap "echo on exit" EXIT; sleep 0.2' & -sleep 0.05 +sleep 0.1 # Note: this is SIGINT, for the KeyboardInterrupt problem $kill -INT $! diff --git a/spec/testdata/builtin-trap-int.sh b/spec/testdata/builtin-trap-int.sh index 1c03f63873..51a21198f9 100755 --- a/spec/testdata/builtin-trap-int.sh +++ b/spec/testdata/builtin-trap-int.sh @@ -7,9 +7,9 @@ kill=$(command -v kill) -$SH -c 'trap "echo int" INT; sleep 0.1' & +$SH -c 'trap "echo int" INT; sleep 0.2' & -sleep 0.05 +sleep 0.1 $kill -INT $! diff --git a/spec/testdata/builtin-trap-usr1.sh b/spec/testdata/builtin-trap-usr1.sh index b7e31b67ea..6b5df0e343 100755 --- a/spec/testdata/builtin-trap-usr1.sh +++ b/spec/testdata/builtin-trap-usr1.sh @@ -1,9 +1,9 @@ kill=$(command -v kill) -$SH -c 'trap "echo usr1" USR1; sleep 0.1' & +$SH -c 'trap "echo usr1" USR1; sleep 0.2' & -sleep 0.05 +sleep 0.1 $kill -USR1 $! From 90090e5dbb2b6124141fec154d176f6ed9d4cd2d Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 23:30:48 -0400 Subject: [PATCH 170/506] [spec/builtin-trap] Remove obsolete test case We don't have shopt -s no_fork_last because the optimizations are essential for pipelines, subshells, etc. --- spec/builtin-trap.test.sh | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index c460dffc75..b46035eb5e 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -154,32 +154,6 @@ pipeline EXIT TRAP ## END -#### trap EXIT doesn't run with shopt -s no_fork_last - -# There doesn't seem to be a way to get it to run, so specify that it doesn't - -$SH -c 'trap "echo exit1" EXIT; /bin/true' - -# newline -$SH -c 'trap "echo exit2" EXIT; /bin/true -' - -# Newline makes a difference! -# It doesn't get a chance to run -$SH -c 'shopt -s no_fork_last -trap "echo exit3" EXIT; /bin/true' - -## STDOUT: -exit1 -exit2 -## END - -## N-I dash/bash/mksh/ash STDOUT: -exit1 -exit2 -exit3 -## END - #### trap 0 is equivalent to EXIT # not sure why this is, but POSIX wants it. trap 'echo EXIT' 0 From 2cc1f482fcfe2c5a2d3cc8a3b645bba176888ebd Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 23:43:15 -0400 Subject: [PATCH 171/506] [spec/builtin-trap] Fix allowed failures --- spec/builtin-trap.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index b46035eb5e..e416bbbda3 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -1,5 +1,5 @@ ## compare_shells: dash bash mksh ash -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 # builtin-trap.test.sh From 5635886bc0660ccd589526e619dc5621b56b9bda Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 21 Aug 2024 23:40:39 -0400 Subject: [PATCH 172/506] [CommandEvaluator refactor] Re-order _Dispatch to commands By itself, this doesn't move benchmarks2/gc-cachegrind at all. That's just testing Fibonacci. Next change: I want to see if we can RunPendingTraps less often. --- osh/cmd_eval.py | 133 ++++++++++++++++++++++++------------------------ 1 file changed, 66 insertions(+), 67 deletions(-) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index c68cf3ecc4..ae7d1a2822 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1548,11 +1548,16 @@ def _Dispatch(self, node, cmd_st): cmd_st.check_errexit = True status = self._DoSimple(node, cmd_st) - elif case(command_e.ExpandedAlias): - node = cast(command.ExpandedAlias, UP_node) - status = self._DoExpandedAlias(node) + elif case(command_e.ShAssignment): # LEAF command + node = cast(command.ShAssignment, UP_node) - elif case(command_e.Sentence): + self.mem.SetTokenForLine(node.pairs[0].left) + self._MaybeRunDebugTrap() + + # Only unqualified assignment a=b + status = self._DoShAssignment(node, cmd_st) + + elif case(command_e.Sentence): # NOT leaf, but put it up front node = cast(command.Sentence, UP_node) # Don't check_errexit since this isn't a leaf command @@ -1561,31 +1566,6 @@ def _Dispatch(self, node, cmd_st): else: status = self.shell_ex.RunBackgroundJob(node.child) - elif case(command_e.Redirect): - node = cast(command.Redirect, UP_node) - - # set -e affects redirect error, like mksh and bash 5.2, but unlike - # dash/ash - cmd_st.check_errexit = True - status = self._DoRedirect(node, cmd_st) - - elif case(command_e.Pipeline): - node = cast(command.Pipeline, UP_node) - status = self._DoPipeline(node, cmd_st) - - elif case(command_e.Subshell): - node = cast(command.Subshell, UP_node) - - # This is a leaf from the parent process POV - cmd_st.check_errexit = True - - if node.is_last_cmd: - # If the subshell is the last command in the process, just - # run it in this process. See _MarkLastCommands(). - status = self._Execute(node.child) - else: - status = self.shell_ex.RunSubshell(node.child) - elif case(command_e.DBracket): # LEAF command node = cast(command.DBracket, UP_node) @@ -1620,29 +1600,23 @@ def _Dispatch(self, node, cmd_st): status = self._DoControlFlow(node) - elif case(command_e.VarDecl): # LEAF command + elif case(command_e.NoOp): # LEAF + status = 0 # make it true + + elif case(command_e.VarDecl): # YSH LEAF command node = cast(command.VarDecl, UP_node) # Point to var name (bare assignment has no keyword) self.mem.SetTokenForLine(node.lhs[0].left) status = self._DoVarDecl(node) - elif case(command_e.Mutation): # LEAF command + elif case(command_e.Mutation): # YSH LEAF command node = cast(command.Mutation, UP_node) self.mem.SetTokenForLine(node.keyword) # point to setvar/set self._DoMutation(node) status = 0 # if no exception is thrown, it succeeds - elif case(command_e.ShAssignment): # LEAF command - node = cast(command.ShAssignment, UP_node) - - self.mem.SetTokenForLine(node.pairs[0].left) - self._MaybeRunDebugTrap() - - # Only unqualified assignment a=b - status = self._DoShAssignment(node, cmd_st) - elif case(command_e.Expr): # YSH LEAF command node = cast(command.Expr, UP_node) @@ -1661,6 +1635,14 @@ def _Dispatch(self, node, cmd_st): val = self.expr_ev.EvalExpr(node.val, node.keyword) raise vm.ValueControlFlow(node.keyword, val) + # + # More commands that involve recursive calls + # + + elif case(command_e.ExpandedAlias): + node = cast(command.ExpandedAlias, UP_node) + status = self._DoExpandedAlias(node) + # Note CommandList and DoGroup have no redirects, but BraceGroup does. # DoGroup has 'do' and 'done' spids for translation. elif case(command_e.CommandList): @@ -1679,6 +1661,23 @@ def _Dispatch(self, node, cmd_st): node = cast(command.AndOr, UP_node) status = self._DoAndOr(node, cmd_st) + elif case(command_e.If): + node = cast(command.If, UP_node) + + # No SetTokenForLine() because + # - $LINENO can't appear directly in 'if' + # - 'if' doesn't directly cause errors + # It will be taken care of by command.Simple, condition, etc. + status = self._DoIf(node) + + elif case(command_e.Case): + node = cast(command.Case, UP_node) + + # Must set location for 'case $LINENO' + self.mem.SetTokenForLine(node.case_kw) + self._MaybeRunDebugTrap() + status = self._DoCase(node) + elif case(command_e.WhileUntil): node = cast(command.WhileUntil, UP_node) @@ -1697,6 +1696,31 @@ def _Dispatch(self, node, cmd_st): self.mem.SetTokenForLine(node.keyword) # for x in $LINENO status = self._DoForExpr(node) + elif case(command_e.Redirect): + node = cast(command.Redirect, UP_node) + + # set -e affects redirect error, like mksh and bash 5.2, but unlike + # dash/ash + cmd_st.check_errexit = True + status = self._DoRedirect(node, cmd_st) + + elif case(command_e.Pipeline): + node = cast(command.Pipeline, UP_node) + status = self._DoPipeline(node, cmd_st) + + elif case(command_e.Subshell): + node = cast(command.Subshell, UP_node) + + # This is a leaf from the parent process POV + cmd_st.check_errexit = True + + if node.is_last_cmd: + # If the subshell is the last command in the process, just + # run it in this process. See _MarkLastCommands(). + status = self._Execute(node.child) + else: + status = self.shell_ex.RunSubshell(node.child) + elif case(command_e.ShFunction): node = cast(command.ShFunction, UP_node) self._DoShFunction(node) @@ -1716,26 +1740,6 @@ def _Dispatch(self, node, cmd_st): self._DoFunc(node) status = 0 - elif case(command_e.If): - node = cast(command.If, UP_node) - - # No SetTokenForLine() because - # - $LINENO can't appear directly in 'if' - # - 'if' doesn't directly cause errors - # It will be taken care of by command.Simple, condition, etc. - status = self._DoIf(node) - - elif case(command_e.NoOp): - status = 0 # make it true - - elif case(command_e.Case): - node = cast(command.Case, UP_node) - - # Must set location for 'case $LINENO' - self.mem.SetTokenForLine(node.case_kw) - self._MaybeRunDebugTrap() - status = self._DoCase(node) - elif case(command_e.TimeBlock): node = cast(command.TimeBlock, UP_node) status = self._DoTimeBlock(node) @@ -1792,13 +1796,8 @@ def _Execute(self, node): # call self.DoTick()? That will RunPendingTraps and check the Ctrl-C flag, # and maybe throw an exception. self.RunPendingTraps() - - # We only need this somewhat hacky check in osh-cpp since python's runtime - # handles SIGINT for us in osh. - #if mylib.CPP: - if 1: - if self.signal_safe.PollUntrappedSigInt(): - raise KeyboardInterrupt() + if self.signal_safe.PollUntrappedSigInt(): + raise KeyboardInterrupt() # Manual GC point before every statement mylib.MaybeCollect() From 7f082dff9449dfd1cf46d8beeb77828f6eebeaf1 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 22 Aug 2024 00:03:17 -0400 Subject: [PATCH 173/506] [CommandEvaluator] GC and check for pending traps less often. I introduced the _LeafTick() function, and it's now called in SOME branches of _Dispatch(). Prior to this change, we periodic GC/trap work in _Execute(), which is like calling it at every node in the AST, rather than just the leaves. This gives a tiny improvement on benchmarks2/gc-cachegrind. But the I also think it makes the code clearer, e.g. when tracing when traps are run. --- osh/cmd_eval.py | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index ae7d1a2822..f6ad542e69 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1523,6 +1523,27 @@ def _DoRedirect(self, node, cmd_st): return status + def _LeafTick(self): + # type: () -> None + """Do periodic work while executing shell. + + We may run traps, check for Ctrl-C, or garbage collect. + """ + # TODO: Do this in "leaf" nodes? SimpleCommand, DBracket, DParen should + # call self.DoTick()? That will RunPendingTraps and check the Ctrl-C flag, + # and maybe throw an exception. + self.RunPendingTraps() + if self.signal_safe.PollUntrappedSigInt(): + raise KeyboardInterrupt() + + # TODO: Does this mess up control flow analysis? If so, we can move it + # back to the top of _Execute(), so there are fewer conditionals + # involved. This function gets called in SOME branches of + # self._Dispatch(). + + # Manual GC point before every statement + mylib.MaybeCollect() + def _Dispatch(self, node, cmd_st): # type: (command_t, CommandStatus) -> int """Switch on the command_t variants and execute them.""" @@ -1547,6 +1568,7 @@ def _Dispatch(self, node, cmd_st): self._MaybeRunDebugTrap() cmd_st.check_errexit = True status = self._DoSimple(node, cmd_st) + self._LeafTick() elif case(command_e.ShAssignment): # LEAF command node = cast(command.ShAssignment, UP_node) @@ -1556,6 +1578,7 @@ def _Dispatch(self, node, cmd_st): # Only unqualified assignment a=b status = self._DoShAssignment(node, cmd_st) + self._LeafTick() elif case(command_e.Sentence): # NOT leaf, but put it up front node = cast(command.Sentence, UP_node) @@ -1578,6 +1601,7 @@ def _Dispatch(self, node, cmd_st): cmd_st.show_code = True # this is a "leaf" for errors result = self.bool_ev.EvalB(node.expr) status = 0 if result else 1 + self._LeafTick() elif case(command_e.DParen): # LEAF command node = cast(command.DParen, UP_node) @@ -1591,6 +1615,7 @@ def _Dispatch(self, node, cmd_st): cmd_st.show_code = True # this is a "leaf" for errors i = self.arith_ev.EvalToBigInt(node.child) status = 1 if mops.Equal(i, mops.ZERO) else 0 + self._LeafTick() elif case(command_e.ControlFlow): # LEAF command node = cast(command.ControlFlow, UP_node) @@ -1599,6 +1624,7 @@ def _Dispatch(self, node, cmd_st): self._MaybeRunDebugTrap() status = self._DoControlFlow(node) + # Omit _LeafTick() since we likely raise an exception above elif case(command_e.NoOp): # LEAF status = 0 # make it true @@ -1609,6 +1635,7 @@ def _Dispatch(self, node, cmd_st): # Point to var name (bare assignment has no keyword) self.mem.SetTokenForLine(node.lhs[0].left) status = self._DoVarDecl(node) + self._LeafTick() elif case(command_e.Mutation): # YSH LEAF command node = cast(command.Mutation, UP_node) @@ -1616,6 +1643,7 @@ def _Dispatch(self, node, cmd_st): self.mem.SetTokenForLine(node.keyword) # point to setvar/set self._DoMutation(node) status = 0 # if no exception is thrown, it succeeds + self._LeafTick() elif case(command_e.Expr): # YSH LEAF command node = cast(command.Expr, UP_node) @@ -1624,6 +1652,7 @@ def _Dispatch(self, node, cmd_st): # YSH debug trap? status = self._DoExpr(node) + self._LeafTick() elif case(command_e.Retval): # YSH LEAF command node = cast(command.Retval, UP_node) @@ -1633,6 +1662,8 @@ def _Dispatch(self, node, cmd_st): # dialect, for speed? val = self.expr_ev.EvalExpr(node.val, node.keyword) + self._LeafTick() + raise vm.ValueControlFlow(node.keyword, val) # @@ -1788,19 +1819,7 @@ def RunPendingTrapsAndCatch(self): def _Execute(self, node): # type: (command_t) -> int - """Call _Dispatch(), and performs the errexit check. - - Also runs trap handlers. - """ - # TODO: Do this in "leaf" nodes? SimpleCommand, DBracket, DParen should - # call self.DoTick()? That will RunPendingTraps and check the Ctrl-C flag, - # and maybe throw an exception. - self.RunPendingTraps() - if self.signal_safe.PollUntrappedSigInt(): - raise KeyboardInterrupt() - - # Manual GC point before every statement - mylib.MaybeCollect() + """Call _Dispatch(), and perform the errexit check.""" # Optimization: These 2 records have rarely-used lists, so we don't pass # alloc_lists=True. We create them on demand. From 4dc04d5ff3e79b38425e9a0e2ae3aa56b3dbc268 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 23 Aug 2024 10:25:40 -0400 Subject: [PATCH 174/506] [doc] Document new method calls rules in proc-func, ysh-tour Update the description in ref/chap-expr-lang. --- doc/proc-func.md | 29 ++++++++++++++++++++++++++++- doc/ref/chap-expr-lang.md | 34 ++++++++++++++++++++-------------- doc/ysh-tour.md | 15 ++++++++++----- 3 files changed, 58 insertions(+), 20 deletions(-) diff --git a/doc/proc-func.md b/doc/proc-func.md index 932fdd07d9..7b9166d09a 100644 --- a/doc/proc-func.md +++ b/doc/proc-func.md @@ -153,7 +153,7 @@ Procs may start external processes and pipelines. Can perform I/O anywhere. -Funcs need an explicit `value.IO` param to perform I/O. +Funcs need an explicit `io` param to perform I/O. @@ -255,6 +255,18 @@ Any type of value, e.g. + + Relation to Objects + none + + +May be bound to objects: + + var x = obj.myMethod() + call obj->myMutatingMethod() + + + Interface Evolution @@ -743,6 +755,21 @@ An "open" proc is nearly is nearly identical to a shell function: write 'args are' @ARGV } +## Methods are Funcs Bound to Objects + +Values of type `Obj` have an ordered set of name-value bindings, as well as a +prototype chain of more `Obj` instances ("parents"). They support these +operators: + +- dot (`.`) looks for attributes or methods with a given name. + - Reference: [ysh-attr](ref/chap-expr-lang.html#ysh-attr) + - Attributes may be in the object, or up the chain. They are returned + literally. + - Methods live up the chain. They are returned as `BoundFunc`, so that the + first `self` argument of a method call is the object itself. +- Thin arrow (`->`) looks for mutating methods, which have an `M/` prefix. + - Reference: [thin-arrow](ref/chap-expr-lang.html#thin-arrow) + ## Usage Notes ### 3 Ways to Return a Value diff --git a/doc/ref/chap-expr-lang.md b/doc/ref/chap-expr-lang.md index 31a331c145..5b29c28331 100644 --- a/doc/ref/chap-expr-lang.md +++ b/doc/ref/chap-expr-lang.md @@ -472,22 +472,28 @@ The ternary operator is borrowed from Python: ### ysh-attr -The `.` operator performs attribute lookup. +The `.` operator looks up values on either `Dict` or `Obj` instances. -On `Dict` instances, the expression `mydict.key` is short for `mydict['key']` -(like JavaScript, but unlike Python.) +On dicts, it looks for the value associated with a key. That is, the +expression `mydict.key` is short for `mydict['key']` (like JavaScript, but +unlike Python.) -On `Obj` instances, the expression `obj.attr` does two things, in order: +--- + +On objects, the expression `obj.x` looks for attributes, with a special rule +for bound methods. The rules are: -1. Searches in the object's properties for a field named `attr`. - - If it exists, return the value literally. -2. Searches up the prototype chain for `attr` - - If it exists, return a **bound method**, which is an (object, function) - pair. +1. Search the properties of `obj` for a field named `x`. + - If it exists, return the value literally. (It can be of any type: `Func`, `Int`, + `Str`, ...) +2. Search up the prototype chain for a field named `x`. + - If it exists, and is **not** a `Func`, return the value literally. + - If it **is** a `Func`, return **bound method**, which is an (object, + function) pair. Later, when the bound method is called, the object is passed as the first -argument to the function, making it a method call. The method can then use the -object's properties. +argument to the function (`self`), making it a method call. This is how a +method has access to the object's properties. Example of first rule: @@ -495,7 +501,7 @@ Example of first rule: return (i + 1) } var module = Object(null, {Free}) - var x = module.Free(42) # => 43 + echo $[module.Free(42)] # => 43 Example of second rule: @@ -503,8 +509,8 @@ Example of second rule: return (self.n + i) } var methods = Object(null, {method}) - var obj = Object(methods, {n: 1}) - var x = obj.method(42) # => 43 + var obj = Object(methods, {n: 10}) + echo $[obj.method(42)] # => 52 ### ysh-slice diff --git a/doc/ysh-tour.md b/doc/ysh-tour.md index 2bd0694dea..2a658f2c10 100644 --- a/doc/ysh-tour.md +++ b/doc/ysh-tour.md @@ -772,13 +772,14 @@ These are like the "standard library" for the expression language. Examples: -### Data Types: `Int`, `Str`, `List`, `Dict`, ... +### Data Types: `Int`, `Str`, `List`, `Dict`, `Obj`, ... YSH has data types, each with an expression syntax and associated methods. ### Methods -Mutating methods are looked up with a thin arrow `->`: +YSH adds mutable data structures to shell, so we have a special syntax for +mutating methods. They are looked up with a thin arrow `->`: var foods = ['ale', 'bean'] var last = foods->pop() # bean @@ -788,13 +789,17 @@ You can ignore the return value with the `call` keyword: call foods->pop() -Transforming methods use a fat arrow `=>`: +Regular methods are looked up with the `.` operator: var line = ' ale bean ' + var caps = last.trim().upper() # 'ALE BEAN' + +You can also use the "chaining" style, with a fat arrow `=>`: + var trimmed = line => trim() => upper() # 'ALE BEAN' -If the `=>` operator doesn't find a method with the given name in the object's -type, it looks for free functions: +The `=>` operator lets you mix methods and free functions. If it doesn't find +a method with the given name, it looks for a `Func`: # list() is a free function taking one arg # join() is a free function taking two args From 66a80ffc810084bbb2c2917228c4764305fd8786 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 23 Aug 2024 11:01:59 -0400 Subject: [PATCH 175/506] [release] Bump version to 0.23.0 --- INSTALL-old.txt | 8 ++++---- INSTALL.txt | 10 +++++----- doc/osh.1 | 2 +- doc/release-index.md | 4 ++-- doc/release-quality.md | 2 +- oil-version.txt | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/INSTALL-old.txt b/INSTALL-old.txt index 1bd0ab941d..a1e6b37ecc 100644 --- a/INSTALL-old.txt +++ b/INSTALL-old.txt @@ -15,8 +15,8 @@ Quick Start If you haven't already done so, extract the tarball: - tar -x --xz < oil-0.22.0.tar.xz - cd oil-0.22.0 + tar -x --xz < oil-0.23.0.tar.xz + cd oil-0.23.0 Either install as /usr/local/bin/osh: @@ -37,7 +37,7 @@ The latter doesn't require root access, but it requires: (See manpath or $MANPATH.) NOTE: Out-of-tree builds are NOT currently supported, so you have to be in the -oil-0.22.0 directory. +oil-0.23.0 directory. Smoke Test ---------- @@ -56,7 +56,7 @@ More Documentation Every release has a home page with links, e.g. - https://oilshell.org/release/0.22.0/ + https://oilshell.org/release/0.23.0/ System Requirements ------------------- diff --git a/INSTALL.txt b/INSTALL.txt index 5721caa173..b5b44779f8 100644 --- a/INSTALL.txt +++ b/INSTALL.txt @@ -9,8 +9,8 @@ Quick Start If you haven't already done so, extract the tarball: - tar -x --gz < oil-for-unix-0.22.0.tar.gz - cd oils-for-unix-0.22.0 + tar -x --gz < oil-for-unix-0.23.0.tar.gz + cd oils-for-unix-0.23.0 This is the traditional way to install it: @@ -54,7 +54,7 @@ More Documentation Every release has a home page with links: - https://oilshell.org/release/0.22.0/ + https://oilshell.org/release/0.23.0/ System Requirements ------------------- @@ -105,7 +105,7 @@ This doesn't require root access, but it requires: $MANPATH.) NOTE: Out-of-tree builds are NOT currently supported, so you have to be in the -oils-for-unix-0.22.0 directory. +oils-for-unix-0.23.0 directory. Build Options ------------- @@ -125,5 +125,5 @@ Links ----- - Notes on portability: - https://oilshell.org/release/0.22.0/doc/portability.html + https://oilshell.org/release/0.23.0/doc/portability.html diff --git a/doc/osh.1 b/doc/osh.1 index 4d5cc59d70..8f7545b9ac 100644 --- a/doc/osh.1 +++ b/doc/osh.1 @@ -72,7 +72,7 @@ The referenced command or script could not be found. .Xr busybox 1 , .Xr sh 1 .Pp -.Lk http://www.oilshell.org/release/0.22.0/doc/ Docs +.Lk http://www.oilshell.org/release/0.23.0/doc/ Docs .Sh AUTHORS The .Nm diff --git a/doc/release-index.md b/doc/release-index.md index 73c24523a2..64494bfbc3 100644 --- a/doc/release-index.md +++ b/doc/release-index.md @@ -4,7 +4,7 @@ all_docs_url: - version_url: - --- -Oils 0.22.0 +Oils 0.23.0 =========== @@ -13,7 +13,7 @@ Oils 0.22.0 -This is the home page for version 0.22.0 of Oils, a Unix shell. To use it, +This is the home page for version 0.23.0 of Oils, a Unix shell. To use it, 1. Download a source tarball. 2. Build it and do a "smoke test", as described in [INSTALL][]. diff --git a/doc/release-quality.md b/doc/release-quality.md index 6f9ced99ed..c58754b41d 100644 --- a/doc/release-quality.md +++ b/doc/release-quality.md @@ -4,7 +4,7 @@ all_docs_url: - version_url: - --- -Oils 0.22.0 Quality +Oils 0.23.0 Quality =================== diff --git a/oil-version.txt b/oil-version.txt index 744add410e..38d1803cfa 100644 --- a/oil-version.txt +++ b/oil-version.txt @@ -1,4 +1,4 @@ -0.22.0 +0.23.0 # The first line of this file is the Oil version, and the rest is ignored. # It's used at build time for the release tarball, and at runtime for oil From d8b2d0d837998600706da101d36a1d56c3312f46 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 23 Aug 2024 20:13:04 -0400 Subject: [PATCH 176/506] [mycpp build] Remove dead code, rebuild prebuilt/ninja/mycpp.mycpp_main - Remove dead code using MYCPP_VENV, MYPY_REPO - we now use wedges from build/dev-shell.sh - Rebuild prebuilt/ninja from build/dynamic-deps.sh write-mycpp - this could be moved - Remove obsolete shell scripts --- build/dynamic-deps.sh | 24 +- build/ninja-rules-py.sh | 38 ++- devtools/types.sh | 5 +- mycpp/common-vars.sh | 10 - mycpp/common.sh | 31 -- mycpp/run.sh | 4 +- prebuilt/ninja/mycpp.mycpp_main/all-pairs.txt | 321 +++++++++--------- prebuilt/ninja/mycpp.mycpp_main/deps.txt | 4 +- prebuilt/translate.sh | 1 - 9 files changed, 212 insertions(+), 226 deletions(-) delete mode 100644 mycpp/common-vars.sh delete mode 100644 mycpp/common.sh diff --git a/build/dynamic-deps.sh b/build/dynamic-deps.sh index ef9b387f6f..c6db803ae5 100755 --- a/build/dynamic-deps.sh +++ b/build/dynamic-deps.sh @@ -11,8 +11,6 @@ set -o errexit REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) -source mycpp/common.sh # $MYPY_REPO - readonly PY_PATH='.:vendor/' # Temporary @@ -173,17 +171,29 @@ write-mycpp() { local dir=prebuilt/ninja/$module mkdir -p $dir - ( source $MYCPP_VENV/bin/activate - PYTHONPATH=$REPO_ROOT:$REPO_ROOT/mycpp:$MYPY_REPO maybe-our-python3 \ - build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt - ) + if false; then + ( source $MYCPP_VENV/bin/activate + PYTHONPATH=$REPO_ROOT:$REPO_ROOT/mycpp:$MYPY_REPO maybe-our-python3 \ + build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt + ) + fi + + # TODO: it would be nicer to put this at the top of the file, but we get + # READONLY errors. + source build/dev-shell.sh + python3 build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt + + local deps=$dir/deps.txt cat $dir/all-pairs.txt \ | grep -v oilshell/oil_DEPS \ | repo-filter \ | exclude-filter py-tool \ | mysort \ - | tee $dir/deps.txt + | tee $deps + + # EXTRA FILE + echo '_bin/datalog/dataflow' >> $deps echo echo $dir/* diff --git a/build/ninja-rules-py.sh b/build/ninja-rules-py.sh index 996d4ec0ea..77aa841a24 100755 --- a/build/ninja-rules-py.sh +++ b/build/ninja-rules-py.sh @@ -15,7 +15,7 @@ set -o errexit REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) source build/dev-shell.sh # python2 in $PATH -source mycpp/common-vars.sh # MYPY_REPO +#source devtools/types.sh # typecheck-files source $REPO_ROOT/test/tsv-lib.sh # time-tsv example-main-wrapper() { @@ -209,14 +209,14 @@ benchmark-table() { } > $out } -# TODO: No longer works. This is called by ninja mycpp-check -# I think it's giving strict warnings. -mypy() { - ( source $MYCPP_VENV/bin/activate - # Don't need this since the virtualenv we created with it? - # source build/dev-shell.sh - PYTHONPATH=$MYPY_REPO python3 -m mypy "$@"; - ) +# Copied from devtools/types.sh + +MYPY_FLAGS='--strict --no-strict-optional' +typecheck-files() { + echo "MYPY $@" + + # TODO: Adjust path for mcypp/examples/modules.py + time MYPYPATH='.:pyext' python3 -m mypy --py2 --follow-imports=silent $MYPY_FLAGS "$@" } typecheck() { @@ -231,9 +231,21 @@ typecheck() { local more_flags='' fi - # $more_flags can be empty - MYPYPATH="$REPO_ROOT:$REPO_ROOT/mycpp" \ - mypy --py2 --strict $more_flags $main_py > $out + # Similar to devtools/types.sh + + local status=0 + + set +o errexit + typecheck-files $main_py > $out + status=$? + set -o errexit + + if test $status != 0; then + echo "FAIL $main_py" + cat $out + fi + + return $status } logs-equal() { @@ -286,7 +298,7 @@ shift 2 tmp=$out.tmp # avoid creating partial files -PYTHONPATH="$REPO_ROOT:$MYPY_REPO" MYPYPATH="$MYPYPATH" \ +PYTHONPATH="$REPO_ROOT:$TODO_MYPY_REPO" MYPYPATH="$MYPYPATH" \ python3 pea/pea_main.py cpp "$@" > $tmp status=$? diff --git a/devtools/types.sh b/devtools/types.sh index efabfb8402..2ae327c8bc 100755 --- a/devtools/types.sh +++ b/devtools/types.sh @@ -63,4 +63,7 @@ soil-run() { check-all } -task-five "$@" +name=$(basename $0) +if test "$name" = 'types.sh'; then + task-five "$@" +fi diff --git a/mycpp/common-vars.sh b/mycpp/common-vars.sh deleted file mode 100644 index 3908ab87a2..0000000000 --- a/mycpp/common-vars.sh +++ /dev/null @@ -1,10 +0,0 @@ -# POSIX shell script sourced by _bin/shwrap/mycpp_main and mycpp/common.sh - -if test -z "${REPO_ROOT:-}"; then - echo '$REPO_ROOT should be set before sourcing' - exit 1 -fi - -readonly MYPY_REPO=$REPO_ROOT/../oil_DEPS/mypy -readonly MYCPP_VENV=$REPO_ROOT/../oil_DEPS/mycpp-venv - diff --git a/mycpp/common.sh b/mycpp/common.sh deleted file mode 100644 index 91320c012b..0000000000 --- a/mycpp/common.sh +++ /dev/null @@ -1,31 +0,0 @@ -# -# Common functions -# - -# Include guard. -test -n "${__MYCPP_COMMON_SH:-}" && return -readonly __MYCPP_COMMON_SH=1 - -if test -z "${REPO_ROOT:-}"; then - echo '$REPO_ROOT should be set before sourcing' - exit 1 -fi - -source mycpp/common-vars.sh - -maybe-our-python3() { - ### Run a command line with Python 3 - - # Use Python 3.10 from deps/from-tar if available. Otherwise use the system - # python3. - - local py3_ours='../oil_DEPS/python3' - if test -f $py3_ours; then - echo "*** Running $py3_ours $@" >& 2 - $py3_ours "$@" - else - # Use system copy - python3 "$@" - fi -} - diff --git a/mycpp/run.sh b/mycpp/run.sh index 49e38fcb06..7d104ab8ef 100755 --- a/mycpp/run.sh +++ b/mycpp/run.sh @@ -12,14 +12,14 @@ set -o errexit readonly THIS_DIR=$(dirname $(readlink -f $0)) readonly REPO_ROOT=$THIS_DIR/.. -source $THIS_DIR/common.sh # MYPY_REPO +source build/dev-shell.sh # # Utilities # gen-ctags() { - ctags -R $MYPY_REPO + ctags -R $TODO_MYPY_REPO } "$@" diff --git a/prebuilt/ninja/mycpp.mycpp_main/all-pairs.txt b/prebuilt/ninja/mycpp.mycpp_main/all-pairs.txt index b4e0abc3ae..8074c9a5f2 100644 --- a/prebuilt/ninja/mycpp.mycpp_main/all-pairs.txt +++ b/prebuilt/ninja/mycpp.mycpp_main/all-pairs.txt @@ -1,168 +1,171 @@ -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/_compression.py _compression.py -/home/andy/git/oilshell/oil_DEPS/py3/build/lib.linux-x86_64-3.10/_sysconfigdata__linux_x86_64-linux-gnu.py _sysconfigdata__linux_x86_64-linux-gnu.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/_weakrefset.py _weakrefset.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/argparse.py argparse.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/ast.py ast.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/bisect.py bisect.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/bz2.py bz2.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/collections/__init__.py collections/__init__.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/collections/abc.py collections/abc.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/configparser.py configparser.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/contextlib.py contextlib.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/copy.py copy.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/copyreg.py copyreg.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/curses/__init__.py curses/__init__.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/dataclasses.py dataclasses.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/difflib.py difflib.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/dis.py dis.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/distutils/__init__.py distutils/__init__.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/distutils/errors.py distutils/errors.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/distutils/sysconfig.py distutils/sysconfig.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/enum.py enum.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/fnmatch.py fnmatch.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/functools.py functools.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/gettext.py gettext.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/glob.py glob.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/hashlib.py hashlib.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/heapq.py heapq.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/importlib/__init__.py importlib/__init__.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/importlib/_bootstrap.py importlib/_bootstrap.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/importlib/_bootstrap_external.py importlib/_bootstrap_external.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/importlib/machinery.py importlib/machinery.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/inspect.py inspect.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/json/__init__.py json/__init__.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/json/decoder.py json/decoder.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/json/encoder.py json/encoder.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/json/scanner.py json/scanner.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/keyword.py keyword.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/linecache.py linecache.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/locale.py locale.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/lzma.py lzma.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/_compression.py _compression.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/_sysconfigdata__linux_x86_64-linux-gnu.py _sysconfigdata__linux_x86_64-linux-gnu.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/_weakrefset.py _weakrefset.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/argparse.py argparse.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/ast.py ast.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/bisect.py bisect.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/bz2.py bz2.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/collections/__init__.py collections/__init__.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/collections/abc.py collections/abc.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/configparser.py configparser.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/contextlib.py contextlib.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/copy.py copy.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/copyreg.py copyreg.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/curses/__init__.py curses/__init__.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/dataclasses.py dataclasses.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/difflib.py difflib.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/dis.py dis.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/distutils/__init__.py distutils/__init__.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/distutils/errors.py distutils/errors.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/distutils/sysconfig.py distutils/sysconfig.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/enum.py enum.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/fnmatch.py fnmatch.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/functools.py functools.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/gettext.py gettext.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/glob.py glob.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/hashlib.py hashlib.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/heapq.py heapq.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/importlib/__init__.py importlib/__init__.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/importlib/_bootstrap.py importlib/_bootstrap.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/importlib/_bootstrap_external.py importlib/_bootstrap_external.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/importlib/machinery.py importlib/machinery.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/inspect.py inspect.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/json/__init__.py json/__init__.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/json/decoder.py json/decoder.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/json/encoder.py json/encoder.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/json/scanner.py json/scanner.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/keyword.py keyword.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/linecache.py linecache.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/locale.py locale.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/lzma.py lzma.py /home/andy/git/oilshell/oil/mycpp/__init__.py mycpp/__init__.py /home/andy/git/oilshell/oil/mycpp/const_pass.py mycpp/const_pass.py +/home/andy/git/oilshell/oil/mycpp/control_flow_pass.py mycpp/control_flow_pass.py /home/andy/git/oilshell/oil/mycpp/cppgen_pass.py mycpp/cppgen_pass.py /home/andy/git/oilshell/oil/mycpp/crash.py mycpp/crash.py /home/andy/git/oilshell/oil/mycpp/debug_pass.py mycpp/debug_pass.py /home/andy/git/oilshell/oil/mycpp/format_strings.py mycpp/format_strings.py +/home/andy/git/oilshell/oil/mycpp/ir_pass.py mycpp/ir_pass.py /home/andy/git/oilshell/oil/mycpp/mycpp_main.py mycpp/mycpp_main.py /home/andy/git/oilshell/oil/mycpp/pass_state.py mycpp/pass_state.py /home/andy/git/oilshell/oil/mycpp/util.py mycpp/util.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/__init__.py mypy/__init__.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/applytype.py mypy/applytype.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/argmap.py mypy/argmap.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/binder.py mypy/binder.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/bogus_type.py mypy/bogus_type.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/build.py mypy/build.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/checker.py mypy/checker.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/checkexpr.py mypy/checkexpr.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/checkmember.py mypy/checkmember.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/checkstrformat.py mypy/checkstrformat.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/config_parser.py mypy/config_parser.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/constraints.py mypy/constraints.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/defaults.py mypy/defaults.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/erasetype.py mypy/erasetype.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/errorcodes.py mypy/errorcodes.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/errors.py mypy/errors.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/expandtype.py mypy/expandtype.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/exprtotype.py mypy/exprtotype.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/fastparse.py mypy/fastparse.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/find_sources.py mypy/find_sources.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/fixup.py mypy/fixup.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/freetree.py mypy/freetree.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/fscache.py mypy/fscache.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/git.py mypy/git.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/indirection.py mypy/indirection.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/infer.py mypy/infer.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/join.py mypy/join.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/literals.py mypy/literals.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/lookup.py mypy/lookup.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/main.py mypy/main.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/maptype.py mypy/maptype.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/meet.py mypy/meet.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/message_registry.py mypy/message_registry.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/messages.py mypy/messages.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/metastore.py mypy/metastore.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/mixedtraverser.py mypy/mixedtraverser.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/modulefinder.py mypy/modulefinder.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/moduleinfo.py mypy/moduleinfo.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/mro.py mypy/mro.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/nodes.py mypy/nodes.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/options.py mypy/options.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/parse.py mypy/parse.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/plugin.py mypy/plugin.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/plugins/__init__.py mypy/plugins/__init__.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/plugins/common.py mypy/plugins/common.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/plugins/default.py mypy/plugins/default.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/reachability.py mypy/reachability.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/renaming.py mypy/renaming.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/sametypes.py mypy/sametypes.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/scope.py mypy/scope.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal.py mypy/semanal.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_classprop.py mypy/semanal_classprop.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_enum.py mypy/semanal_enum.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_infer.py mypy/semanal_infer.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_main.py mypy/semanal_main.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_namedtuple.py mypy/semanal_namedtuple.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_newtype.py mypy/semanal_newtype.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_pass1.py mypy/semanal_pass1.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_shared.py mypy/semanal_shared.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_typeargs.py mypy/semanal_typeargs.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/semanal_typeddict.py mypy/semanal_typeddict.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/server/__init__.py mypy/server/__init__.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/server/aststrip.py mypy/server/aststrip.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/server/trigger.py mypy/server/trigger.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/sharedparse.py mypy/sharedparse.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/sitepkgs.py mypy/sitepkgs.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/solve.py mypy/solve.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/split_namespace.py mypy/split_namespace.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/state.py mypy/state.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/stats.py mypy/stats.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/strconv.py mypy/strconv.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/subtypes.py mypy/subtypes.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/traverser.py mypy/traverser.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/treetransform.py mypy/treetransform.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/tvar_scope.py mypy/tvar_scope.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/type_visitor.py mypy/type_visitor.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/typeanal.py mypy/typeanal.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/typeops.py mypy/typeops.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/types.py mypy/types.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/typestate.py mypy/typestate.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/typetraverser.py mypy/typetraverser.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/typevars.py mypy/typevars.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/util.py mypy/util.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/version.py mypy/version.py -/home/andy/git/oilshell/oil_DEPS/mypy/mypy/visitor.py mypy/visitor.py -/home/andy/.local/lib/python3.10/site-packages/mypy_extensions.py mypy_extensions.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/ntpath.py ntpath.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/opcode.py opcode.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/operator.py operator.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/optparse.py optparse.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/pathlib.py pathlib.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/pipes.py pipes.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/pprint.py pprint.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/random.py random.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/re.py re.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/reprlib.py reprlib.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/selectors.py selectors.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/shlex.py shlex.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/shutil.py shutil.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/signal.py signal.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/sre_compile.py sre_compile.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/sre_constants.py sre_constants.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/sre_parse.py sre_parse.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/subprocess.py subprocess.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/sysconfig.py sysconfig.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/tempfile.py tempfile.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/textwrap.py textwrap.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/threading.py threading.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/token.py token.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/tokenize.py tokenize.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/traceback.py traceback.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/types.py types.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/typing.py typing.py -/home/andy/.local/lib/python3.10/site-packages/typing_extensions.py typing_extensions.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/urllib/__init__.py urllib/__init__.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/urllib/parse.py urllib/parse.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/warnings.py warnings.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/weakref.py weakref.py -/home/andy/git/oilshell/oil/_cache/Python-3.10.4/Lib/__future__.py __future__.py +/home/andy/git/oilshell/oil/mycpp/visitor.py mycpp/visitor.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/__init__.py mypy/__init__.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/applytype.py mypy/applytype.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/argmap.py mypy/argmap.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/binder.py mypy/binder.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/bogus_type.py mypy/bogus_type.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/build.py mypy/build.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/checker.py mypy/checker.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/checkexpr.py mypy/checkexpr.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/checkmember.py mypy/checkmember.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/checkstrformat.py mypy/checkstrformat.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/config_parser.py mypy/config_parser.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/constraints.py mypy/constraints.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/defaults.py mypy/defaults.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/erasetype.py mypy/erasetype.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/errorcodes.py mypy/errorcodes.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/errors.py mypy/errors.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/expandtype.py mypy/expandtype.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/exprtotype.py mypy/exprtotype.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/fastparse.py mypy/fastparse.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/find_sources.py mypy/find_sources.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/fixup.py mypy/fixup.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/freetree.py mypy/freetree.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/fscache.py mypy/fscache.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/git.py mypy/git.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/indirection.py mypy/indirection.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/infer.py mypy/infer.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/join.py mypy/join.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/literals.py mypy/literals.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/lookup.py mypy/lookup.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/main.py mypy/main.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/maptype.py mypy/maptype.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/meet.py mypy/meet.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/message_registry.py mypy/message_registry.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/messages.py mypy/messages.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/metastore.py mypy/metastore.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/mixedtraverser.py mypy/mixedtraverser.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/modulefinder.py mypy/modulefinder.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/moduleinfo.py mypy/moduleinfo.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/mro.py mypy/mro.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/nodes.py mypy/nodes.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/options.py mypy/options.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/parse.py mypy/parse.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/plugin.py mypy/plugin.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/plugins/__init__.py mypy/plugins/__init__.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/plugins/common.py mypy/plugins/common.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/plugins/default.py mypy/plugins/default.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/reachability.py mypy/reachability.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/renaming.py mypy/renaming.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/sametypes.py mypy/sametypes.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/scope.py mypy/scope.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal.py mypy/semanal.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_classprop.py mypy/semanal_classprop.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_enum.py mypy/semanal_enum.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_infer.py mypy/semanal_infer.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_main.py mypy/semanal_main.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_namedtuple.py mypy/semanal_namedtuple.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_newtype.py mypy/semanal_newtype.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_pass1.py mypy/semanal_pass1.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_shared.py mypy/semanal_shared.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_typeargs.py mypy/semanal_typeargs.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/semanal_typeddict.py mypy/semanal_typeddict.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/server/__init__.py mypy/server/__init__.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/server/aststrip.py mypy/server/aststrip.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/server/trigger.py mypy/server/trigger.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/sharedparse.py mypy/sharedparse.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/sitepkgs.py mypy/sitepkgs.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/solve.py mypy/solve.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/split_namespace.py mypy/split_namespace.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/state.py mypy/state.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/stats.py mypy/stats.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/strconv.py mypy/strconv.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/subtypes.py mypy/subtypes.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/traverser.py mypy/traverser.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/treetransform.py mypy/treetransform.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/tvar_scope.py mypy/tvar_scope.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/type_visitor.py mypy/type_visitor.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/typeanal.py mypy/typeanal.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/typeops.py mypy/typeops.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/types.py mypy/types.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/typestate.py mypy/typestate.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/typetraverser.py mypy/typetraverser.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/typevars.py mypy/typevars.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/util.py mypy/util.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/version.py mypy/version.py +/home/andy/wedge/oils-for-unix.org/pkg/mypy/0.780/mypy/visitor.py mypy/visitor.py +/home/andy/wedge/oils-for-unix.org/pkg/py3-libs/2023-03-04/lib/python3.10/site-packages/mypy_extensions.py mypy_extensions.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/ntpath.py ntpath.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/opcode.py opcode.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/operator.py operator.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/optparse.py optparse.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/pathlib.py pathlib.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/pipes.py pipes.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/pprint.py pprint.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/random.py random.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/re.py re.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/reprlib.py reprlib.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/selectors.py selectors.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/shlex.py shlex.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/shutil.py shutil.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/signal.py signal.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/sre_compile.py sre_compile.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/sre_constants.py sre_constants.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/sre_parse.py sre_parse.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/subprocess.py subprocess.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/sysconfig.py sysconfig.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/tempfile.py tempfile.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/textwrap.py textwrap.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/threading.py threading.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/token.py token.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/tokenize.py tokenize.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/traceback.py traceback.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/types.py types.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/typing.py typing.py +/home/andy/wedge/oils-for-unix.org/pkg/py3-libs/2023-03-04/lib/python3.10/site-packages/typing_extensions.py typing_extensions.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/urllib/__init__.py urllib/__init__.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/urllib/parse.py urllib/parse.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/warnings.py warnings.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/weakref.py weakref.py +/wedge/oils-for-unix.org/pkg/python3/3.10.4/lib/python3.10/__future__.py __future__.py diff --git a/prebuilt/ninja/mycpp.mycpp_main/deps.txt b/prebuilt/ninja/mycpp.mycpp_main/deps.txt index eda182b56b..345937e0cc 100644 --- a/prebuilt/ninja/mycpp.mycpp_main/deps.txt +++ b/prebuilt/ninja/mycpp.mycpp_main/deps.txt @@ -1,10 +1,10 @@ -mycpp/ir_pass.py mycpp/const_pass.py +mycpp/control_flow_pass.py mycpp/cppgen_pass.py mycpp/crash.py mycpp/debug_pass.py -mycpp/control_flow_pass.py mycpp/format_strings.py +mycpp/ir_pass.py mycpp/mycpp_main.py mycpp/pass_state.py mycpp/util.py diff --git a/prebuilt/translate.sh b/prebuilt/translate.sh index 033ab21b7c..fca50831df 100755 --- a/prebuilt/translate.sh +++ b/prebuilt/translate.sh @@ -11,7 +11,6 @@ source $LIB_OSH/task-five.sh REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) -source mycpp/common.sh # MYPY_REPO source build/ninja-rules-cpp.sh readonly TEMP_DIR=_build/tmp From 1620bc29ee3a104b1d8c4f4fd9caf8c614b06258 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 23 Aug 2024 20:38:49 -0400 Subject: [PATCH 177/506] [build] Fix tarball creation --- build/ninja_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build/ninja_main.py b/build/ninja_main.py index 7a5447c0a8..aa70911940 100755 --- a/build/ninja_main.py +++ b/build/ninja_main.py @@ -58,7 +58,6 @@ def TarballManifest(cc_h_files): 'build/py2.sh', 'build/dev-shell.sh', 'build/ninja-rules-cpp.sh', - 'mycpp/common.sh', # Generated '_build/oils.sh', From 637c54e7b71c81826aaac77cb686ec33543a6bf1 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 23 Aug 2024 22:41:02 -0400 Subject: [PATCH 178/506] [build/deps] Mirror Python tarballs A release machine couldn't connect to python.org/ftp for some reason. --- build/deps.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/build/deps.sh b/build/deps.sh index bc7e563bd0..d6b1a938a0 100755 --- a/build/deps.sh +++ b/build/deps.sh @@ -464,6 +464,17 @@ mirror-pyflakes() { oilshell.org:oilshell.org/blob/ } +mirror-python() { + ### Can't reach python.org from some machines + scp \ + $DEPS_SOURCE_DIR/python2/"$(basename $PY2_URL)" \ + oilshell.org:oilshell.org/blob/ + + scp \ + $DEPS_SOURCE_DIR/python3/"$(basename $PY3_URL)" \ + oilshell.org:oilshell.org/blob/ +} + wedge-exists() { ### Does an installed wedge already exist? From 5afb4ed6af92b6fb71922e47f0c170dc360ce72e Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Fri, 23 Aug 2024 23:21:36 -0400 Subject: [PATCH 179/506] [build/deps] Add bloaty to 'extra wedges' Still getting this error though. $ metrics/native-code.sh oils-for-unix ninja: no work to do. bloaty: Data is in new DWARF format we don't understand --- build/deps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/deps.sh b/build/deps.sh index d6b1a938a0..5822f985b2 100755 --- a/build/deps.sh +++ b/build/deps.sh @@ -673,6 +673,7 @@ extra-wedges() { # Test both outside the contianer, as well as inside? echo uftrace $UFTRACE_VERSION $ROOT_WEDGE_DIR + echo bloaty $BLOATY_VERSION $ROOT_WEDGE_DIR #echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR } From d15799117838405b0ee65784cfa9d870b63dadc9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 23 Aug 2024 23:49:33 -0400 Subject: [PATCH 180/506] [release] Build tarball binaries with DWARF version 4, for bloaty --- devtools/release.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/devtools/release.sh b/devtools/release.sh index bd1d601540..bdd5e438a0 100755 --- a/devtools/release.sh +++ b/devtools/release.sh @@ -350,8 +350,13 @@ _install() { _build-oils-benchmark-data() { pushd $BENCHMARK_DATA_OILS - _build/oils.sh '' opt SKIP_REBUILD - _build/oils.sh '' dbg SKIP_REBUILD # for metrics/native-code.sh + for variant in dbg opt; do + # DWARF version 4 is a hack for bloaty, which doesn't support version 5. + # I don't think this should affect benchmarks besides + # metrics/native-code.sh, so we don't bother building a separate binary. + # The Soil CI runs without this flag. + CXXFLAGS=-gdwarf-4 _build/oils.sh '' $variant SKIP_REBUILD + done popd } From 97551aade1cc1b0680048dec794ae3b2750c19b6 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 23 Aug 2024 23:51:22 -0400 Subject: [PATCH 181/506] [release] Update benchmark machine names --- benchmarks/common.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmarks/common.sh b/benchmarks/common.sh index e60ed0e7ba..7f0b369c7f 100644 --- a/benchmarks/common.sh +++ b/benchmarks/common.sh @@ -10,10 +10,14 @@ readonly __BENCHMARKS_COMMON_SH=1 #readonly MACHINE1=flanders #readonly MACHINE2=lenny -# 2023-11-29: machine1 is still lenny because it has bloaty, which doesn't -# work with ELF data emitted by newer GCC on Debian 12 -readonly MACHINE1=lenny -readonly MACHINE2=hoover +# 2023-11-29: MACHINE1=lenny MACHINE2=hoover + +# 2024-08-23: MACHINE1=hoover MACHINE2=mercer +# Because we gained a Souffle dependency, which requires C++17. And the base +# image on lenny doesn't support C++17. + +readonly MACHINE1=hoover +readonly MACHINE2=mercer OIL_VERSION=$(head -n 1 oil-version.txt) From 7d4c6f5fd44aed3b043d10cd6221666684c27c9d Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Sat, 24 Aug 2024 00:25:10 -0400 Subject: [PATCH 182/506] [release] Run configure before building tarballs for benchmarks --- devtools/release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/devtools/release.sh b/devtools/release.sh index bdd5e438a0..f630c46097 100755 --- a/devtools/release.sh +++ b/devtools/release.sh @@ -350,6 +350,7 @@ _install() { _build-oils-benchmark-data() { pushd $BENCHMARK_DATA_OILS + ./configure for variant in dbg opt; do # DWARF version 4 is a hack for bloaty, which doesn't support version 5. # I don't think this should affect benchmarks besides From 0b86264c7f60b1546033482b850862eef495ad8a Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 24 Aug 2024 00:37:12 -0400 Subject: [PATCH 183/506] [release] Remove bloaty reports for old tarball Also add build/oil-defs, caused by math.isnan() change, etc. Somehow it didn't get minimized? --- .../Modules/mathmodule.c/math_methods.def | 45 +++++++++++++++++++ doc/release-quality.md | 3 -- metrics/native-code.sh | 11 +++-- 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 build/oil-defs/Python-2.7.13/Modules/mathmodule.c/math_methods.def diff --git a/build/oil-defs/Python-2.7.13/Modules/mathmodule.c/math_methods.def b/build/oil-defs/Python-2.7.13/Modules/mathmodule.c/math_methods.def new file mode 100644 index 0000000000..9eca1cf864 --- /dev/null +++ b/build/oil-defs/Python-2.7.13/Modules/mathmodule.c/math_methods.def @@ -0,0 +1,45 @@ +// Python-2.7.13/Modules/mathmodule.c + +static PyMethodDef math_methods[] = { + {"acos", math_acos, METH_O}, + {"acosh", math_acosh, METH_O}, + {"asin", math_asin, METH_O}, + {"asinh", math_asinh, METH_O}, + {"atan", math_atan, METH_O}, + {"atan2", math_atan2, METH_VARARGS}, + {"atanh", math_atanh, METH_O}, + {"ceil", math_ceil, METH_O}, + {"copysign", math_copysign, METH_VARARGS}, + {"cos", math_cos, METH_O}, + {"cosh", math_cosh, METH_O}, + {"degrees", math_degrees, METH_O}, + {"erf", math_erf, METH_O}, + {"erfc", math_erfc, METH_O}, + {"exp", math_exp, METH_O}, + {"expm1", math_expm1, METH_O}, + {"fabs", math_fabs, METH_O}, + {"factorial", math_factorial, METH_O}, + {"floor", math_floor, METH_O}, + {"fmod", math_fmod, METH_VARARGS}, + {"frexp", math_frexp, METH_O}, + {"fsum", math_fsum, METH_O}, + {"gamma", math_gamma, METH_O}, + {"hypot", math_hypot, METH_VARARGS}, + {"isinf", math_isinf, METH_O}, + {"isnan", math_isnan, METH_O}, + {"ldexp", math_ldexp, METH_VARARGS}, + {"lgamma", math_lgamma, METH_O}, + {"log", math_log, METH_VARARGS}, + {"log1p", math_log1p, METH_O}, + {"log10", math_log10, METH_O}, + {"modf", math_modf, METH_O}, + {"pow", math_pow, METH_VARARGS}, + {"radians", math_radians, METH_O}, + {"sin", math_sin, METH_O}, + {"sinh", math_sinh, METH_O}, + {"sqrt", math_sqrt, METH_O}, + {"tan", math_tan, METH_O}, + {"tanh", math_tanh, METH_O}, + {"trunc", math_trunc, METH_O}, + {0}, +}; diff --git a/doc/release-quality.md b/doc/release-quality.md index c58754b41d..360d8c5e80 100644 --- a/doc/release-quality.md +++ b/doc/release-quality.md @@ -163,8 +163,5 @@ the "experimental" version of Oils. - [src-bin-ratio-with-opy](pub/metrics.wwz/bytecode/src-bin-ratio-with-opy.txt) - How big is the compiled output? - OVM / CPython - - [overview](pub/metrics.wwz/ovm/overview.txt) - An analysis of GCC's - compilation of [OVM][] (a subset of CPython). [Bloaty][] provides the - underlying data. - [cpython-defs/overview](pub/metrics.wwz/cpython-defs/overview.txt) - We try to ship as little of CPython as possible, and this is what's left. diff --git a/metrics/native-code.sh b/metrics/native-code.sh index 539a5c3ba4..b74cc829c4 100755 --- a/metrics/native-code.sh +++ b/metrics/native-code.sh @@ -137,12 +137,15 @@ compare-gcc-clang() { readonly OIL_VERSION=$(head -n 1 oil-version.txt) run-for-release() { - build-ovm + # 2024-08: Not building with DWARF 4 + if false; then + build-ovm - local dbg=_build/oil/ovm-dbg - local opt=_build/oil/ovm-opt + local dbg=_build/oil/ovm-dbg + local opt=_build/oil/ovm-opt - collect-and-report $OVM_BASE_DIR $dbg $opt + collect-and-report $OVM_BASE_DIR $dbg $opt + fi # TODO: consolidate with benchmarks/common.sh, OSH_CPP_BENCHMARK_DATA # For some reason _bin/cxx-opt/ and _bin/cxx-opt-sh can differ by a few bytes From d2350173d4337969a38a94568704837d6d035e47 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 24 Aug 2024 00:44:03 -0400 Subject: [PATCH 184/506] [release] Repeat the DWARF 4 hack when building out of benchmarks-data --- devtools/release-native.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/devtools/release-native.sh b/devtools/release-native.sh index 3ccc189296..376f66d63f 100755 --- a/devtools/release-native.sh +++ b/devtools/release-native.sh @@ -92,8 +92,11 @@ extract-for-benchmarks() { rm -v _bin/cxx-{dbg,opt}-sh/* || true ./configure - _build/oils.sh '' dbg - _build/oils.sh '' opt + + # devtools/release.sh also has this DWARF 4 hack, for bloaty + for variant in dbg opt; do + CXXFLAGS=-gdwarf-4 _build/oils.sh '' $variant + done build/native.sh tarball-demo From 10f4e8c4e6c272d33d136fbc1b98bc0baf8344b8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 24 Aug 2024 00:48:41 -0400 Subject: [PATCH 185/506] [build/cpython-defs] Update for mathmodule.c --- build/cpython_defs.py | 5 +++ .../Modules/mathmodule.c/math_methods.def | 38 ------------------- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/build/cpython_defs.py b/build/cpython_defs.py index ad86257436..f1e450cff7 100755 --- a/build/cpython_defs.py +++ b/build/cpython_defs.py @@ -282,6 +282,7 @@ def out(msg, *args): 'signalmodule.c', 'timemodule.c', 'termios.c', + 'mathmodule.c', ] @@ -372,6 +373,10 @@ def __call__(self, rel_path, def_name, method_name): if basename == 'signalmodule.c' and method_name == 'default_int_handler': return True + # Name collisions + if basename == 'mathmodule.c' and method_name in ('exp', 'log'): + return False + # segfault without this if basename == 'typeobject.c' and method_name == '__new__': return True diff --git a/build/oil-defs/Python-2.7.13/Modules/mathmodule.c/math_methods.def b/build/oil-defs/Python-2.7.13/Modules/mathmodule.c/math_methods.def index 9eca1cf864..4f5a08fea3 100644 --- a/build/oil-defs/Python-2.7.13/Modules/mathmodule.c/math_methods.def +++ b/build/oil-defs/Python-2.7.13/Modules/mathmodule.c/math_methods.def @@ -1,45 +1,7 @@ // Python-2.7.13/Modules/mathmodule.c static PyMethodDef math_methods[] = { - {"acos", math_acos, METH_O}, - {"acosh", math_acosh, METH_O}, - {"asin", math_asin, METH_O}, - {"asinh", math_asinh, METH_O}, - {"atan", math_atan, METH_O}, - {"atan2", math_atan2, METH_VARARGS}, - {"atanh", math_atanh, METH_O}, - {"ceil", math_ceil, METH_O}, - {"copysign", math_copysign, METH_VARARGS}, - {"cos", math_cos, METH_O}, - {"cosh", math_cosh, METH_O}, - {"degrees", math_degrees, METH_O}, - {"erf", math_erf, METH_O}, - {"erfc", math_erfc, METH_O}, - {"exp", math_exp, METH_O}, - {"expm1", math_expm1, METH_O}, - {"fabs", math_fabs, METH_O}, - {"factorial", math_factorial, METH_O}, - {"floor", math_floor, METH_O}, - {"fmod", math_fmod, METH_VARARGS}, - {"frexp", math_frexp, METH_O}, - {"fsum", math_fsum, METH_O}, - {"gamma", math_gamma, METH_O}, - {"hypot", math_hypot, METH_VARARGS}, {"isinf", math_isinf, METH_O}, {"isnan", math_isnan, METH_O}, - {"ldexp", math_ldexp, METH_VARARGS}, - {"lgamma", math_lgamma, METH_O}, - {"log", math_log, METH_VARARGS}, - {"log1p", math_log1p, METH_O}, - {"log10", math_log10, METH_O}, - {"modf", math_modf, METH_O}, - {"pow", math_pow, METH_VARARGS}, - {"radians", math_radians, METH_O}, - {"sin", math_sin, METH_O}, - {"sinh", math_sinh, METH_O}, - {"sqrt", math_sqrt, METH_O}, - {"tan", math_tan, METH_O}, - {"tanh", math_tanh, METH_O}, - {"trunc", math_trunc, METH_O}, {0}, }; From e9f9f425b696eed81c438a5cb9b8bff63cdf628d Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 24 Aug 2024 10:49:01 -0400 Subject: [PATCH 186/506] [test/spec] Work around failures across machines [spec/xtrace] Work around dash bug [spec/builtin-printf] Work around mksh bug [spec/vars-special] Work around bug with lack of Python 2. [spec/case_] This one is weird - I think there is a libc bug only on Debian Buster Slim? It doesn't have to do with LC_ALL. I can't reproduce the bug locally, on my Debian machine, or on an Ubuntu 22 machine. --- spec/builtin-printf.test.sh | 15 ++++++++--- spec/case_.test.sh | 53 ++++++++++++++++++++++++++++++++++--- spec/var-op-len.test.sh | 1 + spec/vars-special.test.sh | 13 ++++++++- spec/xtrace.test.sh | 8 ++++++ 5 files changed, 82 insertions(+), 8 deletions(-) diff --git a/spec/builtin-printf.test.sh b/spec/builtin-printf.test.sh index da34555866..59cd3b0ca9 100644 --- a/spec/builtin-printf.test.sh +++ b/spec/builtin-printf.test.sh @@ -413,7 +413,7 @@ printf '%d\n' \" ## END #### Unicode char with ' -#env +case $SH in mksh) echo 'weird bug'; exit ;; esac # the mu character is U+03BC @@ -480,7 +480,7 @@ echo 47011 ## END -## BUG dash/ash/mksh STDOUT: +## BUG dash/ash STDOUT: ce 206 316 @@ -491,6 +491,10 @@ e4 ## END +## BUG mksh STDOUT: +weird bug +## END + #### Invalid UTF-8 echo bytes1 @@ -549,6 +553,7 @@ e0 #### Too large +case $SH in mksh) echo 'weird bug'; exit ;; esac echo too large too_large=$(python2 -c 'print("\xF4\x91\x84\x91")') @@ -565,7 +570,7 @@ too large ## END -## BUG dash/ash/mksh STDOUT: +## BUG dash/ash STDOUT: too large f4 244 @@ -573,6 +578,10 @@ f4 ## END +## BUG mksh STDOUT: +weird bug +## END + # osh rejects code points that are too large for a DIFFERENT reason ## OK osh STDOUT: diff --git a/spec/case_.test.sh b/spec/case_.test.sh index 62ac7318ee..1114b0ddfd 100644 --- a/spec/case_.test.sh +++ b/spec/case_.test.sh @@ -1,6 +1,5 @@ - ## compare_shells: bash dash mksh zsh -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 # Note: zsh passes most of these tests too @@ -145,9 +144,15 @@ no no ## END -#### case with single byte LC_ALL=C +#### matching the byte 0xff against empty string - DISABLED - CI only bug? + +case $SH in *osh) echo soil-ci-buster-slim-bug; exit ;; esac + +# This doesn't make a difference on my local machine? +# Is the underlying issue how libc fnmatch() respects Unicode? -LC_ALL=C +#LC_ALL=C +#LC_ALL=C.UTF-8 c=$(printf \\377) @@ -159,8 +164,48 @@ case $c in "$c") echo b ;; esac +case "$c" in + '') echo a ;; + "$c") echo b ;; +esac + ## STDOUT: b +b +## END + +## OK osh STDOUT: +soil-ci-buster-slim-bug +## END + +#### matching every byte against itself + +# Why does OSH on the CI machine behave differently? Probably a libc bug fix +# I'd guess? + +sum=0 + +# note: NUL byte crashes OSH! +for i in $(seq 1 255); do + hex=$(printf '%x' "$i") + c="$(printf "\\x$hex")" # command sub quirk: \n or \x0a turns into empty string + + #echo -n $c | od -A n -t x1 + #echo ${#c} + + case "$c" in + # Newline matches empty string somehow. All shells agree. I guess + # fnmatch() ignores trailing newline? + #'') echo "[empty i=$i hex=$hex c=$c]" ;; + "$c") sum=$(( sum + 1 )) ;; + *) echo "[bug i=$i hex=$hex c=$c]" ;; + esac +done + +echo sum=$sum + +## STDOUT: +sum=255 ## END #### \(\) in pattern (regression) diff --git a/spec/var-op-len.test.sh b/spec/var-op-len.test.sh index 49229b738b..8478d43f46 100644 --- a/spec/var-op-len.test.sh +++ b/spec/var-op-len.test.sh @@ -217,3 +217,4 @@ echo ${#x-default} 0 3 ## END + diff --git a/spec/vars-special.test.sh b/spec/vars-special.test.sh index a1d98508f6..2f0b76a9ec 100644 --- a/spec/vars-special.test.sh +++ b/spec/vars-special.test.sh @@ -26,13 +26,24 @@ env | grep PWD #### $PATH is set if unset at startup +# WORKAROUND for Python version of bin/osh -- we can't run bin/oils_for_unix.py +# because it a shebang #!/usr/bin/env python2 +# This test is still useful for the C++ oils-for-unix. + +case $SH in + */bin/osh) + echo yes + echo yes + exit + ;; +esac + # Get absolute path before changing PATH sh=$(which $SH) old_path=$PATH unset PATH -# BUG: when sh=bin/osh, we can't run bin/oils_for_unix.py $sh -c 'echo $PATH' > path.txt PATH=$old_path diff --git a/spec/xtrace.test.sh b/spec/xtrace.test.sh index c1cebf7c0e..cc06966b40 100644 --- a/spec/xtrace.test.sh +++ b/spec/xtrace.test.sh @@ -4,6 +4,8 @@ ## compare_shells: bash dash mksh #### unset PS4 +case $SH in dash) echo 'weird bug'; exit ;; esac + set -x echo 1 unset PS4 @@ -17,6 +19,12 @@ echo 2 echo 2 ## END +## BUG dash STDOUT: +weird bug +## END +## BUG dash STDERR: +## END + #### set -o verbose prints unevaluated code set -o verbose x=foo From f13860fc36748d40b1985f78668cd54eba7e4d3b Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 24 Aug 2024 21:09:06 -0400 Subject: [PATCH 187/506] [benchmarks/ovm-build] Rewrite to work in Soil, as well as release - New style, based on benchmarks/osh-runtime and shell-provenance-2 - added compiler-provenance-2 - remove OVM measurements - bytecode size, etc. - The report now works on a single machine Next: - Test with Clang on the second release machine. - Get this running in CI - Test with release process - 'measure-builds' --- benchmarks/auto.sh | 2 +- benchmarks/id.sh | 69 +++++++++++-- benchmarks/osh-runtime.sh | 7 +- benchmarks/ovm-build.sh | 207 +++++++++++++++++++++----------------- benchmarks/report.R | 18 +--- soil/worker.sh | 2 + 6 files changed, 181 insertions(+), 124 deletions(-) diff --git a/benchmarks/auto.sh b/benchmarks/auto.sh index 36dca7a0b6..5f8c9f7167 100755 --- a/benchmarks/auto.sh +++ b/benchmarks/auto.sh @@ -64,7 +64,7 @@ measure-builds() { # TODO: Use new provenance style, like measure-shells local build_prov - build_prov=$(benchmarks/id.sh compiler-provenance) # capture the filename + build_prov=$(benchmarks/id.sh compiler-provenance $job_id) # capture the filename benchmarks/ovm-build.sh measure $build_prov $out_dir/ovm-build } diff --git a/benchmarks/id.sh b/benchmarks/id.sh index 5c9cab5202..d978ea8a57 100755 --- a/benchmarks/id.sh +++ b/benchmarks/id.sh @@ -73,6 +73,8 @@ _dump-if-exists() { # dump-shell-id() { + ### Write files that identify the shell + local sh_path=$1 local out_dir=$2 @@ -199,6 +201,8 @@ publish-shell-id() { # How to calculate the hash though? dump-host-id() { + ### Write files that identify the host + local out_dir=${1:-_tmp/host-id/$(hostname)} mkdir -p $out_dir @@ -287,6 +291,8 @@ publish-host-id() { # dump-compiler-id() { + ### Write files that identify the compiler + local cc=$1 # path to the compiler local out_dir=${2:-_tmp/compiler-id/$(basename $cc)} @@ -344,7 +350,7 @@ publish-compiler-id() { # is recorded. shell-provenance-2() { - ### Write to _tmp/provenance.{txt,tsv} and $out_dir/{shell,host-id} + ### Write to _tmp/provenance.{txt,tsv} and $out_dir/{shell-id,host-id} local maybe_host=$1 # if it exists, it overrides the host local job_id=$2 @@ -353,8 +359,6 @@ shell-provenance-2() { # log "*** shell-provenance" - mkdir -p _tmp/provenance - local host_name if test -n "$maybe_host"; then # label is often 'no-host' host_name=$maybe_host @@ -362,17 +366,18 @@ shell-provenance-2() { host_name=$(hostname) fi - log "*** $maybe_host $host_name $job_id $out_dir" + log "*** shell-provenance-2 $maybe_host $host_name $job_id $out_dir" local tmp_dir=_tmp/prov-tmp/$host_name dump-host-id $tmp_dir local host_hash host_hash=$(publish-host-id $tmp_dir "$out_dir/host-id") + local shell_hash local out_txt=_tmp/provenance.txt # Legacy text file - echo -n '' > $out_txt # trunacte, no header + echo -n '' > $out_txt # truncated, no header local out_tsv=_tmp/provenance.tsv tsv-row job_id host_name host_hash sh_path shell_hash > $out_tsv @@ -400,15 +405,63 @@ shell-provenance-2() { log "Wrote $out_txt and $out_tsv" } +compiler-provenance-2() { + # Write to _tmp/compiler-provenance.txt and $out_dir/{compiler-id,host-id} + + local maybe_host=$1 # if it exists, it overrides the host + local job_id=$2 + local out_dir=$3 + + local host_name + if test -n "$maybe_host"; then # label is often 'no-host' + host_name=$maybe_host + else + host_name=$(hostname) + fi + + log "*** compiler-provenance-2 $maybe_host $host_name $job_id $out_dir" + + local tmp_dir=_tmp/prov-tmp/$host_name + dump-host-id $tmp_dir + + local host_hash + host_hash=$(publish-host-id $tmp_dir "$out_dir/host-id") + + local compiler_hash + + local out_txt=_tmp/compiler-provenance.txt # Legacy text file + echo -n '' > $out_txt # truncated, no header + + local out_tsv=_tmp/compiler-provenance.tsv + tsv-row job_id host_name host_hash compiler_path compiler_hash > $out_tsv + + for compiler_path in $(which gcc) $CLANG; do + local name=$(basename $compiler_path) + + tmp_dir=_tmp/prov-tmp/$name + dump-compiler-id $compiler_path $tmp_dir + + compiler_hash=$(publish-compiler-id $tmp_dir "$out_dir/compiler-id") + + echo "$job_id $host_name $host_hash $compiler_path $compiler_hash" \ + >> $out_txt + + tsv-row \ + "$job_id" "$host_name" "$host_hash" "$compiler_path" "$compiler_hash" \ + >> $out_tsv + done + + log "Wrote $out_txt and $out_tsv" +} + compiler-provenance() { - local job_id - job_id=$(print-job-id) + local job_id=$1 local host host=$(hostname) # Filename - local out=_tmp/provenance/${host}.${job_id}.compiler-provenance.txt + local out=_tmp/${host}.${job_id}.compiler-provenance.txt local tmp_dir=_tmp/host-id/$host dump-host-id $tmp_dir diff --git a/benchmarks/osh-runtime.sh b/benchmarks/osh-runtime.sh index 4da6167c45..9af40678c1 100755 --- a/benchmarks/osh-runtime.sh +++ b/benchmarks/osh-runtime.sh @@ -310,7 +310,7 @@ run-tasks-wrapper() { measure() { ### For release and CI local host_name=$1 # 'no-host' or 'lenny' - local raw_out_dir=$2 # _tmp/osh-runtime or ../../benchmark-data/osh-runtime + local raw_out_dir=$2 # _tmp/osh-runtime/$X or ../../benchmark-data/osh-runtime/$X local osh_native=$3 # $OSH_CPP_NINJA_BUILD or $OSH_CPP_BENCHMARK_DATA print-tasks "$host_name" "$osh_native" \ @@ -324,13 +324,12 @@ stage1() { local out_dir=$BASE_DIR/stage1 # _tmp/osh-runtime mkdir -p $out_dir - # Globs are in lexicographical order, which works for our dates. - local -a raw_times=() local -a raw_gc_stats=() local -a raw_provenance=() if test -n "$single_machine"; then + # find dir in _tmp/osh-runtime local -a a=( $base_dir/raw.$single_machine.* ) raw_times+=( ${a[-1]}/times.tsv ) @@ -338,6 +337,8 @@ stage1() { raw_provenance+=( ${a[-1]}/provenance.tsv ) else + # find last dirs in ../benchmark-data/osh-runtime + # Globs are in lexicographical order, which works for our dates. local -a a=( $base_dir/raw.$MACHINE1.* ) local -a b=( $base_dir/raw.$MACHINE2.* ) diff --git a/benchmarks/ovm-build.sh b/benchmarks/ovm-build.sh index a69113ea29..cb01aa66f5 100755 --- a/benchmarks/ovm-build.sh +++ b/benchmarks/ovm-build.sh @@ -4,7 +4,7 @@ # different machines, and measure the binary size. # # Usage: -# ./ovm-build.sh +# benchmarks/ovm-build.sh # # Run on its own: # 1. Follow common instructions in benchmarks/osh-parser.sh @@ -34,13 +34,13 @@ set -o nounset set -o pipefail set -o errexit -REPO_ROOT=$(cd $(dirname $0)/..; pwd) -readonly REPO_ROOT - -source test/tsv-lib.sh # uses REPO_ROOT source benchmarks/common.sh # for log, etc. +source benchmarks/id.sh # print-job-id source build/common.sh # for $CLANG +REPO_ROOT=$(cd $(dirname $0)/..; pwd) +source test/tsv-lib.sh # uses REPO_ROOT + readonly BASE_DIR=_tmp/ovm-build readonly TAR_DIR=$PWD/_deps/ovm-build # Make it absolute @@ -48,9 +48,10 @@ readonly TAR_DIR=$PWD/_deps/ovm-build # Make it absolute # Dependencies # -# Leave out mksh for now, because it doesn't follow ./configure make. It just -# has Build.sh. -readonly -a TAR_SUBDIRS=( bash-4.4 dash-0.5.9.1 ) # mksh ) +readonly -a TAR_SUBDIRS=( + dash-0.5.9.1 + bash-4.4 +) # NOTE: Same list in oilshell.org/blob/run.sh. tarballs() { @@ -75,14 +76,9 @@ extract-other() { } # Done automatically by 'measure' function. -# -# NOTE: We assume that _release/oil.tar exists. It should be made by -# scripts/release.sh build-and-test or benchmark-build. -extract-oil() { - # This is different than the others tarballs. - rm -r -f -v $TAR_DIR/oil-* - tar -x --directory $TAR_DIR --file _release/oil.tar +# TODO: CI should download this from previous +extract-oils() { # To run on multiple machines, use the one in the benchmarks-data repo. cp --recursive --no-target-directory \ ../benchmark-data/src/oils-for-unix-$OIL_VERSION/ \ @@ -108,7 +104,7 @@ sizes-tsv() { # NOTE: This should be the same on all x64 machines. But I want to run it on # x64 machines. measure-sizes() { - local prefix=${1:-$BASE_DIR/raw/demo} + local raw_out_dir=$1 # PROBLEM: Do I need provenance for gcc/clang here? I can just join it later # in R. @@ -118,18 +114,13 @@ measure-sizes() { # gcc/oils-for-unix # gcc/oils-for-unix.stripped sizes-tsv $BASE_DIR/bin/*/{oils-for-unix,oils-for-unix.stripped} \ - > ${prefix}.native-sizes.tsv - - sizes-tsv $TAR_DIR/oil-$OIL_VERSION/_build/oil/bytecode-opy.zip \ - > ${prefix}.bytecode-size.tsv - - sizes-tsv $BASE_DIR/bin/*/oil.* \ - > ${prefix}.bin-sizes.tsv + > ${raw_out_dir}/native-sizes.tsv + # Not used - we're not stripping these, etc. sizes-tsv $BASE_DIR/bin/*/*sh \ - > ${prefix}.other-shell-sizes.tsv + > ${raw_out_dir}/other-shell-sizes.tsv - log "Wrote ${prefix}.*.tsv" + log "Wrote ${raw_out_dir}/*.tsv" } # @@ -165,7 +156,7 @@ clang-oil-dbg() { # It would be possible, but it complicates the makefile. build-task() { - local out_dir=$1 + local raw_out_dir=$1 local job_id=$2 local host=$3 local host_hash=$4 @@ -174,7 +165,7 @@ build-task() { local src_dir=$7 local action=$8 - local times_out="$PWD/$out_dir/$host.$job_id.times.tsv" + local times_out="$PWD/$raw_out_dir/times.tsv" # Definitions that depends on $PWD. local -a TIME_PREFIX=( @@ -274,21 +265,13 @@ build-task() { log "DONE BUILD TASK $action $src_dir __ status=$?" } -oil-tasks() { +oils-tasks() { local provenance=$1 - # NOTE: it MUST be a tarball and not the git repo, because we don't build - # bytecode-*.zip! We care about the "packager's experience". - local oil_dir="$TAR_DIR/oil-$OIL_VERSION" local ofu_dir="$TAR_DIR/oils-for-unix-$OIL_VERSION" # Add 1 field for each of 5 fields. cat $provenance | while read line; do - # NOTE: configure is independent of compiler. - echo "$line" $oil_dir configure - echo "$line" $oil_dir _bin/oil.ovm - echo "$line" $oil_dir _bin/oil.ovm-dbg - echo "$line" $ofu_dir oils-for-unix echo "$line" $ofu_dir oils-for-unix.stripped done @@ -297,10 +280,6 @@ oil-tasks() { other-shell-tasks() { local provenance=$1 - # NOTE: it MUST be a tarball and not the git repo, because we do the build - # of bytecode.zip! We care about the "package experience". - local tarball='_release/oil.0.5.alpha1.gz' - # Add 1 field for each of 5 fields. cat $provenance | while read line; do case $line in @@ -326,19 +305,32 @@ oil-historical-tasks() { # action is 'configure', a target name, etc. readonly NUM_COLUMNS=7 # 5 from provenence, then tarball/target -measure() { - local provenance=$1 # from benchmarks/id.sh compiler-provenance - local out_dir=${2:-$BASE_DIR/raw} +print-tasks() { + local build_prov=$1 - extract-oil + local t1=$BASE_DIR/oils-tasks.txt + local t2=$BASE_DIR/other-shell-tasks.txt + + oils-tasks $build_prov > $t1 + other-shell-tasks $build_prov > $t2 + + if test -n "${QUICKLY:-}"; then + head -n 2 $t1 # debug and opt binary + head -n 2 $t2 # do dash configure make + else + cat $t1 $t2 + fi +} - # Job ID is everything up to the first dot in the filename. - local name=$(basename $provenance) - local prefix=${name%.compiler-provenance.txt} # strip suffix +measure() { + local build_prov=$1 # from benchmarks/id.sh compiler-provenance + local raw_out_dir=$2 # _tmp/ovm-build/$X or ../../benchmark-data/ovm-build/$X - local times_out="$out_dir/$prefix.times.tsv" + extract-oils + + local times_out="$raw_out_dir/times.tsv" # NOTE: Do we need two raw dirs? - mkdir -p $BASE_DIR/{raw,stage1,bin} $out_dir + mkdir -p $BASE_DIR/{stage1,bin} $raw_out_dir # TODO: the $times_out calculation is duplicated in build-task() @@ -348,16 +340,13 @@ measure() { host_name host_hash compiler_path compiler_hash \ src_dir action > $times_out - local t1=$BASE_DIR/oil-tasks.txt - local t2=$BASE_DIR/other-shell-tasks.txt - - oil-tasks $provenance > $t1 - other-shell-tasks $provenance > $t2 + # TODO: remove xargs + # - print-tasks | run-tasks with a loop + # - exit code is more reliable, and we're not running in parallel anyway - #grep dash $t2 | - #time cat $t1 | set +o errexit - time cat $t1 $t2 | xargs --verbose -n $NUM_COLUMNS -- $0 build-task $out_dir + time print-tasks $build_prov \ + | xargs --verbose -n $NUM_COLUMNS -- $0 build-task $raw_out_dir local status=$? set -o errexit @@ -365,9 +354,7 @@ measure() { die "*** Some tasks failed. (xargs status=$status) ***" fi - measure-sizes $out_dir/$prefix - - cp -v $provenance $out_dir + measure-sizes $raw_out_dir } # @@ -375,35 +362,36 @@ measure() { # stage1() { - local raw_dir=${1:-$BASE_DIR/raw} + local base_dir=${1:-$BASE_DIR} # _tmp/ovm-build or ../benchmark-data/ovm-build + local single_machine=${2:-} + + local out_dir=$BASE_DIR/stage1 + mkdir -p $out_dir - local out=$BASE_DIR/stage1 - mkdir -p $out + local -a raw_times=() + local -a raw_sizes=() - local x - local -a a b + if test -n "$single_machine"; then + # find dir in _tmp/ovm-build + local -a a=( $base_dir/raw.$single_machine.* ) - # Globs are in lexicographical order, which works for our dates. - x=$out/times.tsv - a=($raw_dir/$MACHINE1.*.times.tsv) - b=($raw_dir/$MACHINE2.*.times.tsv) - tsv-concat ${a[-1]} ${b[-1]} > $x + raw_times+=( ${a[-1]}/times.tsv ) + raw_sizes+=( ${a[-1]}/native-sizes.tsv ) - x=$out/bytecode-size.tsv - a=($raw_dir/$MACHINE1.*.bytecode-size.tsv) - b=($raw_dir/$MACHINE2.*.bytecode-size.tsv) - tsv-concat ${a[-1]} ${b[-1]} > $x + else + # find last dirs in ../benchmark-data/ovm-build + # Globs are in lexicographical order, which works for our dates. + local -a a=( $base_dir/raw.$MACHINE1.* ) + local -a b=( $base_dir/raw.$MACHINE2.* ) - x=$out/bin-sizes.tsv - a=($raw_dir/$MACHINE1.*.bin-sizes.tsv) - b=($raw_dir/$MACHINE2.*.bin-sizes.tsv) - tsv-concat ${a[-1]} ${b[-1]} > $x + raw_times+=( ${a[-1]}/times.tsv ${b[-1]}/times.tsv ) + raw_sizes+=( ${a[-1]}/native-sizes.tsv ${b[-1]}/native-sizes.tsv ) + fi + + tsv-concat "${raw_times[@]}" > $out_dir/times.tsv + tsv-concat "${raw_sizes[@]}" > $out_dir/native-sizes.tsv - x=$out/native-sizes.tsv - a=($raw_dir/$MACHINE1.*.native-sizes.tsv) - b=($raw_dir/$MACHINE2.*.native-sizes.tsv) - #tsv-concat ${b[-1]} > $x - tsv-concat ${a[-1]} ${b[-1]} > $x + return # NOTE: unused # Construct a one-column TSV file @@ -451,18 +439,6 @@ EOF EOF tsv2html --css-class-pattern 'special ^gcc' $in_dir/native-sizes.tsv - cmark << 'EOF' -### OVM Binary Size - -The oil binary has two portions: - -- Architecture-independent `bytecode.zip` -- Architecture- and compiler- dependent native code (`_build/oil/ovm*`) - -EOF - # Highlight the "default" production build - tsv2html --css-class-pattern 'special /gcc/oil.ovm$' $in_dir/sizes.tsv - cmark << 'EOF' ### Host and Compiler Details @@ -476,4 +452,45 @@ EOF EOF } +soil-run() { + rm -r -f $BASE_DIR + mkdir -p $BASE_DIR + + download + extract-other + + # Copied from benchmarks/osh-runtime.sh soil-run + + # could add _bin/cxx-bumpleak/oils-for-unix, although sometimes it's slower + local -a osh_bin=( $OSH_CPP_NINJA_BUILD ) + ninja "${osh_bin[@]}" + + local single_machine='no-host' + + local single_machine='no-host' + + local job_id + job_id=$(print-job-id) + + # Like benchmarks/auto.sh + #local build_prov + #build_prov=$(benchmarks/id.sh compiler-provenance $job_id) + + compiler-provenance-2 \ + $single_machine $job_id _tmp + + local host_job_id="$single_machine.$job_id" + local raw_out_dir="$BASE_DIR/raw.$host_job_id" + mkdir -p $raw_out_dir $BASE_DIR/stage1 + + measure _tmp/compiler-provenance.txt $raw_out_dir + + # Trivial concatenation for 1 machine + stage1 '' $single_machine + + benchmarks/report.sh stage2 $BASE_DIR + + benchmarks/report.sh stage3 $BASE_DIR +} + "$@" diff --git a/benchmarks/report.R b/benchmarks/report.R index 9add34ad9a..938249afca 100755 --- a/benchmarks/report.R +++ b/benchmarks/report.R @@ -674,10 +674,8 @@ WriteOvmBuildDetails = function(distinct_hosts, distinct_compilers, out_dir) { OvmBuildReport = function(in_dir, out_dir) { times = readTsv(file.path(in_dir, 'times.tsv')) - bytecode_size = readTsv(file.path(in_dir, 'bytecode-size.tsv')) - bin_sizes = readTsv(file.path(in_dir, 'bin-sizes.tsv')) native_sizes = readTsv(file.path(in_dir, 'native-sizes.tsv')) - raw_data = readTsv(file.path(in_dir, 'raw-data.tsv')) + #raw_data = readTsv(file.path(in_dir, 'raw-data.tsv')) times %>% filter(status != 0) -> failed if (nrow(failed) != 0) { @@ -716,18 +714,6 @@ OvmBuildReport = function(in_dir, out_dir) { #print(times) - bytecode_size %>% - rename(bytecode_size = num_bytes) %>% - select(-c(path)) -> - bytecode_size - - bin_sizes %>% - # reorder - select(c(host_label, path, num_bytes)) %>% - left_join(bytecode_size, by = c('host_label')) %>% - mutate(native_code_size = num_bytes - bytecode_size) -> - sizes - # paths look like _tmp/ovm-build/bin/clang/oils_cpp.stripped native_sizes %>% select(c(host_label, path, num_bytes)) %>% @@ -742,8 +728,6 @@ OvmBuildReport = function(in_dir, out_dir) { # NOTE: These don't have the host and compiler. writeTsv(times, file.path(out_dir, 'times')) - writeTsv(bytecode_size, file.path(out_dir, 'bytecode-size')) - writeTsv(sizes, file.path(out_dir, 'sizes')) writeTsv(native_sizes, file.path(out_dir, 'native-sizes')) # TODO: I want a size report too diff --git a/soil/worker.sh b/soil/worker.sh index 8f0c7121b8..bd4a4dd871 100755 --- a/soil/worker.sh +++ b/soil/worker.sh @@ -305,6 +305,8 @@ EOF cpp-coverage-tasks() { # dep notes: hnode_asdl.h required by expr_asdl.h in mycpp/examples + # TODO: make this work +#tar-compile benchmarks/ovm-build.sh soil-run - cat < Date: Sun, 25 Aug 2024 01:04:06 -0400 Subject: [PATCH 188/506] [build] Add -stdlib compiler flag for Clang For some reason this is needed on the Ubuntu 22 release machine. It hasn't been needed in the past. --- build/common.sh | 5 ++++- build/ninja-rules-cpp.sh | 21 ++++++++++++++++----- deps/from-binary.sh | 19 +++++++++++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/build/common.sh b/build/common.sh index 4d5ff59510..d54c439b7a 100644 --- a/build/common.sh +++ b/build/common.sh @@ -14,8 +14,11 @@ set -o nounset set -o errexit #eval 'set -o pipefail' +#LLVM_VERSION=18.1.8 +LLVM_VERSION=14.0.0 + # New version is slightly slower -- 13 seconds vs. 11.6 seconds on oils-for-unix -readonly CLANG_DIR_RELATIVE='../oil_DEPS/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04' +readonly CLANG_DIR_RELATIVE="../oil_DEPS/clang+llvm-$LLVM_VERSION-x86_64-linux-gnu-ubuntu-18.04" CLANG_DIR_1=$REPO_ROOT/$CLANG_DIR_RELATIVE CLANG_DIR_FALLBACK=~/git/oilshell/oil/$CLANG_DIR_RELATIVE diff --git a/build/ninja-rules-cpp.sh b/build/ninja-rules-cpp.sh index 51ea77f5be..ee25369886 100755 --- a/build/ninja-rules-cpp.sh +++ b/build/ninja-rules-cpp.sh @@ -255,20 +255,27 @@ compile_one() { setglobal_compile_flags "$variant" "$more_cxx_flags" "$dotd" case $out in - (_build/preprocessed/*) + _build/preprocessed/*) flags="$flags -E" ;; # DISABLE spew for mycpp-generated code. mycpp/pea could flag this at the # PYTHON level, rather than doing it at the C++ level. - (_build/obj/*/_gen/bin/oils_for_unix.mycpp.o) + _build/obj/*/_gen/bin/oils_for_unix.mycpp.o) flags="$flags -Wno-unused-variable -Wno-unused-but-set-variable" ;; esac - # TODO: exactly when is -fPIC needed? Clang needs it sometimes? - if test $compiler = 'clang' && test $variant != 'opt'; then - flags="$flags -fPIC" + if test "$compiler" = 'clang'; then + # 2024-08 - Clang needs -stdlib=libc++ for some reason + # https://stackoverflow.com/questions/26333823/clang-doesnt-see-basic-headers + # https://stackoverflow.com/questions/19774778/when-is-it-necessary-to-use-the-flag-stdlib-libstdc + flags="$flags -stdlib=libc++" + + # TODO: exactly when is -fPIC needed? Clang needs it sometimes? + if test $variant != 'opt'; then + flags="$flags -fPIC" + fi fi # this flag is only valid in Clang, doesn't work in continuous build @@ -305,6 +312,10 @@ link() { setglobal_cxx $compiler + if test "$compiler" = 'clang'; then + link_flags="$link_flags -stdlib=libc++" + fi + local prefix='' if test -n "${TIME_TSV_OUT:-}"; then prefix="benchmarks/time_.py --tsv --out $TIME_TSV_OUT --append --rusage --field link --field $out --" diff --git a/deps/from-binary.sh b/deps/from-binary.sh index 2103a4c9c4..c3a8e99b3b 100755 --- a/deps/from-binary.sh +++ b/deps/from-binary.sh @@ -19,13 +19,24 @@ source build/common.sh readonly DEPS_DIR=$REPO_ROOT/../oil_DEPS +# TODO: Make Clang into a wedge? + +if false; then + # This version if 7.6 GB, ugh + LLVM_VERSION=18.1.8 + CLANG_URL='https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04.tar.xz' +else + # This version was 4.7 GB + LLVM_VERSION=14.0.0 + CLANG_URL='https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz' +fi + download-clang() { # download into $DEPS_DIR and not _cache because Dockerfile.clang stores the # compressed version - wget --no-clobber --directory _cache \ - https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz + wget --no-clobber --directory _cache $CLANG_URL } extract-clang() { @@ -34,7 +45,7 @@ extract-clang() { # TODO: retire ../oil_DEPS dir in favor of wedge mkdir -p $DEPS_DIR pushd $DEPS_DIR - time tar -x --xz < ../oil/_cache/clang+llvm-14.0.0*.tar.xz + time tar -x --xz < ../oil/_cache/clang+llvm-$LLVM_VERSION*.tar.xz popd } @@ -42,7 +53,7 @@ extract-clang-in-container() { ### For Dockerfile.clang pushd $DEPS_DIR - time tar -x --xz < clang+llvm-14.0.0*.tar.xz + time tar -x --xz < clang+llvm-$LLVM_VERSION*.tar.xz popd } From fce1aa0f644dd1bb11d8d468b466fad97e2c60f1 Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Sun, 25 Aug 2024 01:36:33 -0400 Subject: [PATCH 189/506] [build] Special case for clang-coverage: don't pass -stdlib This is annoying --- build/ninja-rules-cpp.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build/ninja-rules-cpp.sh b/build/ninja-rules-cpp.sh index ee25369886..2275cbf957 100755 --- a/build/ninja-rules-cpp.sh +++ b/build/ninja-rules-cpp.sh @@ -270,7 +270,14 @@ compile_one() { # 2024-08 - Clang needs -stdlib=libc++ for some reason # https://stackoverflow.com/questions/26333823/clang-doesnt-see-basic-headers # https://stackoverflow.com/questions/19774778/when-is-it-necessary-to-use-the-flag-stdlib-libstdc - flags="$flags -stdlib=libc++" + + # But don't do it for clang-coverage binary, because the CI machine doesn't + # like it? + # It fails on the release machine - sigh + + if test $variant != 'coverage'; then + flags="$flags -stdlib=libc++" + fi # TODO: exactly when is -fPIC needed? Clang needs it sometimes? if test $variant != 'opt'; then From 60bfcfbe595418a106ef4ddd851a779cd8e08140 Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Sun, 25 Aug 2024 01:43:33 -0400 Subject: [PATCH 190/506] [build] Add special case for linking too --- build/ninja-rules-cpp.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/ninja-rules-cpp.sh b/build/ninja-rules-cpp.sh index 2275cbf957..ef2b1117a4 100755 --- a/build/ninja-rules-cpp.sh +++ b/build/ninja-rules-cpp.sh @@ -320,7 +320,9 @@ link() { setglobal_cxx $compiler if test "$compiler" = 'clang'; then - link_flags="$link_flags -stdlib=libc++" + if test $variant != 'coverage'; then + link_flags="$link_flags -stdlib=libc++" + fi fi local prefix='' From 99b1a7c7962ffa51aadcdbc304f0fd4acfe8daeb Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Sun, 25 Aug 2024 01:54:52 -0400 Subject: [PATCH 191/506] [build] Fix heuristic to include coverage+bumpleak --- build/ninja-rules-cpp.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/build/ninja-rules-cpp.sh b/build/ninja-rules-cpp.sh index ef2b1117a4..6dcb123716 100755 --- a/build/ninja-rules-cpp.sh +++ b/build/ninja-rules-cpp.sh @@ -271,13 +271,12 @@ compile_one() { # https://stackoverflow.com/questions/26333823/clang-doesnt-see-basic-headers # https://stackoverflow.com/questions/19774778/when-is-it-necessary-to-use-the-flag-stdlib-libstdc - # But don't do it for clang-coverage binary, because the CI machine doesn't - # like it? - # It fails on the release machine - sigh - - if test $variant != 'coverage'; then - flags="$flags -stdlib=libc++" - fi + # But don't do it for clang-coverage* builds, because the CI machine + # doesn't like it? This makes it fail on the release machine - sigh + case $variant in + coverage*) ;; # include coverage+bumpleak + *) flags="$flags -stdlib=libc++" ;; + esac # TODO: exactly when is -fPIC needed? Clang needs it sometimes? if test $variant != 'opt'; then @@ -320,9 +319,10 @@ link() { setglobal_cxx $compiler if test "$compiler" = 'clang'; then - if test $variant != 'coverage'; then - link_flags="$link_flags -stdlib=libc++" - fi + case $variant in + coverage*) ;; # include coverage+bumpleak + *) link_flags="$link_flags -stdlib=libc++" + esac fi local prefix='' From 30f7d6a33c655e9e32b97aabc9240ace42a11d8e Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Sun, 25 Aug 2024 11:15:19 -0400 Subject: [PATCH 192/506] [benchmarks/ovm-build] Port release automation to new style It runs and produces data in ../benchmark-data/ovm-build/raw.$MACHINE2 ... Though I still need to update the merging of data. --- benchmarks/auto.sh | 32 ++++++++++++++------------------ benchmarks/id.sh | 35 ----------------------------------- benchmarks/ovm-build.sh | 5 ----- 3 files changed, 14 insertions(+), 58 deletions(-) diff --git a/benchmarks/auto.sh b/benchmarks/auto.sh index 5f8c9f7167..644401b59b 100755 --- a/benchmarks/auto.sh +++ b/benchmarks/auto.sh @@ -35,9 +35,8 @@ measure-shells() { local host_job_id="$host_name.$job_id" local raw_out_dir - raw_out_dir="$out_dir/osh-runtime/raw.$host_job_id" - # New Style doesn't need provenance -- it's joined later + raw_out_dir="$out_dir/osh-runtime/raw.$host_job_id" benchmarks/osh-runtime.sh measure \ $host_name $raw_out_dir $OSH_CPP_BENCHMARK_DATA $out_dir @@ -57,23 +56,12 @@ measure-shells() { $provenance $host_job_id $out_dir/compute } -measure-builds() { - local host_name=$1 - local job_id=$2 - local out_dir=$3 - - # TODO: Use new provenance style, like measure-shells - local build_prov - build_prov=$(benchmarks/id.sh compiler-provenance $job_id) # capture the filename - - benchmarks/ovm-build.sh measure $build_prov $out_dir/ovm-build -} - # Run all benchmarks from a clean git checkout. # Before this, run devtools/release.sh benchmark-build. all() { local do_machine1=${1:-} + local resume1=${2:-} # skip past measure-shells local host_name host_name=$(hostname) # Running on multiple machines @@ -88,8 +76,6 @@ all() { $host_name $job_id $out_dir \ "${SHELLS[@]}" $OSH_CPP_BENCHMARK_DATA python2 - # TODO: probably move compiler-provenance here - # Notes: # - During release, this happens on machine1, but not machine2 if test -n "$do_machine1"; then @@ -103,8 +89,18 @@ all() { _tmp/provenance.txt $host_job_id $out_dir/osh-parser $OSH_CPP_BENCHMARK_DATA fi - measure-shells $host_name $job_id $out_dir - measure-builds $host_name $job_id $out_dir + if test -z "${resume1:-}"; then + measure-shells $host_name $job_id $out_dir + fi + + compiler-provenance-2 \ + $host_name $job_id $out_dir + + local raw_out_dir + raw_out_dir="$out_dir/ovm-build/raw.$host_job_id" + + local build_prov=_tmp/compiler-provenance.txt + benchmarks/ovm-build.sh measure $build_prov $raw_out_dir } "$@" diff --git a/benchmarks/id.sh b/benchmarks/id.sh index d978ea8a57..47897b248d 100755 --- a/benchmarks/id.sh +++ b/benchmarks/id.sh @@ -454,41 +454,6 @@ compiler-provenance-2() { log "Wrote $out_txt and $out_tsv" } -compiler-provenance() { - local job_id=$1 - - local host - host=$(hostname) - - # Filename - local out=_tmp/${host}.${job_id}.compiler-provenance.txt - - local tmp_dir=_tmp/host-id/$host - dump-host-id $tmp_dir - - local host_hash - host_hash=$(publish-host-id $tmp_dir) - - local compiler_hash - - # gcc is assumed to be in the $PATH. - for compiler_path in $(which gcc) $CLANG; do - local name=$(basename $compiler_path) - - tmp_dir=_tmp/compiler-id/$name - dump-compiler-id $compiler_path $tmp_dir - - compiler_hash=$(publish-compiler-id $tmp_dir) - - echo "$job_id $host $host_hash $compiler_path $compiler_hash" - done > $out - - log "Wrote $out" - - # Return value used in command sub - echo $out -} - out-param() { declare -n out=$1 diff --git a/benchmarks/ovm-build.sh b/benchmarks/ovm-build.sh index cb01aa66f5..8952742848 100755 --- a/benchmarks/ovm-build.sh +++ b/benchmarks/ovm-build.sh @@ -58,7 +58,6 @@ tarballs() { cat < Date: Sun, 25 Aug 2024 12:58:56 -0400 Subject: [PATCH 193/506] [release] Shell functions for 0.23.0 Also adjust hard-coded host name in report.R --- benchmarks/report.R | 6 +++--- devtools/release-note.sh | 2 +- devtools/release-version.sh | 9 +++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/benchmarks/report.R b/benchmarks/report.R index 938249afca..48a1c8559d 100755 --- a/benchmarks/report.R +++ b/benchmarks/report.R @@ -253,9 +253,9 @@ ParserReport = function(in_dir, out_dir) { spread(key = host_label, value = lines_per_ms) -> times_summary - # Sort by parsing rate on the fast machine - if ("host lenny" %in% colnames(times_summary)) { - times_summary %>% arrange(desc(`host lenny`)) -> times_summary + # Sort by parsing rate on machine 1 + if ("host hoover" %in% colnames(times_summary)) { + times_summary %>% arrange(desc(`host hoover`)) -> times_summary } else { times_summary %>% arrange(desc(`host no-host`)) -> times_summary } diff --git a/devtools/release-note.sh b/devtools/release-note.sh index 0cf0541ef1..d3e0eaffb9 100755 --- a/devtools/release-note.sh +++ b/devtools/release-note.sh @@ -15,7 +15,7 @@ source build/dev-shell.sh # PYTHONPATH source devtools/release-version.sh # for escape-segments readonly OILS_VERSION=$(head -n 1 oil-version.txt) -readonly PREV_VERSION='0.21.0' +readonly PREV_VERSION='0.22.0' # adapted from release-version.sh _git-changelog-body() { diff --git a/devtools/release-version.sh b/devtools/release-version.sh index 3c452f5acc..e8516714c6 100755 --- a/devtools/release-version.sh +++ b/devtools/release-version.sh @@ -616,6 +616,11 @@ git-changelog-0.22.0() { > _release/VERSION/changelog.html } +git-changelog-0.23.0() { + _git-changelog origin/release/0.22.0 release/0.23.0 \ + > _release/VERSION/changelog.html +} + # For announcement.html html-redirect() { local url=$1 @@ -1032,6 +1037,10 @@ announcement-0.22.0() { write-no-announcement } +announcement-0.23.0() { + write-no-announcement +} + blog-redirect() { html-redirect 'making-plans.html' > $SITE_DEPLOY_DIR/blog/2020/01/11.html } From be2e341657a160d3f6836d0df45bd9a78955ae32 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 25 Aug 2024 14:13:30 -0400 Subject: [PATCH 194/506] [soil] Switch to uuu.oilshell.org and use wwup HTTP uploader That wasn't too hard! We still have to replace SSH in these other tasks: - rewriting the jobs index - cleaning up old jobs - status-api - cleaning up old status entries [release] Add missing period --- doc/release-quality.md | 2 +- soil/common.sh | 8 +++++--- soil/web-init.sh | 2 +- soil/web-worker.sh | 35 ++++++++++++++++++++++++----------- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/doc/release-quality.md b/doc/release-quality.md index 360d8c5e80..60320d5dfa 100644 --- a/doc/release-quality.md +++ b/doc/release-quality.md @@ -66,7 +66,7 @@ This is a supplement to the [main release page](index.html). - [osh-usage](more-tests.wwz/suite-logs/osh-usage.txt). Misc tests of the `osh` binary. - [tools-deps](more-tests.wwz/suite-logs/tools-deps.txt). Tests for a subcommand in progress. -- [syscall](more-tests.wwz/syscall/-wwz-index) How many syscalls do we make, +- [syscall](more-tests.wwz/syscall/-wwz-index). How many syscalls do we make, and how many processes do we start? - [ysh-ify Tests](more-tests.wwz/suite-logs/ysh-ify.txt). Test OSH to YSH translation. diff --git a/soil/common.sh b/soil/common.sh index c85d39b4c8..872720e066 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -22,9 +22,9 @@ dump-env() { if true; then readonly SOIL_USER='travis_admin' - readonly SOIL_HOST='travis-ci.oilshell.org' - readonly SOIL_HOST_DIR=~/travis-ci.oilshell.org # used on server - readonly SOIL_REMOTE_DIR=travis-ci.oilshell.org # used on client + readonly SOIL_HOST='uuu.oilshell.org' + readonly SOIL_HOST_DIR=~/uuu.oilshell.org # used on server + readonly SOIL_REMOTE_DIR=uuu.oilshell.org # used on client elif false; then readonly SOIL_USER='oils' readonly SOIL_HOST='mb.oils.pub' @@ -40,6 +40,8 @@ fi readonly SOIL_USER_HOST="$SOIL_USER@$SOIL_HOST" +readonly WWUP_URL="https://$SOIL_HOST/wwup.cgi" + html-head() { # TODO: Shebang line should change too PYTHONPATH=. python3 doctools/html_head.py "$@" diff --git a/soil/web-init.sh b/soil/web-init.sh index 4238d35ba2..4be71ad4ef 100755 --- a/soil/web-init.sh +++ b/soil/web-init.sh @@ -26,7 +26,7 @@ source soil/common.sh # for SOIL_USER and SOIL_HOST home-page() { ### travis-ci.oilshell.org home page - local domain=${1:-'travis-ci.oilshell.org'} + local domain=${1:-$SOIL_HOST} local title="Soil on $domain" soil-html-head "$title" diff --git a/soil/web-worker.sh b/soil/web-worker.sh index d87ce265fa..f54118142e 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -48,7 +48,7 @@ sshq() { # # This is Bernstein chaining through ssh. - ssh $SOIL_USER@$SOIL_HOST "$(printf '%q ' "$@")" + my-ssh $SOIL_USER_HOST "$(printf '%q ' "$@")" } remote-rewrite-jobs-index() { @@ -252,9 +252,9 @@ test-collect-json() { deploy-job-results() { ### Copy .wwz, .tsv, and .json to a new dir - local prefix=$1 # e.g. example.com/github-jobs/ - local subdir=$2 # e.g. example.com/github-jobs/1234/ # make this dir - local job_name=$3 # e.g. example.com/github-jobs/1234/foo.wwz + local prefix=$1 # e.g. github- for example.com/github-jobs/ + local run_dir=$2 # e.g. 1234 # make this dir + local job_name=$3 # e.g. cpp-small for example.com/github-jobs/1234/cpp-small.wwz shift 2 # rest of args are more env vars @@ -273,17 +273,28 @@ deploy-job-results() { # So we don't have to unzip it cp _tmp/soil/INDEX.tsv $job_name.tsv - local remote_dest_dir="$SOIL_REMOTE_DIR/${prefix}jobs/$subdir" - my-ssh $SOIL_USER_HOST "mkdir -p $remote_dest_dir" - - # Do JSON last because that's what 'list-json' looks for - my-scp $job_name.{wwz,tsv,json} "$SOIL_USER_HOST:$remote_dest_dir" + if false; then + local remote_dest_dir="$SOIL_REMOTE_DIR/${prefix}jobs/$run_dir" + my-ssh $SOIL_USER_HOST "mkdir -p $remote_dest_dir" + + # Do JSON last because that's what 'list-json' looks for + my-scp $job_name.{wwz,tsv,json} "$SOIL_USER_HOST:$remote_dest_dir" + else + curl \ + --verbose \ + --form "payload-type=${prefix}jobs" \ + --form "subdir=$run_dir" \ + --form "file1=@${job_name}.wwz" \ + --form "file2=@${job_name}.tsv" \ + --form "file3=@${job_name}.json" \ + $WWUP_URL + fi log '' log 'View CI results here:' log '' - log "http://$SOIL_HOST/${prefix}jobs/$subdir/" - log "http://$SOIL_HOST/${prefix}jobs/$subdir/$job_name.wwz/" + log "https://$SOIL_HOST/${prefix}jobs/$run_dir/" + log "https://$SOIL_HOST/${prefix}jobs/$run_dir/$job_name.wwz/" log '' } @@ -343,6 +354,8 @@ remote-event-job-done() { log "remote-event-job-done" + #set -x + # Deployed code dir sshq soil-web/soil/web.sh event-job-done "$@" } From fb8f111761055ed96330a7443e4a4947ab91ca04 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 02:16:57 -0400 Subject: [PATCH 195/506] [soil] Replace most ssh/scp invocations with http to wwup.cgi The only remaining ssh call is for the cpp-tarball task. - Add fast-subset.yml to test the CI more quickly TODO: curl to https is broken on some CI clients. There is an SSL cert error. --- .github/workflows/fast-subset.yml_DISABLED | 131 +++++++++++++++++++++ soil/admin.sh | 9 ++ soil/common.sh | 2 +- soil/github-actions.sh | 3 + soil/web-worker.sh | 50 ++++++-- soil/web.sh | 41 ++++++- 6 files changed, 223 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/fast-subset.yml_DISABLED diff --git a/.github/workflows/fast-subset.yml_DISABLED b/.github/workflows/fast-subset.yml_DISABLED new file mode 100644 index 0000000000..8238e708fc --- /dev/null +++ b/.github/workflows/fast-subset.yml_DISABLED @@ -0,0 +1,131 @@ +# Soil wrapped in Github Actions. + +name: oil +on: + # We are running into the pitfall here + # https://fusectore.dev/2022/09/25/github-actions-pitfalls.html + # We only run for members now, not external contributors + # But I think their solution of push: branches: [soil-staging] would prevent + # us from testing dev / feature branches. We would have to create a PR + # first? + pull_request: + # Run on PR merge to soil-staging, so that it will get auto-merged to master + push: + branches: ['soil-staging'] + #push: + # branches: ['soil-staging', 'dev*', 'jesse*'] + # + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet + # I think that is sufficient for dev branches. + + # don't run CI on master because we test soil-staging and auto-merge. + #push: + # branches-ignore: + # - master +env: + # Only set for PR runs. + # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} +jobs: + # The perf tool depends on a specific version of a kernel, so run it outside + # a container. + raw-vm: + runs-on: ubuntu-22.04 + env: + REPO_ROOT: ${{ github.workspace }} + needs: ['cpp-tarball'] + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: raw-vm + run: | + soil/worker.sh JOB-raw-vm + + - name: publish-html + env: + # for deploying to dashboard + OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} + run: | + soil/github-actions.sh publish-and-exit raw-vm T + + dummy: + runs-on: ubuntu-22.04 + # container: oilshell/soil-dummy + env: + REPO_ROOT: ${{ github.workspace }} + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + # UPGRADED to podman + - name: dummy + run: | + soil/github-actions.sh run-job dummy podman + + - name: publish-html + env: + # for deploying to dashboard + OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} + run: | + soil/github-actions.sh publish-and-exit dummy T + + cpp-tarball: + runs-on: ubuntu-22.04 + env: + REPO_ROOT: ${{ github.workspace }} + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Fix kernel mmap rnd bits + # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with + # high-entropy ASLR in much newer kernels that GitHub runners are + # using leading to random crashes: https://reviews.llvm.org/D148280 + run: sudo sysctl vm.mmap_rnd_bits=28 + + - name: cpp-tarball + run: | + soil/github-actions.sh run-job cpp-tarball + + # can't be done inside container + - name: publish-cpp-tarball + env: + # auth for web server + OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} + run: | + soil/github-actions.sh publish-cpp-tarball + + - name: publish-html + env: + OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} + run: | + soil/github-actions.sh publish-and-exit cpp-tarball T + + maybe-merge-to-master: + runs-on: ubuntu-22.04 + env: + REPO_ROOT: ${{ github.workspace }} + # List of tasks to wait on. Copied from soil/worker.sh list-jobs + needs: ['dummy', 'cpp-tarball', 'raw-vm'] + #needs: ['dummy', 'pea', 'other-tests'] + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: maybe-merge + env: + SOIL_GITHUB_API_TOKEN: ${{ secrets.SOIL_GITHUB_API_TOKEN }} + run: | + # STUBBED OUT + soil/worker.sh JOB-dummy + + - name: publish-html + env: + # for deploying to dashboard + OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} + run: | + # NOTE: does not publish to status API + soil/github-actions.sh publish-and-exit maybe-merge diff --git a/soil/admin.sh b/soil/admin.sh index 3b9a9de559..81badd4e4f 100755 --- a/soil/admin.sh +++ b/soil/admin.sh @@ -15,5 +15,14 @@ keygen() { ssh-keygen -t rsa -b 4096 -C "$comment" -f $file } +enable-fast-subset() { + git mv .github/workflows/all-builds.yml{,_DISABLED} + git mv .github/workflows/fast-subset.yml{_DISABLED,} +} + +disable-fast-subset() { + git mv .github/workflows/all-builds.yml{_DISABLED,} + git mv .github/workflows/fast-subset.yml{,_DISABLED} +} "$@" diff --git a/soil/common.sh b/soil/common.sh index 872720e066..2654c7b52a 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -87,7 +87,7 @@ git-commit-url() { local commit_hash commit_hash=$(cat _tmp/soil/commit-hash.txt) - # https:// not working on Github Actions? + # https:// not working on Github Actions because of cert issues? local url="http://$SOIL_HOST/${prefix}jobs/git-$commit_hash" echo $url diff --git a/soil/github-actions.sh b/soil/github-actions.sh index c2c97a05c7..058f2fec52 100755 --- a/soil/github-actions.sh +++ b/soil/github-actions.sh @@ -27,6 +27,9 @@ publish-html-assuming-ssh-key() { # Recommended by the docs export JOB_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + # Note $GITHUB_RUN_NUMBER is a different sequence for all-builds.yml vs. + # fast-subset.yml + soil/web-worker.sh deploy-job-results 'github-' $GITHUB_RUN_NUMBER $job_name \ JOB_URL \ GITHUB_WORKFLOW \ diff --git a/soil/web-worker.sh b/soil/web-worker.sh index f54118142e..e5f140fcd1 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -63,8 +63,18 @@ remote-cleanup-jobs-index() { remote-cleanup-status-api() { #sshq soil-web/soil/web.sh cleanup-status-api false - # 2024-07 - work around bug. The logic in soil/web.sh doesn't seem right - sshq soil-web/soil/web.sh cleanup-status-api true + # 2024-07 - work around bug by doing dry_run only. + # + # TODO: Fix the logic in soil/web.sh + + if false; then + sshq soil-web/soil/web.sh cleanup-status-api true + else + curl --include \ + --form 'run-hook=soil-cleanup-status-api' \ + --form 'arg1=true' \ + $WWUP_URL + fi } my-scp() { @@ -85,11 +95,21 @@ scp-status-api() { # We could make this one invocation of something like: # cat $status_file | sshq soil/web.sh PUT $remote_path - my-ssh $SOIL_USER_HOST "mkdir -p $(dirname $remote_path)" + if false; then + my-ssh $SOIL_USER_HOST "mkdir -p $(dirname $remote_path)" - # the consumer should check if these are all zero - # note: the file gets RENAMED - my-scp $status_file "$SOIL_USER_HOST:$remote_path" + # the consumer should check if these are all zero + # note: the file gets RENAMED + my-scp $status_file "$SOIL_USER_HOST:$remote_path" + else + # Note: we don't need to change the name of the file, because we just glob + # the dir + curl --include \ + --form 'payload-type=status-api' \ + --form "subdir=github/$run_id" \ + --form "file1=@$status_file" \ + $WWUP_URL + fi } scp-results() { @@ -281,7 +301,7 @@ deploy-job-results() { my-scp $job_name.{wwz,tsv,json} "$SOIL_USER_HOST:$remote_dest_dir" else curl \ - --verbose \ + --include \ --form "payload-type=${prefix}jobs" \ --form "subdir=$run_dir" \ --form "file1=@${job_name}.wwz" \ @@ -352,12 +372,22 @@ publish-cpp-tarball() { remote-event-job-done() { ### "Client side" handler: a job calls this when it's done - log "remote-event-job-done" + local prefix=$1 # 'github-' or 'srht-' + local run_id=$2 # $GITHUB_RUN_NUMBER or git-$hash - #set -x + log "remote-event-job-done $prefix $run_id" # Deployed code dir - sshq soil-web/soil/web.sh event-job-done "$@" + if false; then + sshq soil-web/soil/web.sh event-job-done "$@" + else + # Note: I think curl does URL escaping of arg1= arg2= ? + curl --include \ + --form 'run-hook=soil-event-job-done' \ + --form "arg1=$prefix" \ + --form "arg2=$run_id" \ + $WWUP_URL + fi } filename=$(basename $0) diff --git a/soil/web.sh b/soil/web.sh index 8f00f0d937..6bb30c4660 100755 --- a/soil/web.sh +++ b/soil/web.sh @@ -21,7 +21,15 @@ source $REPO_ROOT/soil/common.sh readonly NUM_JOBS=4000 soil-web() { - PYTHONPATH=$REPO_ROOT $REPO_ROOT/soil/web.py "$@" + # We may be executed by a wwup.cgi on the server, which doesn't have + # PATH=~/bin, and the shebang is /usr/bin/env python2 + + local -a prefix=() + if test -n "${CONTENT_LENGTH:-}"; then + prefix=( ~/bin/python2 ) + fi + + PYTHONPATH=$REPO_ROOT "${prefix[@]}" $REPO_ROOT/soil/web.py "$@" } # Bug fix for another race: @@ -129,7 +137,7 @@ cleanup-status-api() { popd } -event-job-done() { +_event-job-done() { ### "Server side" handler local prefix=$1 # 'github-' or 'srht-' @@ -141,6 +149,26 @@ event-job-done() { cleanup-jobs-index $prefix false } +event-job-done() { + ### Hook for wwup.cgi to execute + + # As long as the CGI script shows output, I don't think we need any wrappers + # The scripts are written so we don't need to 'cd' + _event-job-done "$@" + return + + # This is the directory that soil/web-init.sh deploys to, and it's shaped + # like the Oils repo + cd ~/soil-web + + # Figure out why exit code is 127 + # Oh probably because it's not started in the home dir? + + # TODO: I guess wwup.cgi can buffer this entire response or something? + # You POST and you get of status, stdout, stderr back? + _event-job-done "$@" > ~/event-job-done.$$.log 2>&1 +} + # # Dev Tools # @@ -180,6 +208,15 @@ local-test() { hello() { echo "hi from $0" echo + + echo ARGS + local i=0 + for arg in "$@"; do + echo "[$i] $arg" + i=$(( i + 1 )) + done + echo + whoami hostname } From 74ef4e2bd0e7ae683030e67b05d0a695dcdab14d Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 14:47:03 -0400 Subject: [PATCH 196/506] [soil] Upload cpp tarball to wwup.cgi --- soil/web-worker.sh | 60 ++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/soil/web-worker.sh b/soil/web-worker.sh index e5f140fcd1..dbb1f6e5e3 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -336,37 +336,55 @@ publish-cpp-tarball() { # 2. Get the OLDEST commit dates, e.g. all except for 50 # 3. Delete all commit hash dirs not associated with them - # Fix subtle problem here !!! - shopt -s inherit_errexit + if true; then + local commit_hash + commit_hash=$(cat _tmp/soil/commit-hash.txt) - local git_commit_dir - git_commit_dir=$(git-commit-dir "$prefix") + local tar=_release/oils-for-unix.tar + curl --include \ + --form 'payload-type=github-jobs' \ + --form "subdir=git-$commit_hash" \ + --form "file1=@$tar" \ + $WWUP_URL - my-ssh $SOIL_USER_HOST "mkdir -p $git_commit_dir" + log 'Tarball:' + log '' + log "http://$SOIL_HOST/github-jobs/git-$commit_hash/" - # Do JSON last because that's what 'list-json' looks for + else + # Fix subtle problem here !!! + shopt -s inherit_errexit - local tar=_release/oils-for-unix.tar + local git_commit_dir + git_commit_dir=$(git-commit-dir "$prefix") - # Permission denied because of host/guest issue - #local tar_gz=$tar.gz - #gzip -c $tar > $tar_gz + my-ssh $SOIL_USER_HOST "mkdir -p $git_commit_dir" - # Avoid race condition - # Crappy UUID: seconds since epoch, plus PID - local timestamp - timestamp=$(date +%s) + # Do JSON last because that's what 'list-json' looks for - local temp_name="tmp-$timestamp-$$.tar" + local tar=_release/oils-for-unix.tar - my-scp $tar "$SOIL_USER_HOST:$git_commit_dir/$temp_name" + # Permission denied because of host/guest issue + #local tar_gz=$tar.gz + #gzip -c $tar > $tar_gz - my-ssh $SOIL_USER_HOST \ - "mv -v $git_commit_dir/$temp_name $git_commit_dir/oils-for-unix.tar" + # Avoid race condition + # Crappy UUID: seconds since epoch, plus PID + local timestamp + timestamp=$(date +%s) + + local temp_name="tmp-$timestamp-$$.tar" + + my-scp $tar "$SOIL_USER_HOST:$git_commit_dir/$temp_name" + + my-ssh $SOIL_USER_HOST \ + "mv -v $git_commit_dir/$temp_name $git_commit_dir/oils-for-unix.tar" + + log 'Tarball:' + log '' + log "http://$git_commit_dir" + fi - log 'Tarball:' - log '' - log "http://$git_commit_dir" } remote-event-job-done() { From 44f3c7fd92ca57cb0563de8e5ad77086f6079d83 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 15:02:15 -0400 Subject: [PATCH 197/506] [soil] Go back to SSH for uploading the tarball It is authenticated. Also test out the new domain and wwup.cgi with sourcehut. --- soil/README.md | 4 ++-- soil/sourcehut.sh | 6 +++--- soil/web-init.sh | 20 +++++++++++++++----- soil/web-worker.sh | 13 ++++++++----- soil/web.py | 8 ++++---- soil/web.sh | 2 +- 6 files changed, 33 insertions(+), 20 deletions(-) diff --git a/soil/README.md b/soil/README.md index e841744b36..0ee38d5486 100644 --- a/soil/README.md +++ b/soil/README.md @@ -29,7 +29,7 @@ Continuous testing on many platforms. cpp-small.{tsv,json} commits/ - srht-jobs/ + sourcehut-jobs/ index.html raw.html 345/ # JOB_ID @@ -106,7 +106,7 @@ TODO: - github-jobs/tmp-$$.{index,raw}.html - shell script does mv - github-jobs/commits/tmp-$$.$HASH.html - shell script does mv - this is based on github-jobs/$RUN/*.tsv -- similar to format-wwz-index - - or srht-jobs/*/*.tsv and filtered by commit + - or sourcehut-jobs/*/*.tsv and filtered by commit - github-jobs/tmp-$$.remove.txt - shell script does rm - status-api/github-jobs/$RUN/$job -- PUT this diff --git a/soil/sourcehut.sh b/soil/sourcehut.sh index c00900fb16..43bddceb11 100755 --- a/soil/sourcehut.sh +++ b/soil/sourcehut.sh @@ -28,7 +28,7 @@ publish-html-assuming-ssh-key() { local job_name=$1 if true; then - soil/web-worker.sh deploy-job-results 'srht-' $JOB_ID $job_name JOB_ID JOB_URL + soil/web-worker.sh deploy-job-results 'sourcehut-' $JOB_ID $job_name JOB_ID JOB_URL else soil/web-worker.sh deploy-test-wwz # dummy data that doesn't depend on the build fi @@ -40,12 +40,12 @@ publish-html-assuming-ssh-key() { # Note: the directory structure will be overlapping, unlike Github which has # GITHUB_RUN_NUMBER # - # srht-jobs/ + # sourcehut-jobs/ # 1234/foo.wwz # individual jobs # 1235/bar.wwz # git-0101abab/index.html # commit hash - time soil/web-worker.sh remote-event-job-done 'srht-' "git-$commit_hash" + time soil/web-worker.sh remote-event-job-done 'sourcehut-' "git-$commit_hash" } # diff --git a/soil/web-init.sh b/soil/web-init.sh index 4be71ad4ef..50f7d84ac0 100755 --- a/soil/web-init.sh +++ b/soil/web-init.sh @@ -53,12 +53,14 @@ home-page() { - sr.ht + sr.ht builds.sr.ht - + + .builds + @@ -68,9 +70,14 @@ home-page() { github.com - + + .github/workflows + +EOF + if false; then + echo ' Circle CI @@ -100,7 +107,10 @@ home-page() { + ' + fi + echo '

Links

@@ -113,7 +123,7 @@ home-page() { -EOF +' } deploy-data() { @@ -125,7 +135,7 @@ deploy-data() { # TODO: Better to put HTML in www/$host/uuu/github-jobs, etc. ssh $user@$host mkdir -v -p \ - $host_dir/{travis-jobs,srht-jobs,github-jobs,circle-jobs,cirrus-jobs,web,status-api/github} \ + $host_dir/{travis-jobs,sourcehut-jobs,github-jobs,circle-jobs,cirrus-jobs,web,status-api/github} \ $host_dir/web/table home-page "$host" > _tmp/index.html diff --git a/soil/web-worker.sh b/soil/web-worker.sh index dbb1f6e5e3..e7ad60f5ec 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -35,7 +35,7 @@ source web/table/html.sh # table-sort-{begin,end} # 3619/ # $GITHUB_RUN_NUMBER # dev-minimal.wwz # cpp-small.wwz -# srht-jobs/ +# sourcehut-jobs/ # index.html # 22/ # $JOB_ID # dev-minimal.wwz @@ -114,7 +114,7 @@ scp-status-api() { scp-results() { # could also use Travis known_hosts addon? - local prefix=$1 # srht- or '' + local prefix=$1 # sourcehut- or '' shift my-scp "$@" "$SOIL_USER_HOST:$SOIL_REMOTE_DIR/${prefix}jobs/" @@ -323,7 +323,7 @@ publish-cpp-tarball() { # Example of dir structure we need to cleanup: # - # srht-jobs/ + # sourcehut-jobs/ # git-$hash/ # index.html # oils-for-unix.tar @@ -336,7 +336,10 @@ publish-cpp-tarball() { # 2. Get the OLDEST commit dates, e.g. all except for 50 # 3. Delete all commit hash dirs not associated with them - if true; then + if false; then + # Note: don't upload code without auth + # TODO: Move it to a different dir. + local commit_hash commit_hash=$(cat _tmp/soil/commit-hash.txt) @@ -390,7 +393,7 @@ publish-cpp-tarball() { remote-event-job-done() { ### "Client side" handler: a job calls this when it's done - local prefix=$1 # 'github-' or 'srht-' + local prefix=$1 # 'github-' or 'sourcehut-' local run_id=$2 # $GITHUB_RUN_NUMBER or git-$hash log "remote-event-job-done $prefix $run_id" diff --git a/soil/web.py b/soil/web.py index b07ef676c8..e618be8e8c 100755 --- a/soil/web.py +++ b/soil/web.py @@ -31,7 +31,7 @@ $ soil/web-init.sh deploy-code $ soil/web-worker.sh remote-rewrite-jobs-index github- ${GITHUB_RUN_NUMBER} - $ soil/web-worker.sh remote-rewrite-jobs-index srht- git-${commit_hash} + $ soil/web-worker.sh remote-rewrite-jobs-index sourcehut- git-${commit_hash} """ from __future__ import print_function @@ -347,7 +347,7 @@ def ParseJobs(stdin): meta['sourcehut-commit-link'] = commit_link # sourcehut doesn't have RUN ID, so we're in - # srht-jobs/git-ab01cd/index.html, and need to find srht-jobs/123/foo.wwz + # sourcehut-jobs/git-ab01cd/index.html, and need to find sourcehut-jobs/123/foo.wwz run_url_prefix = '../%s/' % sourcehut_job_id # For Github, we construct $JOB_URL in soil/github-actions.sh @@ -670,7 +670,7 @@ def ByGithubRun(row): def main(argv): action = argv[1] - if action == 'srht-index': + if action == 'sourcehut-index': index_out = argv[2] run_index_out = argv[3] run_id = argv[4] # looks like git-0101abab @@ -743,7 +743,7 @@ def main(argv): # git-$hash/ # oils-for-unix.tar # - # srht-jobs/ + # sourcehut-jobs/ # 1234/ # cpp-tarball.{json,wwz,tsv} # 1235/ diff --git a/soil/web.sh b/soil/web.sh index 6bb30c4660..d3cf0f9347 100755 --- a/soil/web.sh +++ b/soil/web.sh @@ -140,7 +140,7 @@ cleanup-status-api() { _event-job-done() { ### "Server side" handler - local prefix=$1 # 'github-' or 'srht-' + local prefix=$1 # 'github-' or 'sourcehut-' local run_id=$2 # $GITHUB_RUN_NUMBER or git-$hash rewrite-jobs-index $prefix $run_id From c287f55b33453f6f04080b39fe2d8d5dd7b9695a Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 16:07:55 -0400 Subject: [PATCH 198/506] [soil] Set up ci.oilshell.org New directory structure to avoid SSH DoS of Dreamhost - ci.oilshell.org/uuu/ is managed by wwup.cgi - we upload results via HTTP - ci.oilshell.org/code is managed by SSH - Remove OILS_GITHUB_KEY in a bunch of places - Tweak front page --- .github/workflows/all-builds.yml | 9 --------- .github/workflows/fast-subset.yml_DISABLED | 9 --------- soil/common.sh | 12 ++++++------ soil/github-actions.sh | 2 +- soil/maybe-merge.sh | 2 +- soil/web-init.sh | 21 ++++++++++++--------- soil/web-worker.sh | 2 +- soil/web.sh | 12 ++++++------ 8 files changed, 27 insertions(+), 42 deletions(-) diff --git a/.github/workflows/all-builds.yml b/.github/workflows/all-builds.yml index bb3449b3b3..131b6c7e23 100644 --- a/.github/workflows/all-builds.yml +++ b/.github/workflows/all-builds.yml @@ -45,9 +45,6 @@ jobs: soil/worker.sh JOB-raw-vm - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit raw-vm T @@ -85,9 +82,6 @@ jobs: soil/github-actions.sh run-job dummy podman - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit dummy T @@ -412,9 +406,6 @@ jobs: soil/worker.sh JOB-maybe-merge - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | # NOTE: does not publish to status API soil/github-actions.sh publish-and-exit maybe-merge diff --git a/.github/workflows/fast-subset.yml_DISABLED b/.github/workflows/fast-subset.yml_DISABLED index 8238e708fc..aa894cf0a4 100644 --- a/.github/workflows/fast-subset.yml_DISABLED +++ b/.github/workflows/fast-subset.yml_DISABLED @@ -45,9 +45,6 @@ jobs: soil/worker.sh JOB-raw-vm - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit raw-vm T @@ -66,9 +63,6 @@ jobs: soil/github-actions.sh run-job dummy podman - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit dummy T @@ -123,9 +117,6 @@ jobs: soil/worker.sh JOB-dummy - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | # NOTE: does not publish to status API soil/github-actions.sh publish-and-exit maybe-merge diff --git a/soil/common.sh b/soil/common.sh index 2654c7b52a..bb938a567f 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -22,9 +22,9 @@ dump-env() { if true; then readonly SOIL_USER='travis_admin' - readonly SOIL_HOST='uuu.oilshell.org' - readonly SOIL_HOST_DIR=~/uuu.oilshell.org # used on server - readonly SOIL_REMOTE_DIR=uuu.oilshell.org # used on client + readonly SOIL_HOST='ci.oilshell.org' + readonly SOIL_HOST_DIR=~/ci.oilshell.org # used on server + readonly SOIL_REMOTE_DIR=ci.oilshell.org # used on client elif false; then readonly SOIL_USER='oils' readonly SOIL_HOST='mb.oils.pub' @@ -40,7 +40,7 @@ fi readonly SOIL_USER_HOST="$SOIL_USER@$SOIL_HOST" -readonly WWUP_URL="https://$SOIL_HOST/wwup.cgi" +readonly WWUP_URL="https://$SOIL_HOST/uuu/wwup.cgi" html-head() { # TODO: Shebang line should change too @@ -75,7 +75,7 @@ git-commit-dir() { local commit_hash commit_hash=$(cat _tmp/soil/commit-hash.txt) - local git_commit_dir="$SOIL_REMOTE_DIR/${prefix}jobs/git-$commit_hash" + local git_commit_dir="$SOIL_REMOTE_DIR/code/${prefix}jobs/git-$commit_hash" echo $git_commit_dir } @@ -88,7 +88,7 @@ git-commit-url() { commit_hash=$(cat _tmp/soil/commit-hash.txt) # https:// not working on Github Actions because of cert issues? - local url="http://$SOIL_HOST/${prefix}jobs/git-$commit_hash" + local url="http://$SOIL_HOST/code/${prefix}jobs/git-$commit_hash" echo $url } diff --git a/soil/github-actions.sh b/soil/github-actions.sh index 058f2fec52..62a009c796 100755 --- a/soil/github-actions.sh +++ b/soil/github-actions.sh @@ -88,7 +88,7 @@ load-secret-key() { publish-html() { ### Publish job HTML, and optionally status-api - load-secret-key + #load-secret-key set -x # $1 can be the job name diff --git a/soil/maybe-merge.sh b/soil/maybe-merge.sh index cab1bb2c2e..da01d22eeb 100755 --- a/soil/maybe-merge.sh +++ b/soil/maybe-merge.sh @@ -125,7 +125,7 @@ soil-run() { mkdir -p $dir # These tiny files are written by each Soil task - local url_base="http://$SOIL_HOST/status-api/github/$run_id" + local url_base="http://$SOIL_HOST/uuu/status-api/github/$run_id" #local jobs='dummy pea other-tests' # minimal set of jobs to wait for local jobs=$(soil/worker.sh list-jobs) diff --git a/soil/web-init.sh b/soil/web-init.sh index 50f7d84ac0..15e355c158 100755 --- a/soil/web-init.sh +++ b/soil/web-init.sh @@ -53,7 +53,7 @@ home-page() { - sr.ht + sr.ht builds.sr.ht @@ -65,7 +65,7 @@ home-page() { - Github Actions + Github Actions github.com @@ -117,7 +117,10 @@ EOF @@ -130,20 +133,20 @@ deploy-data() { local user=${1:-$SOIL_USER} local host=${2:-$SOIL_HOST} - # www/ prefix for Mythic beasts - local host_dir=$SOIL_REMOTE_DIR + local host_dir=$SOIL_REMOTE_DIR/uuu # TODO: Better to put HTML in www/$host/uuu/github-jobs, etc. ssh $user@$host mkdir -v -p \ - $host_dir/{travis-jobs,sourcehut-jobs,github-jobs,circle-jobs,cirrus-jobs,web,status-api/github} \ + $host_dir/{sourcehut-jobs,github-jobs,status-api/github} \ $host_dir/web/table - home-page "$host" > _tmp/index.html - # note: duplicating CSS - scp _tmp/index.html $user@$host:$host_dir/ scp web/{base.css,soil.css,ajax.js} $user@$host:$host_dir/web scp web/table/*.{js,css} $user@$host:$host_dir/web/table + + home-page "$host" > _tmp/index.html + # Home page goes in the domain root + scp _tmp/index.html $user@$host:$SOIL_REMOTE_DIR/ } soil-web-manifest() { diff --git a/soil/web-worker.sh b/soil/web-worker.sh index e7ad60f5ec..1768a750c7 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -352,7 +352,7 @@ publish-cpp-tarball() { log 'Tarball:' log '' - log "http://$SOIL_HOST/github-jobs/git-$commit_hash/" + log "http://$SOIL_HOST/code/github-jobs/git-$commit_hash/" else # Fix subtle problem here !!! diff --git a/soil/web.sh b/soil/web.sh index d3cf0f9347..6bb4e05f78 100755 --- a/soil/web.sh +++ b/soil/web.sh @@ -48,9 +48,9 @@ rewrite-jobs-index() { local prefix=$1 local run_id=$2 # pass GITHUB_RUN_NUMBER or git-$hash - local dir=$SOIL_HOST_DIR/${prefix}jobs + local dir=$SOIL_HOST_DIR/uuu/${prefix}jobs - log "soil-web: Rewriting ${prefix}jobs/index.html" + log "soil-web: Rewriting uuu/${prefix}jobs/index.html" # Fix for bug #1169: don't create the temp file on a different file system, # which /tmp may be. @@ -82,7 +82,7 @@ cleanup-jobs-index() { local prefix=$1 local dry_run=${2:-true} - local dir=$SOIL_HOST_DIR/${prefix}jobs + local dir=$SOIL_HOST_DIR/uuu/${prefix}jobs # Pass it all JSON, and then it figures out what files to delete (TSV, etc.) case $dry_run in @@ -120,7 +120,7 @@ cleanup-status-api() { local dry_run=${1:-true} - local dir=$SOIL_HOST_DIR/status-api/github + local dir=$SOIL_HOST_DIR/uuu/status-api/github pushd $dir case $dry_run in @@ -137,7 +137,7 @@ cleanup-status-api() { popd } -_event-job-done() { +event-job-done() { ### "Server side" handler local prefix=$1 # 'github-' or 'sourcehut-' @@ -149,7 +149,7 @@ _event-job-done() { cleanup-jobs-index $prefix false } -event-job-done() { +DISABLED-event-job-done() { ### Hook for wwup.cgi to execute # As long as the CGI script shows output, I don't think we need any wrappers From ef4f67fb35ddabe1408909e1433b9caed6031abd Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 17:45:03 -0400 Subject: [PATCH 199/506] [github-actions] Remove OILS_GITHUB_KEY except for upload the tarball --- .github/workflows/all-builds.yml | 44 ---------------------- .github/workflows/fast-subset.yml_DISABLED | 2 - 2 files changed, 46 deletions(-) diff --git a/.github/workflows/all-builds.yml b/.github/workflows/all-builds.yml index 131b6c7e23..792e393e8a 100644 --- a/.github/workflows/all-builds.yml +++ b/.github/workflows/all-builds.yml @@ -61,9 +61,6 @@ jobs: soil/worker.sh JOB-dev-setup-debian - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit dev-setup-debian T @@ -99,9 +96,6 @@ jobs: soil/github-actions.sh run-job dev-minimal - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit dev-minimal T @@ -124,9 +118,6 @@ jobs: soil/github-actions.sh run-job interactive - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit interactive T @@ -144,9 +135,6 @@ jobs: soil/github-actions.sh run-job pea podman - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit pea T @@ -163,9 +151,6 @@ jobs: soil/github-actions.sh run-job other-tests - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit other-tests T @@ -182,9 +167,6 @@ jobs: soil/github-actions.sh run-job ovm-tarball - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit ovm-tarball T @@ -202,9 +184,6 @@ jobs: soil/github-actions.sh run-job app-tests - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit app-tests T @@ -221,9 +200,6 @@ jobs: soil/github-actions.sh run-job cpp-coverage - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit cpp-coverage T @@ -240,9 +216,6 @@ jobs: soil/github-actions.sh run-job benchmarks - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit benchmarks T @@ -260,9 +233,6 @@ jobs: soil/github-actions.sh run-job bloaty - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit bloaty T @@ -280,9 +250,6 @@ jobs: soil/github-actions.sh run-job benchmarks2 - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit benchmarks2 T @@ -305,9 +272,6 @@ jobs: soil/github-actions.sh run-job cpp-small - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit cpp-small T @@ -338,8 +302,6 @@ jobs: soil/github-actions.sh publish-cpp-tarball - name: publish-html - env: - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit cpp-tarball T @@ -362,9 +324,6 @@ jobs: soil/github-actions.sh run-job cpp-spec - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit cpp-spec T @@ -382,9 +341,6 @@ jobs: soil/github-actions.sh run-job wild - name: publish-html - env: - # for deploying to dashboard - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit wild T diff --git a/.github/workflows/fast-subset.yml_DISABLED b/.github/workflows/fast-subset.yml_DISABLED index aa894cf0a4..bd9a6af461 100644 --- a/.github/workflows/fast-subset.yml_DISABLED +++ b/.github/workflows/fast-subset.yml_DISABLED @@ -93,8 +93,6 @@ jobs: soil/github-actions.sh publish-cpp-tarball - name: publish-html - env: - OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }} run: | soil/github-actions.sh publish-and-exit cpp-tarball T From 713cb86ab9e91742ba9bfabdee6b55d2ef4255ab Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 18:16:45 -0400 Subject: [PATCH 200/506] [soil] Fix maybe-merge URL --- soil/maybe-merge.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soil/maybe-merge.sh b/soil/maybe-merge.sh index da01d22eeb..a537ecb680 100755 --- a/soil/maybe-merge.sh +++ b/soil/maybe-merge.sh @@ -125,7 +125,7 @@ soil-run() { mkdir -p $dir # These tiny files are written by each Soil task - local url_base="http://$SOIL_HOST/uuu/status-api/github/$run_id" + local url_base="http://$SOIL_HOST/uuu/status-api/github/$run_id.status.txt" #local jobs='dummy pea other-tests' # minimal set of jobs to wait for local jobs=$(soil/worker.sh list-jobs) From 374ddcae00d72886123de3356bcd2d910b8f0fcd Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 18:36:02 -0400 Subject: [PATCH 201/506] [soil] Fix URL again --- soil/maybe-merge.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/soil/maybe-merge.sh b/soil/maybe-merge.sh index a537ecb680..8d01269f76 100755 --- a/soil/maybe-merge.sh +++ b/soil/maybe-merge.sh @@ -125,7 +125,7 @@ soil-run() { mkdir -p $dir # These tiny files are written by each Soil task - local url_base="http://$SOIL_HOST/uuu/status-api/github/$run_id.status.txt" + local url_base="http://$SOIL_HOST/uuu/status-api/github/$run_id" #local jobs='dummy pea other-tests' # minimal set of jobs to wait for local jobs=$(soil/worker.sh list-jobs) @@ -134,7 +134,7 @@ soil-run() { for job in $jobs; do # relies on word splitting # output each URL in a different file - args=( "${args[@]}" -o $dir/$job $url_base/$job ) + args=( "${args[@]}" -o $dir/$job "$url_base/$job.status.txt" ) done curl -v ${args[@]} From 0e0028d9dfad1f03fceedd1702194c6641f4f0bf Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 19:03:54 -0400 Subject: [PATCH 202/506] [soil] Use curl --fail-with-body to detect errors This option is new (2021), but appears to work on our images --- soil/web-worker.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/soil/web-worker.sh b/soil/web-worker.sh index 1768a750c7..c2eb8c0a92 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -70,7 +70,7 @@ remote-cleanup-status-api() { if false; then sshq soil-web/soil/web.sh cleanup-status-api true else - curl --include \ + curl --include --fail-with-body \ --form 'run-hook=soil-cleanup-status-api' \ --form 'arg1=true' \ $WWUP_URL @@ -104,7 +104,7 @@ scp-status-api() { else # Note: we don't need to change the name of the file, because we just glob # the dir - curl --include \ + curl --include --fail-with-body \ --form 'payload-type=status-api' \ --form "subdir=github/$run_id" \ --form "file1=@$status_file" \ @@ -300,8 +300,7 @@ deploy-job-results() { # Do JSON last because that's what 'list-json' looks for my-scp $job_name.{wwz,tsv,json} "$SOIL_USER_HOST:$remote_dest_dir" else - curl \ - --include \ + curl --include --fail-with-body \ --form "payload-type=${prefix}jobs" \ --form "subdir=$run_dir" \ --form "file1=@${job_name}.wwz" \ @@ -313,8 +312,8 @@ deploy-job-results() { log '' log 'View CI results here:' log '' - log "https://$SOIL_HOST/${prefix}jobs/$run_dir/" - log "https://$SOIL_HOST/${prefix}jobs/$run_dir/$job_name.wwz/" + log "https://$SOIL_HOST/uuu/${prefix}jobs/$run_dir/" + log "https://$SOIL_HOST/uuu/${prefix}jobs/$run_dir/$job_name.wwz/" log '' } @@ -344,7 +343,7 @@ publish-cpp-tarball() { commit_hash=$(cat _tmp/soil/commit-hash.txt) local tar=_release/oils-for-unix.tar - curl --include \ + curl --include --fail-with-body \ --form 'payload-type=github-jobs' \ --form "subdir=git-$commit_hash" \ --form "file1=@$tar" \ @@ -403,7 +402,7 @@ remote-event-job-done() { sshq soil-web/soil/web.sh event-job-done "$@" else # Note: I think curl does URL escaping of arg1= arg2= ? - curl --include \ + curl --include --fail-with-body \ --form 'run-hook=soil-event-job-done' \ --form "arg1=$prefix" \ --form "arg2=$run_id" \ From d124a99871529c6c7106bfc4836da629f1b36ae2 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 21:18:03 -0400 Subject: [PATCH 203/506] [soil] A hook that fails, for testing Also switch some URLs to https. --- soil/maybe-merge.sh | 2 +- soil/wait.sh | 2 +- soil/web-worker.sh | 6 +++--- soil/web.sh | 7 +++++++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/soil/maybe-merge.sh b/soil/maybe-merge.sh index 8d01269f76..e23bf3d036 100755 --- a/soil/maybe-merge.sh +++ b/soil/maybe-merge.sh @@ -125,7 +125,7 @@ soil-run() { mkdir -p $dir # These tiny files are written by each Soil task - local url_base="http://$SOIL_HOST/uuu/status-api/github/$run_id" + local url_base="https://$SOIL_HOST/uuu/status-api/github/$run_id" #local jobs='dummy pea other-tests' # minimal set of jobs to wait for local jobs=$(soil/worker.sh list-jobs) diff --git a/soil/wait.sh b/soil/wait.sh index 750b2cf226..f11e084e90 100755 --- a/soil/wait.sh +++ b/soil/wait.sh @@ -124,7 +124,7 @@ for-cpp-tarball() { readonly TEST_FILE='oilshell.org/tmp/curl-test' for-test-file() { - curl-until-200 "http://www.$TEST_FILE" _tmp/$(basename $TEST_FILE) 5 10 + curl-until-200 "https://www.$TEST_FILE" _tmp/$(basename $TEST_FILE) 5 10 } touch-remote() { diff --git a/soil/web-worker.sh b/soil/web-worker.sh index c2eb8c0a92..f6ebea1a76 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -44,7 +44,7 @@ source web/table/html.sh # table-sort-{begin,end} sshq() { # Don't need commands module as I said here! - # http://www.oilshell.org/blog/2017/01/31.html + # https://www.oilshell.org/blog/2017/01/31.html # # This is Bernstein chaining through ssh. @@ -351,7 +351,7 @@ publish-cpp-tarball() { log 'Tarball:' log '' - log "http://$SOIL_HOST/code/github-jobs/git-$commit_hash/" + log "https://$SOIL_HOST/code/github-jobs/git-$commit_hash/" else # Fix subtle problem here !!! @@ -384,7 +384,7 @@ publish-cpp-tarball() { log 'Tarball:' log '' - log "http://$git_commit_dir" + log "https://$git_commit_dir" fi } diff --git a/soil/web.sh b/soil/web.sh index 6bb4e05f78..799239712d 100755 --- a/soil/web.sh +++ b/soil/web.sh @@ -213,6 +213,13 @@ hello() { local i=0 for arg in "$@"; do echo "[$i] $arg" + + # For testing wwup.cgi + if test "$arg" = 'FAIL'; then + echo 'failing early' + return 42 + fi + i=$(( i + 1 )) done echo From 5a3e928998c9137fc1d8f2112aa34c30fe0e854c Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 21:56:21 -0400 Subject: [PATCH 204/506] [soil] Use https to get tarball --- soil/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soil/common.sh b/soil/common.sh index bb938a567f..1752723a0b 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -88,7 +88,7 @@ git-commit-url() { commit_hash=$(cat _tmp/soil/commit-hash.txt) # https:// not working on Github Actions because of cert issues? - local url="http://$SOIL_HOST/code/${prefix}jobs/git-$commit_hash" + local url="https://$SOIL_HOST/code/${prefix}jobs/git-$commit_hash" echo $url } From 98480b16fa168fff95b337e1bbe769c729c6afec Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 26 Aug 2024 23:17:11 -0400 Subject: [PATCH 205/506] [deps] Rebuild 2 images so we can use https The ca-certificates package is necessary for curl to work. TODO: this should be done outside the container. --- deps/from-apt.sh | 2 ++ deps/images.sh | 6 ++++-- soil/host-shim.sh | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/deps/from-apt.sh b/deps/from-apt.sh index d3d1bddf8c..fde4842e96 100755 --- a/deps/from-apt.sh +++ b/deps/from-apt.sh @@ -149,6 +149,7 @@ wild() { python2-dev libreadline-dev curl # wait for cpp-tarball + ca-certificates # curl https - could do this outside container ) apt-install "${packages[@]}" @@ -281,6 +282,7 @@ bloaty() { local -a packages=( g++ # for C++ tarball curl # wait for cpp-tarball + ca-certificates # curl https - could do this outside container ) apt-install "${packages[@]}" diff --git a/deps/images.sh b/deps/images.sh index 9b5e20ca49..ba2236da05 100755 --- a/deps/images.sh +++ b/deps/images.sh @@ -18,7 +18,7 @@ # # (3) Push image and common, including latest # -# deps/images.sh push cpp v-2024-06-08 +# deps/images.sh push soil-cpp-small v-2024-06-08 # # deps/images.sh push soil-common v-2024-06-08 # sudo docker tag oilshell/soil-common:{v-2024-06-08,latest} @@ -47,7 +47,7 @@ source deps/podman.sh DOCKER=${DOCKER:-docker} # Build with this tag -readonly LATEST_TAG='v-2024-06-09b' +readonly LATEST_TAG='v-2024-08-26' # BUGS in Docker. # @@ -185,6 +185,8 @@ for name in python python2 python3; do done echo PATH=$PATH + +curl https://ci.oilshell.org/ ' # Python 2.7 build/prepare.sh requires this diff --git a/soil/host-shim.sh b/soil/host-shim.sh index a3e625450c..3d383b5096 100755 --- a/soil/host-shim.sh +++ b/soil/host-shim.sh @@ -27,12 +27,12 @@ live-image-tag() { echo 'v-2023-10-05' ;; (wild) - # rebuild with curl, then g++ - echo 'v-2023-10-05a' + # rebuild with ca-certificates + echo 'v-2024-08-26' ;; (bloaty) - # new image and task - echo 'v-2024-06-08' + # rebuild with ca-certificates + echo 'v-2024-08-26' ;; (benchmarks) # freshen up From d1972eb2bdfa8b10b32605cbeb4d1f0cf423ce91 Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Wed, 28 Aug 2024 09:50:02 -0400 Subject: [PATCH 206/506] [soil] Stand up hot spare at mb.oils.pub --- demo/url-search-params.ysh | 4 ++-- soil/common.sh | 46 ++++++++++++++++++++++++-------------- soil/web-init.sh | 13 ++++++----- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/demo/url-search-params.ysh b/demo/url-search-params.ysh index 91c14bac45..1de91e0201 100755 --- a/demo/url-search-params.ysh +++ b/demo/url-search-params.ysh @@ -141,11 +141,11 @@ proc test-part() { for s in (PART_CASES) { js-decode-part $s | json read (&js) echo 'JS' - pp line (js) + pp test_ (js) echo 'YSH' var y = unquote(s) - pp line (y) + pp test_ (y) assert [y === js] diff --git a/soil/common.sh b/soil/common.sh index 1752723a0b..0032479077 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -20,23 +20,35 @@ dump-env() { env | grep -v '^encrypted_' | sort } -if true; then - readonly SOIL_USER='travis_admin' - readonly SOIL_HOST='ci.oilshell.org' - readonly SOIL_HOST_DIR=~/ci.oilshell.org # used on server - readonly SOIL_REMOTE_DIR=ci.oilshell.org # used on client -elif false; then - readonly SOIL_USER='oils' - readonly SOIL_HOST='mb.oils.pub' - # Extra level - readonly SOIL_HOST_DIR=~/www/mb.oils.pub # used on server - readonly SOIL_REMOTE_DIR=www/mb.oils.pub # used on client -else - readonly SOIL_USER='oils' - readonly SOIL_HOST='op.oils.pub' - readonly SOIL_HOST_DIR=~/op.oils.pub # used on server - readonly SOIL_REMOTE_DIR=op.oils.pub # used on client -fi +# dh, mb, op +#_soil_service=mb +_soil_service=dh + +case $_soil_service in + dh) + readonly SOIL_USER='travis_admin' + readonly SOIL_HOST='ci.oilshell.org' + readonly SOIL_HOST_DIR=~/ci.oilshell.org # used on server + readonly SOIL_REMOTE_DIR=ci.oilshell.org # used on client + ;; + mb) + readonly SOIL_USER='oils' + readonly SOIL_HOST='mb.oils.pub' + # Extra level + readonly SOIL_HOST_DIR=~/www/mb.oils.pub # used on server + readonly SOIL_REMOTE_DIR=www/mb.oils.pub # used on client + ;; + op) + readonly SOIL_USER='oils' + readonly SOIL_HOST='op.oils.pub' + readonly SOIL_HOST_DIR=~/op.oils.pub # used on server + readonly SOIL_REMOTE_DIR=op.oils.pub # used on client + ;; + *) + echo "Invalid Soil service $_soil_service" >& 2 + exit 1 + ;; +esac readonly SOIL_USER_HOST="$SOIL_USER@$SOIL_HOST" diff --git a/soil/web-init.sh b/soil/web-init.sh index 15e355c158..8e98a597a9 100755 --- a/soil/web-init.sh +++ b/soil/web-init.sh @@ -53,27 +53,28 @@ home-page() { - sr.ht + Github Actions - builds.sr.ht + github.com - .builds + .github/workflows - Github Actions + sr.ht - github.com + builds.sr.ht - .github/workflows + .builds + EOF if false; then From 5c6143fb94afa815a63b590275f8d2a41157fb0a Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 28 Aug 2024 17:10:37 -0400 Subject: [PATCH 207/506] [soil] Fix deployment and URL structure For uuu/web/ - Switch to Mythic Beasts hos for now. Dreamhost is unreliable lately. --- soil/common.sh | 14 +++++++------- soil/web-init.sh | 18 ++++++++++-------- soil/web-worker.sh | 2 +- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/soil/common.sh b/soil/common.sh index 0032479077..dfab8aa013 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -21,8 +21,8 @@ dump-env() { } # dh, mb, op -#_soil_service=mb -_soil_service=dh +_soil_service=mb +#_soil_service=dh case $_soil_service in dh) @@ -33,10 +33,10 @@ case $_soil_service in ;; mb) readonly SOIL_USER='oils' - readonly SOIL_HOST='mb.oils.pub' + readonly SOIL_HOST='mb.oilshell.org' # Extra level - readonly SOIL_HOST_DIR=~/www/mb.oils.pub # used on server - readonly SOIL_REMOTE_DIR=www/mb.oils.pub # used on client + readonly SOIL_HOST_DIR=~/www/mb.oilshell.org # used on server + readonly SOIL_REMOTE_DIR=www/mb.oilshell.org # used on client ;; op) readonly SOIL_USER='oils' @@ -63,8 +63,8 @@ html-head() { # collide with styling and so forth soil-html-head() { - local title="$1" - local web_base_url=${2:-'/web'} + local title=$1 + local web_base_url=$2 html-head --title "$title" \ "$web_base_url/base.css?cache=0" "$web_base_url/soil.css?cache=0" diff --git a/soil/web-init.sh b/soil/web-init.sh index 8e98a597a9..fa6069765f 100755 --- a/soil/web-init.sh +++ b/soil/web-init.sh @@ -28,7 +28,7 @@ home-page() { local domain=${1:-$SOIL_HOST} local title="Soil on $domain" - soil-html-head "$title" + soil-html-head "$title" 'uuu/web' cat < @@ -134,20 +134,22 @@ deploy-data() { local user=${1:-$SOIL_USER} local host=${2:-$SOIL_HOST} - local host_dir=$SOIL_REMOTE_DIR/uuu + local host_dir=$SOIL_REMOTE_DIR # TODO: Better to put HTML in www/$host/uuu/github-jobs, etc. ssh $user@$host mkdir -v -p \ - $host_dir/{sourcehut-jobs,github-jobs,status-api/github} \ - $host_dir/web/table + $host_dir/uuu/{sourcehut-jobs,github-jobs,status-api/github} \ + $host_dir/uuu/web/table + # Soil HTML has relative links like ../web/base.css, so we want + # uuu/web/base.css + # # note: duplicating CSS - scp web/{base.css,soil.css,ajax.js} $user@$host:$host_dir/web - scp web/table/*.{js,css} $user@$host:$host_dir/web/table + scp web/{base.css,soil.css,ajax.js} $user@$host:$host_dir/uuu/web + scp web/table/*.{js,css} $user@$host:$host_dir/uuu/web/table home-page "$host" > _tmp/index.html - # Home page goes in the domain root - scp _tmp/index.html $user@$host:$SOIL_REMOTE_DIR/ + scp _tmp/index.html $user@$host:$host_dir/ } soil-web-manifest() { diff --git a/soil/web-worker.sh b/soil/web-worker.sh index f6ebea1a76..ac5e23508a 100755 --- a/soil/web-worker.sh +++ b/soil/web-worker.sh @@ -146,7 +146,7 @@ format-wwz-index() { local job_id=$1 local tsv=${2:-_tmp/soil/INDEX.tsv} - soil-html-head "$job_id.wwz" + soil-html-head "$job_id.wwz" /uuu/web cat < From 3004b060b1157f289010990c6bc7407b06a73bdb Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 28 Aug 2024 17:58:48 -0400 Subject: [PATCH 208/506] [soil] Able to deploy to op.oilshell.org Adjust py2 hack for OpalStack -- needs bash 4.2 fix, sigh Unrelated: [demo] Use . instead of => pp line -> pp test_ --- demo/url-search-params.ysh | 38 +++++++++++++++++--------------------- soil/common.sh | 11 ++++++----- soil/web.sh | 13 +++++++++---- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/demo/url-search-params.ysh b/demo/url-search-params.ysh index 1de91e0201..cb51951799 100755 --- a/demo/url-search-params.ysh +++ b/demo/url-search-params.ysh @@ -35,10 +35,6 @@ # # - Eggex can use multiline /// syntax, though you can use \ for line continuation # - Eggex could use "which" match -# - m=>group('lit') sorta bothers me, it should be -# - m.group('lit') -# - $lit - probably! -# - with vars(m.groupDict()) { ... } # - Alternative to printf -v probably needed, or at least wrap it in the YSH # stdlib # @@ -78,14 +74,14 @@ func unquote (s) { var pos = 0 var parts = [] while (true) { - var m = s => leftMatch(Quoted, pos=pos) + var m = s.leftMatch(Quoted, pos=pos) if (not m) { break } - var lit = m => group('lit') - var plus = m => group('plus') - var two_hex = m => group('two_hex') + var lit = m.group('lit') + var plus = m.group('plus') + var two_hex = m.group('two_hex') var part if (lit) { @@ -102,7 +98,7 @@ func unquote (s) { } call parts->append(part) - setvar pos = m => end(0) + setvar pos = m.end(0) #echo } if (pos !== len(s)) { @@ -174,24 +170,24 @@ func URLSearchParams(s) { var pairs = [] while (true) { - var m = s => leftMatch(Pairs, pos=pos) + var m = s.leftMatch(Pairs, pos=pos) if (not m) { break } - #pp line (m) - #pp line (m => group(0)) - var k = m => group('key') - var v = m => group('value') + #pp test_ (m) + #pp test_ (m => group(0)) + var k = m.group('key') + var v = m.group('value') - #pp line (k) - #pp line (v) + #pp test_ (k) + #pp test_ (v) call pairs->append([unquote(k), unquote(v)]) - setvar pos = m => end(0) - #pp line (pos) + setvar pos = m.end(0) + #pp test_ (pos) - var sep = m => group('sep') + var sep = m.group('sep') if (not sep) { break } @@ -256,11 +252,11 @@ proc test-query() { js-decode-query $s | json read (&js) echo 'JS' - pp line (js) + pp test_ (js) echo 'YSH' var pairs = URLSearchParams(s) - pp line (pairs) + pp test_ (pairs) assert [pairs === js] diff --git a/soil/common.sh b/soil/common.sh index dfab8aa013..8aeb9949e4 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -21,8 +21,9 @@ dump-env() { } # dh, mb, op -_soil_service=mb -#_soil_service=dh +#_soil_service=op +#_soil_service=mb +_soil_service=dh case $_soil_service in dh) @@ -40,9 +41,9 @@ case $_soil_service in ;; op) readonly SOIL_USER='oils' - readonly SOIL_HOST='op.oils.pub' - readonly SOIL_HOST_DIR=~/op.oils.pub # used on server - readonly SOIL_REMOTE_DIR=op.oils.pub # used on client + readonly SOIL_HOST='op.oilshell.org' + readonly SOIL_HOST_DIR=~/op.oilshell.org # used on server + readonly SOIL_REMOTE_DIR=op.oilshell.org # used on client ;; *) echo "Invalid Soil service $_soil_service" >& 2 diff --git a/soil/web.sh b/soil/web.sh index 799239712d..ed55b4a430 100755 --- a/soil/web.sh +++ b/soil/web.sh @@ -24,12 +24,17 @@ soil-web() { # We may be executed by a wwup.cgi on the server, which doesn't have # PATH=~/bin, and the shebang is /usr/bin/env python2 - local -a prefix=() - if test -n "${CONTENT_LENGTH:-}"; then - prefix=( ~/bin/python2 ) + # OpalStack doesn't need this + # Also it still uses bash 4.2 with the empty array bug! + + local py2=~/bin/python2 + local prefix='' + if test -f $py2; then + prefix=$py2 fi - PYTHONPATH=$REPO_ROOT "${prefix[@]}" $REPO_ROOT/soil/web.py "$@" + # Relies on empty elision of $prefix + PYTHONPATH=$REPO_ROOT $prefix $REPO_ROOT/soil/web.py "$@" } # Bug fix for another race: From 7da9c256775f00ccf59b139b861d64a2653095a0 Mon Sep 17 00:00:00 2001 From: Aidan <46799759+PossiblyAShrub@users.noreply.github.com> Date: Sat, 31 Aug 2024 09:44:21 -0600 Subject: [PATCH 209/506] [builtin] Str.split() supports eggex separator (#2051) * Guard against zero-width matches by throwing an error --- builtin/method_str.py | 89 +++++++++++++++++++++++++++++-------- demo/survey-str-api.sh | 63 ++++++++++++++++++++++++++ doc/ref/chap-type-method.md | 18 ++++++-- spec/ysh-methods.test.sh | 49 ++++++++++++++++++-- 4 files changed, 193 insertions(+), 26 deletions(-) diff --git a/builtin/method_str.py b/builtin/method_str.py index 5704410727..4fb75d0e0a 100644 --- a/builtin/method_str.py +++ b/builtin/method_str.py @@ -488,38 +488,89 @@ def __init__(self): def Call(self, rd): # type: (typed_args.Reader) -> value_t """ - s.split(sep, count=-1) + s.split(string_sep, count=-1) + s.split(eggex_sep, count=-1) Count behaves like in replace() in that: - `count` < 0 -> ignore - `count` >= 0 -> there will be at most `count` splits """ string = rd.PosStr() - sep = rd.PosStr() + + string_sep = None # type: str + eggex_sep = None # type: value.Eggex + + sep = rd.PosValue() + with tagswitch(sep) as case: + if case(value_e.Eggex): + eggex_sep_ = cast(value.Eggex, sep) + eggex_sep = eggex_sep_ + + elif case(value_e.Str): + string_sep_ = cast(value.Str, sep) + string_sep = string_sep_.s + + else: + raise error.TypeErr(sep, 'expected separator to be Eggex or Str', + rd.LeftParenToken()) + count = mops.BigTruncate(rd.NamedInt("count", -1)) rd.Done() - if len(sep) == 0: - raise error.Structured(3, "sep must be non-empty", rd.LeftParenToken()) - if len(string) == 0: return value.List([]) - cursor = 0 - chunks = [] # type: List[value_t] - while cursor < len(string) and count != 0: - next = string.find(sep, cursor) - if next == -1: - break + if string_sep is not None: + if len(string_sep) == 0: + raise error.Structured(3, "separator must be non-empty", + rd.LeftParenToken()) - chunks.append(value.Str(string[cursor:next])) - cursor = next + len(sep) - count -= 1 + cursor = 0 + chunks = [] # type: List[value_t] + while cursor < len(string) and count != 0: + next = string.find(string_sep, cursor) + if next == -1: + break + + chunks.append(value.Str(string[cursor:next])) + cursor = next + len(string_sep) + count -= 1 - if cursor == len(string): - # An instance of sep was against the end of the string - chunks.append(value.Str("")) - else: chunks.append(value.Str(string[cursor:])) - return value.List(chunks) + return value.List(chunks) + + if eggex_sep is not None: + if '\0' in string: + raise error.Structured( + 3, "cannot split a string with a NUL byte", + rd.LeftParenToken()) + + regex = regex_translate.AsPosixEre(eggex_sep) + cflags = regex_translate.LibcFlags(eggex_sep.canonical_flags) + + cursor = 0 + chunks = [] + while cursor < len(string) and count != 0: + m = libc.regex_search(regex, cflags, string, 0, cursor) + if m is None: + break + + start = m[0] + end = m[1] + if start == end: + raise error.Structured( + 3, + "eggex separators should never match the empty string", + rd.LeftParenToken()) + + chunks.append(value.Str(string[cursor:start])) + cursor = end + + count -= 1 + + chunks.append(value.Str(string[cursor:])) + + return value.List(chunks) + + raise AssertionError() diff --git a/demo/survey-str-api.sh b/demo/survey-str-api.sh index 681aaf1fc3..fc6017f749 100755 --- a/demo/survey-str-api.sh +++ b/demo/survey-str-api.sh @@ -122,4 +122,67 @@ survey-trim() { nodejs -e 'var s = process.argv[1]; var t = s.trim(); console.log(`[${s}] [${t}]`);' "$str" } +survey-split() { + echo '============== PYTHON' + echo + + python3 << EOF +print('a,b,c'.split(',')) +print('aa'.split('a')) +print('a<>b<>c')) +print('a;b;;c'.split(';')) +print(''.split('foo')) + +import re + +print(re.split(',|;', 'a,b;c')) +print(re.split('.*', 'aa')) +print(re.split('.', 'aa')) +print(re.split('<>|@@', 'a<>b@@cb<>c')) +console.log('a;b;;c'.split(';')) +console.log(''.split('foo')) + +console.log('a,b;c'.split(/,|;/)) +console.log('aa'.split(/.*/)) +console.log('aa'.split(/./)) +console.log('a<>b@@c|@@/)) +console.log('a b cd'.split(/\s*/)) +console.log('a b cd'.split(/\s+/)) +console.log(''.split(/./)) +EOF + + echo + echo '============== YSH' + echo + + bin/ysh << EOF +pp test_ ('a,b,c'.split(',')) +pp test_ ('aa'.split('a')) +pp test_ ('a<>b<>c')) +pp test_ ('a;b;;c'.split(';')) +pp test_ (''.split('foo')) + +pp test_ ('a,b;c'.split(/ ',' | ';' /)) +pp test_ ('aa'.split(/ dot* /)) +pp test_ ('aa'.split(/ dot /)) +pp test_ ('a<>b@@c' | '@@' /)) +pp test_ ('a b cd'.split(/ space* /)) +pp test_ ('a b cd'.split(/ space+ /)) +pp test_ (''.split(/ dot /)) +EOF +} + "$@" diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 55d9a56300..02ca37a9ee 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -261,18 +261,30 @@ The `%start` or `^` metacharacter will only match when `pos` is zero. Split a string by a `Str` separator `sep` into a `List` of chunks. pp ('a;b;;c'.split(';')) # => ["a", "b", "", "c"] - pp ('a<>b<>c')) # => ["a","b","cb<>c')) # => ["a", "b", "c ["🌞", "🌞", "🌞"] +Or split using an `Eggex`. + + pp ('a b cd'.split(/ space+ /)) # => ["a", "b", "cd"] + pp ('a,b;c'.split(/ ',' | ';' /)) # => ["a", "b", "c"] + Optionally, provide a `count` to split on `sep` at most `count` times. A negative `count` will split on all occurrences of `sep`. pp ('a;b;;c'.split(';', count=2)) # => ["a", "b", ";c"] pp ('a;b;;c'.split(';', count=-1)) # => ["a", "b", "", "c"] -Passing an empty `sep` will result in an error: +Passing an empty `sep` will result in an error. + + pp ('abc'.split('')) # => Error: Sep cannot be "" + +Splitting by an `Eggex` has some limitations: - pp test_ ('abc'.split('')) # => Error: Sep cannot be "" +- If a `search()` results in an empty string match, eg. + `'abc'.split(/ space* /)`, then we raise an error to avoid an infinite loop. +- The string to split cannot contain NUL bytes because we use the libc regex + engine. ## List diff --git a/spec/ysh-methods.test.sh b/spec/ysh-methods.test.sh index 61e9d969a0..1744a1bb59 100644 --- a/spec/ysh-methods.test.sh +++ b/spec/ysh-methods.test.sh @@ -382,7 +382,7 @@ pp test_ (en2fr => keys()) (List) ["hello","friend","cat"] ## END -#### Str => split(sep), non-empty sep +#### Str => split(sep), non-empty str sep pp test_ ('a,b,c'.split(',')) pp test_ ('aa'.split('a')) pp test_ ('a<>b<>c')) @@ -396,7 +396,21 @@ pp test_ (''.split('foo')) (List) [] ## END -#### Str => split(sep, count), non-empty sep +#### Str => split(sep), eggex sep +pp test_ ('a,b;c'.split(/ ',' | ';' /)) +pp test_ ('aa'.split(/ dot /)) +pp test_ ('a<>b@@c' | '@@' /)) +pp test_ ('a b cd'.split(/ space+ /)) +pp test_ (''.split(/ dot /)) +## STDOUT: +(List) ["a","b","c"] +(List) ["","",""] +(List) ["a","b","c split(sep, count), non-empty str sep pp test_ ('a,b,c'.split(',', count=-1)) pp test_ ('a,b,c'.split(',', count=-2)) # Any negative count means "ignore count" pp test_ ('aa'.split('a', count=1)) @@ -416,20 +430,47 @@ pp test_ (''.split(',', count=0)) (List) [] ## END +#### Str => split(sep, count), eggex sep +pp test_ ('a,b;c'.split(/ ',' | ';' /, count=-1)) +pp test_ ('aa'.split(/ dot /, count=1)) +pp test_ ('a<>b@@c' | '@@' /, count=50)) +pp test_ ('a b c'.split(/ space+ /, count=0)) +pp test_ (''.split(/ dot /, count=1)) +## STDOUT: +(List) ["a","b","c"] +(List) ["","a"] +(List) ["a","b","c split(), usage errors -try { pp test_ ('abc'.split('')) } # Sep cannot be "" +try { pp test_ ('abc'.split('')) } # Sep cannot be "" +echo status=$[_error.code] +try { pp test_ ('abc'.split()) } # Sep must be present +echo status=$[_error.code] +try { pp test_ (b'\y00a\y01'.split(/ 'a' /)) } # Cannot split by eggex when str has NUL-byte echo status=$[_error.code] -try { pp test_ ('abc'.split()) } # Sep must be present +try { pp test_ (b'abc'.split(/ space* /)) } # Eggex cannot accept empty string +echo status=$[_error.code] +try { pp test_ (b'abc'.split(/ dot* /)) } # But in some cases the input doesn't cause an + # infinite loop, so we actually allow it! echo status=$[_error.code] ## STDOUT: status=3 status=3 +status=3 +status=3 +(List) ["",""] +status=0 ## END #### Str => split(), non-ascii pp test_ ('🌞🌝🌞🌝🌞'.split('🌝')) +pp test_ ('🌞🌝🌞🌝🌞'.split(/ '🌝' /)) ## STDOUT: (List) ["🌞","🌞","🌞"] +(List) ["🌞","🌞","🌞"] ## END #### Dict => values() From 0465b3e1e96c01a4b7d721a24e3e42374b3683ba Mon Sep 17 00:00:00 2001 From: Steven Oliver Date: Sun, 1 Sep 2024 22:07:19 -0400 Subject: [PATCH 210/506] [doc] Fix typo in INSTALL.txt (#2067) --- INSTALL.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.txt b/INSTALL.txt index b5b44779f8..2bfb740df3 100644 --- a/INSTALL.txt +++ b/INSTALL.txt @@ -9,7 +9,7 @@ Quick Start If you haven't already done so, extract the tarball: - tar -x --gz < oil-for-unix-0.23.0.tar.gz + tar -x --gz < oils-for-unix-0.23.0.tar.gz cd oils-for-unix-0.23.0 This is the traditional way to install it: From 5eaf087a737085f643d41715eb7bb7fbfcdfbb5f Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 6 Sep 2024 22:18:55 -0400 Subject: [PATCH 211/506] [soil] Switch to op.oilshell.org Dreamhost is being flaky, e.g. on the latest PR. --- soil/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/soil/common.sh b/soil/common.sh index 8aeb9949e4..c0c57a2ea8 100644 --- a/soil/common.sh +++ b/soil/common.sh @@ -21,9 +21,9 @@ dump-env() { } # dh, mb, op -#_soil_service=op +_soil_service=op #_soil_service=mb -_soil_service=dh +#_soil_service=dh case $_soil_service in dh) From 25d842b0165c5d4821c27709eb25bbca60317043 Mon Sep 17 00:00:00 2001 From: Jason Miller Date: Sat, 7 Sep 2024 17:24:22 -0700 Subject: [PATCH 212/506] [builtin/trap] Remove trap with integer arg, as well as dash arg (#2069) The POSIX specification requires this. This is issue #2055 --- builtin/trap_osh.py | 15 ++++++++++++++- spec/builtin-trap.test.sh | 11 +++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index eb415b34e5..77480f67bf 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -19,6 +19,7 @@ from frontend import reader from mycpp import mylib from mycpp.mylib import iteritems, print_stderr +from mycpp import mops from typing import Dict, List, Optional, TYPE_CHECKING if TYPE_CHECKING: @@ -152,6 +153,15 @@ def ThisProcessHasTraps(self): return len(self.traps) != 0 or len(self.hooks) != 0 +def _IsUnsignedInteger(s): + # type: (str) -> bool + + try: + intval = mops.FromStr(s) + except ValueError: + return False + return not mops.Greater(mops.ZERO, intval) + def _GetSignalNumber(sig_spec): # type: (str) -> int @@ -262,7 +272,10 @@ def Run(self, cmd_val): return 1 # NOTE: sig_spec isn't validated when removing handlers. - if code_str == '-': + # Per POSIX, if the first argument to trap is an unsigned integer + # then reset every condition + # https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/utilities/V3_chap02.html#tag_18_28 + if code_str == '-' or _IsUnsignedInteger(code_str): if sig_key in _HOOK_NAMES: self.trap_state.RemoveUserHook(sig_key) return 0 diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index e416bbbda3..105bda1053 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -288,3 +288,14 @@ on exit status=0 ## END +#### Remove trap with an unsigned integer + +trap 'echo noprint' EXIT +trap 1 EXIT +echo printed + +## STDOUT: +printed +## END + + From 2e38500a01583734d1ca536600ab0c0a2359e2f1 Mon Sep 17 00:00:00 2001 From: Aidan <46799759+PossiblyAShrub@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:26:49 -0600 Subject: [PATCH 213/506] [osh] Fix string -> int conversions in shell arithmetic (#2064) This caused an infinite loop with `grep -e` in autoconf, found by Samuel. https://oilshell.zulipchat.com/#narrow/stream/121539-oil-dev/topic/ArithSub.20evaluation.20bug.20in.20autotools.20configure This is related to #2066. --- frontend/consts.py | 13 +++ frontend/consts_gen.py | 13 ++- osh/sh_expr_eval.py | 209 +++++++++++++++++++++------------------ osh/sh_expr_eval_test.py | 87 ++++++++++++++++ spec/arith.test.sh | 80 +++++++++++++++ spec/bugs.test.sh | 15 +++ 6 files changed, 318 insertions(+), 99 deletions(-) create mode 100755 osh/sh_expr_eval_test.py diff --git a/frontend/consts.py b/frontend/consts.py index bb9e1a4079..b3f90ed8e8 100644 --- a/frontend/consts.py +++ b/frontend/consts.py @@ -352,6 +352,19 @@ def IfsEdge(state, ch): ASSIGN_ARG_RE = '^(' + lexer_def.VAR_NAME_RE + r')((=|\+=)(.*))?$' +# Patterns for validating integer constants in arithmetic substitutions. +# 0xAB -- hex constant +# 042 -- octal constant +# 42 -- decimal constant +# 64#z -- arbitrary base constant + +_ARITH_WS = '[ \t\r\n]*' + +ARITH_INT_HEX_RE = '^' + _ARITH_WS + '0x([0-9A-Fa-f]+)' + _ARITH_WS + '$' +ARITH_INT_OCT_RE = '^' + _ARITH_WS + '0([0-7]+)' + _ARITH_WS + '$' +ARITH_INT_DEC_RE = '^' + _ARITH_WS + '([1-9][0-9]*|0)' + _ARITH_WS + '$' +ARITH_INT_ARB_RE = '^' + _ARITH_WS + '([1-9][0-9]*)#([0-9a-zA-Z@_]+)' + _ARITH_WS + '$' + # Eggex equivalent: # # VarName = / diff --git a/frontend/consts_gen.py b/frontend/consts_gen.py index de468d6530..c17826269d 100755 --- a/frontend/consts_gen.py +++ b/frontend/consts_gen.py @@ -385,6 +385,10 @@ def out(fmt, *args): extern BigStr* ASSIGN_ARG_RE; extern BigStr* TEST_V_RE; +extern BigStr* ARITH_INT_HEX_RE; +extern BigStr* ARITH_INT_OCT_RE; +extern BigStr* ARITH_INT_DEC_RE; +extern BigStr* ARITH_INT_ARB_RE; } // namespace consts @@ -570,7 +574,14 @@ def _CString(s): import json return json.dumps(s) - GLOBAL_STRINGS = ['ASSIGN_ARG_RE', 'TEST_V_RE'] + GLOBAL_STRINGS = [ + 'ASSIGN_ARG_RE', + 'TEST_V_RE', + 'ARITH_INT_HEX_RE', + 'ARITH_INT_OCT_RE', + 'ARITH_INT_DEC_RE', + 'ARITH_INT_ARB_RE', + ] for var_name in GLOBAL_STRINGS: out('GLOBAL_STR(%s, %s);', var_name, _CString(getattr(consts, var_name))) diff --git a/osh/sh_expr_eval.py b/osh/sh_expr_eval.py index b5fb30620e..0938b930ea 100644 --- a/osh/sh_expr_eval.py +++ b/osh/sh_expr_eval.py @@ -55,7 +55,6 @@ from frontend import lexer from frontend import location from frontend import match -from frontend import parse_lib from frontend import reader from mycpp import mops from mycpp import mylib @@ -70,6 +69,7 @@ from typing import Tuple, Optional, cast, TYPE_CHECKING if TYPE_CHECKING: from core import optview + from frontend import parse_lib _ = log @@ -294,6 +294,80 @@ def ParseVarRef(self, ref_str, blame_tok): return bvs_part +def _MaybeParseInt(s, blame_loc): + # type: (str, loc_t) -> Tuple[bool, mops.BigInt] + """ + 0xAB -- hex constant + 042 -- octal constant + 42 -- decimal constant + 64#z -- arbitrary base constant + + Returns the tuple (err, value) where err is true if this string is not an integer literal. + """ + m = util.RegexSearch(consts.ARITH_INT_HEX_RE, s) + if m is not None: + try: + integer = mops.FromStr(m[1], 16) + except ValueError: + e_strict('Invalid hex constant %r' % s, blame_loc) + return (False, integer) + + m = util.RegexSearch(consts.ARITH_INT_OCT_RE, s) + if m is not None: + try: + integer = mops.FromStr(s, 8) + except ValueError: + e_strict('Invalid octal constant %r' % s, blame_loc) + return (False, integer) + + m = util.RegexSearch(consts.ARITH_INT_ARB_RE, s) + if m is not None: + b = m[1] + try: + base = int(b) # machine integer, not BigInt + except ValueError: + # Unreachable per the regex validation above + raise AssertionError() + + if base > 64: + e_strict('Base %d cannot be larger than 64' % base, blame_loc) + if base < 2: + e_strict('Base %d must be larger than 2' % base, blame_loc) + + integer = mops.ZERO + digits = m[2] + for ch in digits: + if IsLower(ch): + digit = ord(ch) - ord('a') + 10 + elif IsUpper(ch): + digit = ord(ch) - ord('A') + 36 + elif ch == '@': # horrible syntax + digit = 62 + elif ch == '_': + digit = 63 + elif ch.isdigit(): + digit = int(ch) + else: + # Unreachable per the regex validation above + raise AssertionError() + + if digit >= base: + e_strict('Digits %r out of range for base %d' % (digits, base), + blame_loc) + + #integer = integer * base + digit + integer = mops.Add(mops.Mul(integer, mops.BigInt(base)), + mops.BigInt(digit)) + return (False, integer) + + m = util.RegexSearch(consts.ARITH_INT_DEC_RE, s) + if m is not None: + # Normal base 10 integer. + return (False, mops.FromStr(m[1])) + + return (True, mops.BigInt(0)) + + class ArithEvaluator(object): """Shared between arith and bool evaluators. @@ -329,114 +403,53 @@ def _StringToBigInt(self, s, blame_loc): Runtime parsing enables silly stuff like $(( $(echo 1)$(echo 2) + 1 )) => 13 - 0xAB -- hex constant - 042 -- octal constant - 42 -- decimal constant - 64#z -- arbitrary base constant - bare word: variable quoted word: string (not done?) """ - if s.startswith('0x'): - try: - integer = mops.FromStr(s, 16) - except ValueError: - e_strict('Invalid hex constant %r' % s, blame_loc) - # TODO: don't truncate - return integer + err, i = _MaybeParseInt(s, blame_loc) + if not err: + return i - if s.startswith('0'): - try: - integer = mops.FromStr(s, 8) - except ValueError: - e_strict('Invalid octal constant %r' % s, blame_loc) - return integer + # Doesn't look like an integer - b, digits = mylib.split_once(s, '#') # see if it has # - if digits is not None: - try: - base = int(b) # machine integer, not BigInt - except ValueError: - e_strict('Invalid base for numeric constant %r' % b, blame_loc) - - integer = mops.ZERO - for ch in digits: - if IsLower(ch): - digit = ord(ch) - ord('a') + 10 - elif IsUpper(ch): - digit = ord(ch) - ord('A') + 36 - elif ch == '@': # horrible syntax - digit = 62 - elif ch == '_': - digit = 63 - elif ch.isdigit(): - digit = int(ch) - else: - e_strict('Invalid digits for numeric constant %r' % digits, - blame_loc) + # note: 'test' and '[' never evaluate recursively + if self.parse_ctx is None: + if len(s.strip()) == 0 or match.IsValidVarName(s): + # x42 could evaluate to 0 + e_strict("Invalid integer constant %r" % s, blame_loc) + else: + # 42x is always fatal! + e_die("Invalid integer constant %r" % s, blame_loc) - if digit >= base: - e_strict( - 'Digits %r out of range for base %d' % (digits, base), - blame_loc) + # Special case so we don't get EOF error + if len(s.strip()) == 0: + return mops.ZERO - #integer = integer * base + digit - integer = mops.Add(mops.Mul(integer, mops.BigInt(base)), - mops.BigInt(digit)) - return integer + # For compatibility: Try to parse it as an expression and evaluate it. + a_parser = self.parse_ctx.MakeArithParser(s) try: - # Normal base 10 integer. This includes negative numbers like '-42'. - integer = mops.FromStr(s) - except ValueError: - # doesn't look like an integer - - # note: 'test' and '[' never evaluate recursively - if self.parse_ctx: - arena = self.parse_ctx.arena - - # Special case so we don't get EOF error - if len(s.strip()) == 0: - return mops.ZERO - - # For compatibility: Try to parse it as an expression and evaluate it. - a_parser = self.parse_ctx.MakeArithParser(s) - - # TODO: Fill in the variable name - with alloc.ctx_SourceCode(arena, - source.Variable(None, blame_loc)): - try: - node2 = a_parser.Parse() # may raise error.Parse - except error.Parse as e: - self.errfmt.PrettyPrintError(e) - e_die('Parse error in recursive arithmetic', - e.location) - - # Prevent infinite recursion of $(( 1x )) -- it's a word that evaluates - # to itself, and you don't want to reparse it as a word. - if node2.tag() == arith_expr_e.Word: - e_die("Invalid integer constant %r" % s, blame_loc) - - if self.exec_opts.eval_unsafe_arith(): - integer = self.EvalToBigInt(node2) - else: - # BoolEvaluator doesn't have parse_ctx or mutable_opts - assert self.mutable_opts is not None + node2 = a_parser.Parse() # may raise error.Parse + except error.Parse as e: + self.errfmt.PrettyPrintError(e) + e_die('Parse error in recursive arithmetic', e.location) - # We don't need to flip _allow_process_sub, because they can't be - # parsed. See spec/bugs.test.sh. - with state.ctx_Option(self.mutable_opts, - [option_i._allow_command_sub], - False): - integer = self.EvalToBigInt(node2) + # Prevent infinite recursion of $(( 1x )) -- it's a word that evaluates + # to itself, and you don't want to reparse it as a word. + if node2.tag() == arith_expr_e.Word: + e_die("Invalid integer constant %r" % s, blame_loc) - else: - if len(s.strip()) == 0 or match.IsValidVarName(s): - # x42 could evaluate to 0 - e_strict("Invalid integer constant %r" % s, blame_loc) - else: - # 42x is always fatal! - e_die("Invalid integer constant %r" % s, blame_loc) + if self.exec_opts.eval_unsafe_arith(): + integer = self.EvalToBigInt(node2) + else: + # BoolEvaluator doesn't have parse_ctx or mutable_opts + assert self.mutable_opts is not None + + # We don't need to flip _allow_process_sub, because they can't be + # parsed. See spec/bugs.test.sh. + with state.ctx_Option(self.mutable_opts, + [option_i._allow_command_sub], False): + integer = self.EvalToBigInt(node2) return integer diff --git a/osh/sh_expr_eval_test.py b/osh/sh_expr_eval_test.py new file mode 100755 index 0000000000..e3613825b9 --- /dev/null +++ b/osh/sh_expr_eval_test.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import unittest + +from _devbuild.gen.syntax_asdl import loc +from core import error +from mycpp import mops +from osh import sh_expr_eval + + +class ParsingTest(unittest.TestCase): + + def checkCases(self, cases): + for s, expected in cases: + try: + err, actual = sh_expr_eval._MaybeParseInt(s, loc.Missing) + except error.Strict: + err = True + + if err: + actual = None + + #print(expected and expected.i, actual and actual.i) + self.assertEqual(expected, actual) + + def testDecimalConst(self): + CASES = [ + ('0', mops.BigInt(0)), + ('42042', mops.BigInt(42042)), + (' 2 ', mops.BigInt(2)), + (' 2\t', mops.BigInt(2)), + ('\r\n2\r\n', mops.BigInt(2)), + ('1F', None), + ('011', mops.BigInt(9)), # Parsed as an octal + ('1_1', None), + ('1 1', None), + ] + self.checkCases(CASES) + + def testOctalConst(self): + CASES = [ + ('0777', mops.BigInt(511)), + ('00012', mops.BigInt(10)), + (' 010\t', mops.BigInt(8)), + ('\n010\r\n', mops.BigInt(8)), + ('019', None), + ('0_9', None), + ('0 9', None), + ('0F0', None), + ] + self.checkCases(CASES) + + def testHexConst(self): + CASES = [ + ('0xFF', mops.BigInt(255)), + ('0xff', mops.BigInt(255)), + ('0x0010', mops.BigInt(16)), + (' 0x1A ', mops.BigInt(26)), + ('\t0x1A\r\n', mops.BigInt(26)), + ('FF', None), + ('0xG', None), + ('0x1_0', None), + ('0x1 0', None), + ('0X12', None), + ] + self.checkCases(CASES) + + def testArbitraryBaseConst(self): + CASES = [ + ('2#0110', mops.BigInt(6)), + ('8#777', mops.BigInt(511)), + ('16#ff', mops.BigInt(255)), + (' 16#ff\r ', mops.BigInt(255)), + ('\t16#ff\n', mops.BigInt(255)), + ('64#123abcABC@_', mops.BigInt(1189839476434038719)), + ('16#FF', None), # F != f, so F is out of range of the base + ('010#42', None), # Base cannot start with 0 + ('65#1', None), # Base too large + ('0#1', None), # Base too small + ('1#1', None), # Base too small + ] + self.checkCases(CASES) + + +if __name__ == '__main__': + unittest.main() diff --git a/spec/arith.test.sh b/spec/arith.test.sh index b2aa41b839..40dc21f215 100644 --- a/spec/arith.test.sh +++ b/spec/arith.test.sh @@ -96,6 +96,86 @@ should not get here ## END ## N-I bash/mksh/zsh status: 0 +#### Integer constant parsing +echo $(( 0x12A )) +echo $(( 0x0A )) +echo $(( 0777 )) +echo $(( 0010 )) +echo $(( 24#ag7 )) +## STDOUT: +298 +10 +511 +8 +6151 +## END + +## N-I dash status: 2 +## N-I dash STDOUT: +298 +10 +511 +8 +## END + +## BUG zsh STDOUT: +298 +10 +777 +10 +6151 +## END + +## BUG mksh STDOUT: +298 +10 +777 +10 +6151 +## END + +#### Integer constant validation +check() { + $SH -c "shopt --set strict_arith; echo $1" + echo status=$? +} + +check '$(( 0x1X ))' +check '$(( 09 ))' +check '$(( 2#A ))' +check '$(( 02#0110 ))' +## STDOUT: +status=1 +status=1 +status=1 +status=1 +## END + +## OK dash STDOUT: +status=2 +status=2 +status=2 +status=2 +## END + +## BUG zsh STDOUT: +status=1 +9 +status=0 +status=1 +6 +status=0 +## END + +## BUG mksh STDOUT: +status=1 +9 +status=0 +status=1 +6 +status=0 +## END + #### Newline in the middle of expression echo $((1 + 2)) diff --git a/spec/bugs.test.sh b/spec/bugs.test.sh index f7eb3e08ab..f9b0851103 100644 --- a/spec/bugs.test.sh +++ b/spec/bugs.test.sh @@ -388,3 +388,18 @@ yes ## N-I dash/ash STDOUT: ## END + +#### autotools as_fn_arith bug in configure + +# Causes 'grep -e' check to infinite loop. +# Reduced from a configure script. + +as_fn_arith() { + as_val=$(( $* )) +} + +as_fn_arith 0 + 1 +echo as_val=$as_val +## STDOUT: +as_val=1 +## END From 74f1c3f1d227918f4f0b5a916a714b46060b1a23 Mon Sep 17 00:00:00 2001 From: Aidan <46799759+PossiblyAShrub@users.noreply.github.com> Date: Mon, 9 Sep 2024 23:05:12 -0600 Subject: [PATCH 214/506] [test/spec] Add failing arith-dynamic cases (#2070) These cases were extracted from the discussion on Zulip (#oil-dev>ArithSub evaluation bug in autotools configure) --- spec/arith-dynamic.test.sh | 95 ++++++++++++++++++++++++++++++++++++++ test/spec.sh | 4 ++ 2 files changed, 99 insertions(+) create mode 100644 spec/arith-dynamic.test.sh diff --git a/spec/arith-dynamic.test.sh b/spec/arith-dynamic.test.sh new file mode 100644 index 0000000000..506cc8d5ea --- /dev/null +++ b/spec/arith-dynamic.test.sh @@ -0,0 +1,95 @@ +## compare_shells: bash dash mksh zsh +## oils_failures_allowed: 3 + +# Various tests for dynamic parsing of arithmetic substitutions. + +#### Double quotes +echo $(( "1 + 2" * 3 )) +echo $(( "1+2" * 3 )) +## STDOUT: +7 +7 +## END + +## N-I dash status: 2 +## N-I dash STDOUT: +## END + +## N-I mksh status: 1 +## N-I mksh STDOUT: +## END + +## N-I zsh status: 1 +## N-I zsh STDOUT: +## END + +#### Single quotes +echo $(( '1' + '2' * 3 )) +echo status=$? + +echo $(( '1 + 2' * 3 )) +echo status=$? +## STDOUT: +status=1 +status=1 +## END + +## N-I dash status: 2 +## N-I dash STDOUT: +## END + +## BUG mksh status: 1 +## BUG mksh STDOUT: +199 +status=0 +## END + +## N-I zsh status: 1 +## N-I zsh STDOUT: +## END + +#### Substitutions +x='1 + 2' +echo $(( $x * 3 )) +echo $(( "$x" * 3 )) +## STDOUT: +7 +7 +## END + +## N-I dash status: 2 +## N-I dash STDOUT: +7 +## END + +## N-I mksh status: 1 +## N-I mksh STDOUT: +7 +## END + +## N-I zsh status: 1 +## N-I zsh STDOUT: +7 +## END + +#### Variable references +x='1' +echo $(( x + 2 * 3 )) +echo status=$? + +# Expression like values are evaluated first (this is unlike double quotes) +x='1 + 2' +echo $(( x * 3 )) +echo status=$? +## STDOUT: +7 +status=0 +9 +status=0 +## END + +## N-I dash status: 2 +## N-I dash STDOUT: +7 +status=0 +## END diff --git a/test/spec.sh b/test/spec.sh index 0a5108244b..c1bd9c99d0 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -302,6 +302,10 @@ arith() { run-file arith "$@" } +arith-dynamic() { + run-file arith-dynamic "$@" +} + command-sub() { sh-spec spec/command-sub.test.sh \ ${REF_SHELLS[@]} $OSH_LIST "$@" From f10d2c5b631ae32272ea28134e6b464371bc4efa Mon Sep 17 00:00:00 2001 From: Aidan <46799759+PossiblyAShrub@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:39:05 -0600 Subject: [PATCH 215/506] [builtin] Fix 2 issues in Str.replace(eggex, mystr) (#2071) * raise error on zero-width match * raise error if string contains NUL * document replace by eggex limitations --- builtin/method_str.py | 12 ++++++++++++ doc/ref/chap-type-method.md | 7 +++++++ spec/ysh-regex-api.test.sh | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/builtin/method_str.py b/builtin/method_str.py index 4fb75d0e0a..d37e33d20b 100644 --- a/builtin/method_str.py +++ b/builtin/method_str.py @@ -405,6 +405,12 @@ def Call(self, rd): return value.Str(result) if eggex_val: + if '\0' in string: + raise error.Structured( + 3, + "cannot replace by eggex on a string with NUL bytes", + rd.LeftParenToken()) + ere = regex_translate.AsPosixEre(eggex_val) cflags = regex_translate.LibcFlags(eggex_val.canonical_flags) @@ -464,6 +470,12 @@ def Call(self, rd): start = indices[0] end = indices[1] + if pos == end: + raise error.Structured( + 3, + "eggex should never match the empty string", + rd.LeftParenToken()) + parts.append(string[pos:start]) # Unmatched substring parts.append(s) # Replacement pos = end # Move to end of match diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 02ca37a9ee..5ce3319e89 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -143,6 +143,13 @@ The following matrix of signatures are supported by `replace()`: s => replace(eggex_val, subst_str) s => replace(eggex_val, subst_expr) +Replacing by an `Eggex` has some limitations: + +- If a `search()` results in an empty string match, eg. + `'abc'.split(/ space* /)`, then we raise an error to avoid an infinite loop. +- The string to replace on cannot contain NUL bytes because we use the libc + regex engine. + ### startsWith() Checks if a string starts with a pattern, returning true if it does or false if diff --git a/spec/ysh-regex-api.test.sh b/spec/ysh-regex-api.test.sh index 334b643d54..f6874db877 100644 --- a/spec/ysh-regex-api.test.sh +++ b/spec/ysh-regex-api.test.sh @@ -827,3 +827,21 @@ write $[mystr => replace(/ ^ d+ ; reg_newline /, ^"[$0]")] [1]-2-3 [4]-5 ## END + +#### Str => replace(Eggex, *), guard against infinite loop +shopt --set ysh:all + +var mystr = 'foo bar baz' +write $[mystr => replace(/ space* /, ' ')] +## status: 3 +## STDOUT: +## END + +#### Str => replace(Eggex, *), str cannot contain NUL bytes +shopt --set ysh:all + +var mystr = b'foo bar baz\y00' +write $[mystr => replace(/ space+ /, ' ')] +## status: 3 +## STDOUT: +## END From 820df4ba5f0a0e26e110d8feb5ba228815581a1e Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 14 Sep 2024 11:01:51 -0400 Subject: [PATCH 216/506] [mycpp] Use in-class default zero initialization for all members This addresses issue #2074 - members of context managers could be uninitialized and rooted, causing a GC crash. TODO: now we should be able to remove the special memset(0) in Alloc. That will be a separate change. --- mycpp/cppgen_pass.py | 4 +- mycpp/demo/target_lang.cc | 52 ++++++++++++++++++++++++ mycpp/examples/test_ctx_pattern.py | 65 ++++++++++++++++++++++++++++++ spec/ysh-bugs.test.sh | 27 +++++++++++++ 4 files changed, 147 insertions(+), 1 deletion(-) create mode 100755 mycpp/examples/test_ctx_pattern.py diff --git a/mycpp/cppgen_pass.py b/mycpp/cppgen_pass.py index c9e397caaa..92b1490867 100644 --- a/mycpp/cppgen_pass.py +++ b/mycpp/cppgen_pass.py @@ -2501,7 +2501,9 @@ def _MemberDecl(self, o, base_class_name): for name in sorted_member_names: _, c_type, _ = self.current_member_vars[name] - self.always_write_ind('%s %s;\n', c_type, name) + # use default zero initialization for all members + # (context managers may be on the stack) + self.always_write_ind('%s %s{};\n', c_type, name) if _IsContextManager(self.current_class_name): # Copy ctx member vars out of this class diff --git a/mycpp/demo/target_lang.cc b/mycpp/demo/target_lang.cc index ed4d02fe4d..8c96ae3d61 100644 --- a/mycpp/demo/target_lang.cc +++ b/mycpp/demo/target_lang.cc @@ -1019,6 +1019,56 @@ TEST asdl_namespace_demo() { PASS(); } +class C1 { + public: + int i_; +}; + +class C2 { + public: + C2() { + } + C2(int i) : i_(i) { + } + int i_ = 42; +}; + +// Demo: we can use {} initialization for all fields +// +// Later, if we turn self.i = i into a initialization list, this will be +// cheaper than memset() in theory +class C3 { + public: + C3() { + } + C3(int i) : i_(i) { + } + int i_{}; + double f_{}; + C2* c2_{}; + C2* uninitialized; +}; + +TEST member_init_demo() { + C1 c1; + // Uninitialized + log("c1.i_ = %d", c1.i_); + + C2 c2; + log("c2.i_ = %d", c2.i_); // from in-class initialization + + C2 cc2(99); + log("cc2.i_ = %d", cc2.i_); // from constructor + + C3 c3; + log("c3.i_ = %d", c3.i_); // in-class + log("c3.f_ = %f", c3.f_); // in-class + log("c3.c2_ = %p", c3.c2_); // in-class + log("c3.uninitialized = %p", c3.uninitialized); // in-class + + PASS(); +} + GREATEST_MAIN_DEFS(); int main(int argc, char** argv) { @@ -1050,6 +1100,8 @@ int main(int argc, char** argv) { RUN_TEST(asdl_namespace_demo); + RUN_TEST(member_init_demo); + GREATEST_MAIN_END(); /* display results */ return 0; } diff --git a/mycpp/examples/test_ctx_pattern.py b/mycpp/examples/test_ctx_pattern.py new file mode 100755 index 0000000000..39fcb556b9 --- /dev/null +++ b/mycpp/examples/test_ctx_pattern.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python2 +""" +test_scoped_resource.py +""" +from __future__ import print_function + +import os +import sys + +from mycpp import mylib +from mycpp.mylib import log +from typing import List, Dict, Optional, Any + + +class ctx_Eval(object): + """ + Based on bug #1986 + """ + + def __init__(self, vars): + # type: (Optional[Dict[str, str]]) -> None + self.vars = vars + if vars is not None: + self.restore = [] # type: List[str] + self.restore.append('x') + + # Collection must be here to trigger bug + mylib.MaybeCollect() + + def __enter__(self): + # type: () -> None + pass + + def __exit__(self, type, value, traceback): + # type: (Any, Any, Any) -> None + if self.vars is not None: + self.restore.pop() + + +def run_tests(): + # type: () -> None + + d = {'x': 'y'} # type: Dict[str, str] + for i in xrange(0, 1000): + #with ctx_Eval(d): + # print('d %d' % i) + + with ctx_Eval(None): + print('none %d' % i) + + # Not enough to trigger bug + # mylib.MaybeCollect() + + +def run_benchmarks(): + # type: () -> None + pass + + +if __name__ == '__main__': + if os.getenv('BENCHMARK'): + log('Benchmarking...') + run_benchmarks() + else: + run_tests() diff --git a/spec/ysh-bugs.test.sh b/spec/ysh-bugs.test.sh index 2230feb1ca..12484e2788 100644 --- a/spec/ysh-bugs.test.sh +++ b/spec/ysh-bugs.test.sh @@ -234,3 +234,30 @@ case (WEIGHT) { ## status: 2 ## STDOUT: ## END + +#### Crash due to incorrect of context manager rooting - issue #1986 + +proc p { + var s = "hi" + for q in (1..50) { + shvar Q="whatever" { + setvar s = "." ++ s + } + } +} + +for i in (1..10) { + p +} + +if false { + echo 'testing for longer' + for i in (1 .. 1000) { + p + } +} + +## STDOUT: +## END + + From 2baca9fabbb13e97abc6c85974314df2a54c11f8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 15 Sep 2024 01:09:24 -0400 Subject: [PATCH 217/506] [prebuilt] Regenerate code with new in-class member initialization Comment about memset(0) --- mycpp/gc_alloc.h | 4 ++- prebuilt/asdl/runtime.mycpp.cc | 2 +- prebuilt/asdl/runtime.mycpp.h | 10 +++---- prebuilt/core/error.mycpp.cc | 4 +-- prebuilt/core/error.mycpp.h | 24 ++++++++-------- prebuilt/frontend/args.mycpp.cc | 26 ++++++++--------- prebuilt/frontend/args.mycpp.h | 50 ++++++++++++++++----------------- 7 files changed, 61 insertions(+), 59 deletions(-) diff --git a/mycpp/gc_alloc.h b/mycpp/gc_alloc.h index 4bedfeb51d..3797f4d03f 100644 --- a/mycpp/gc_alloc.h +++ b/mycpp/gc_alloc.h @@ -137,7 +137,9 @@ T* Alloc(Args&&... args) { #endif #endif void* obj = header->ObjectAddress(); - // mycpp doesn't generated constructors that initialize every field + // TODO: now that mycpp generates code to initialize every field, we should + // get rid of this. I saw a failure in benchmarks/uftrace in Soil though. + // We may need to check the hand-written classes? memset(obj, 0, sizeof(T)); return new (obj) T(std::forward(args)...); } diff --git a/prebuilt/asdl/runtime.mycpp.cc b/prebuilt/asdl/runtime.mycpp.cc index 722fadca59..1bcc3a9010 100644 --- a/prebuilt/asdl/runtime.mycpp.cc +++ b/prebuilt/asdl/runtime.mycpp.cc @@ -122,7 +122,7 @@ class PrettyPrinter { PrettyPrinter(int max_width); bool _Fits(int prefix_len, doc::Group* group, pretty_asdl::Measure* suffix_measure); void PrintDoc(pretty_asdl::MeasuredDoc* document, mylib::BufWriter* buf); - int max_width; + int max_width{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(0, sizeof(PrettyPrinter)); diff --git a/prebuilt/asdl/runtime.mycpp.h b/prebuilt/asdl/runtime.mycpp.h index 60dfd3d46d..f6ae15e6a4 100644 --- a/prebuilt/asdl/runtime.mycpp.h +++ b/prebuilt/asdl/runtime.mycpp.h @@ -37,8 +37,8 @@ hnode::Leaf* NewLeaf(BigStr* s, hnode_asdl::color_t e_color); class TraversalState { public: TraversalState(); - Dict* seen; - Dict* ref_count; + Dict* seen{}; + Dict* ref_count{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(2, sizeof(TraversalState)); @@ -68,8 +68,8 @@ class ColorOutput { void WriteRaw(Tuple2* raw); int NumChars(); Tuple2 GetRaw(); - mylib::Writer* f; - int num_chars; + mylib::Writer* f{}; + int num_chars{}; static constexpr uint32_t field_mask() { return maskbit(offsetof(ColorOutput, f)); @@ -147,7 +147,7 @@ class _PrettyPrinter { bool _PrintWholeArray(List* array, int prefix_len, format::ColorOutput* f, int indent); void _PrintRecord(hnode::Record* node, format::ColorOutput* f, int indent); void PrintNode(hnode_asdl::hnode_t* node, format::ColorOutput* f, int indent); - int max_col; + int max_col{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(0, sizeof(_PrettyPrinter)); diff --git a/prebuilt/core/error.mycpp.cc b/prebuilt/core/error.mycpp.cc index cee7d3cf20..d8db6f89d0 100644 --- a/prebuilt/core/error.mycpp.cc +++ b/prebuilt/core/error.mycpp.cc @@ -36,8 +36,8 @@ hnode::Leaf* NewLeaf(BigStr* s, hnode_asdl::color_t e_color); class TraversalState { public: TraversalState(); - Dict* seen; - Dict* ref_count; + Dict* seen{}; + Dict* ref_count{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(2, sizeof(TraversalState)); diff --git a/prebuilt/core/error.mycpp.h b/prebuilt/core/error.mycpp.h index 584979c783..3e24124a73 100644 --- a/prebuilt/core/error.mycpp.h +++ b/prebuilt/core/error.mycpp.h @@ -44,8 +44,8 @@ class _ErrorWithLocation { _ErrorWithLocation(BigStr* msg, syntax_asdl::loc_t* location); bool HasLocation(); BigStr* UserErrorString(); - syntax_asdl::loc_t* location; - BigStr* msg; + syntax_asdl::loc_t* location{}; + BigStr* msg{}; static constexpr uint32_t field_mask() { return maskbit(offsetof(_ErrorWithLocation, location)) @@ -124,7 +124,7 @@ class FatalRuntime : public ::error::_ErrorWithLocation { FatalRuntime(int exit_status, BigStr* msg, syntax_asdl::loc_t* location); int ExitStatus(); - int exit_status; + int exit_status{}; static constexpr uint32_t field_mask() { return ::error::_ErrorWithLocation::field_mask(); @@ -156,7 +156,7 @@ class ErrExit : public ::error::FatalRuntime { public: ErrExit(int exit_status, BigStr* msg, syntax_asdl::loc_t* location, bool show_code = false); - bool show_code; + bool show_code{}; static constexpr uint32_t field_mask() { return ::error::FatalRuntime::field_mask(); @@ -189,7 +189,7 @@ class Structured : public ::error::FatalRuntime { Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties = nullptr); value::Dict* ToDict(); - Dict* properties; + Dict* properties{}; static constexpr uint32_t field_mask() { return ::error::FatalRuntime::field_mask() @@ -252,7 +252,7 @@ class Runtime { public: Runtime(BigStr* msg); BigStr* UserErrorString(); - BigStr* msg; + BigStr* msg{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(1, sizeof(Runtime)); @@ -266,11 +266,11 @@ class Decode { Decode(BigStr* msg, BigStr* s, int start_pos, int end_pos, int line_num); BigStr* Message(); BigStr* __str__(); - BigStr* msg; - BigStr* s; - int start_pos; - int end_pos; - int line_num; + BigStr* msg{}; + BigStr* s{}; + int start_pos{}; + int end_pos{}; + int line_num{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(2, sizeof(Decode)); @@ -283,7 +283,7 @@ class Encode { public: Encode(BigStr* msg); BigStr* Message(); - BigStr* msg; + BigStr* msg{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(1, sizeof(Encode)); diff --git a/prebuilt/frontend/args.mycpp.cc b/prebuilt/frontend/args.mycpp.cc index e8be119da6..fd9ba6796f 100644 --- a/prebuilt/frontend/args.mycpp.cc +++ b/prebuilt/frontend/args.mycpp.cc @@ -207,7 +207,7 @@ class PrettyPrinter { PrettyPrinter(int max_width); bool _Fits(int prefix_len, doc::Group* group, pretty_asdl::Measure* suffix_measure); void PrintDoc(pretty_asdl::MeasuredDoc* document, mylib::BufWriter* buf); - int max_width; + int max_width{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(0, sizeof(PrettyPrinter)); @@ -244,8 +244,8 @@ class _ErrorWithLocation { _ErrorWithLocation(BigStr* msg, syntax_asdl::loc_t* location); bool HasLocation(); BigStr* UserErrorString(); - syntax_asdl::loc_t* location; - BigStr* msg; + syntax_asdl::loc_t* location{}; + BigStr* msg{}; static constexpr uint32_t field_mask() { return maskbit(offsetof(_ErrorWithLocation, location)) @@ -324,7 +324,7 @@ class FatalRuntime : public ::error::_ErrorWithLocation { FatalRuntime(int exit_status, BigStr* msg, syntax_asdl::loc_t* location); int ExitStatus(); - int exit_status; + int exit_status{}; static constexpr uint32_t field_mask() { return ::error::_ErrorWithLocation::field_mask(); @@ -356,7 +356,7 @@ class ErrExit : public ::error::FatalRuntime { public: ErrExit(int exit_status, BigStr* msg, syntax_asdl::loc_t* location, bool show_code = false); - bool show_code; + bool show_code{}; static constexpr uint32_t field_mask() { return ::error::FatalRuntime::field_mask(); @@ -389,7 +389,7 @@ class Structured : public ::error::FatalRuntime { Structured(int status, BigStr* msg, syntax_asdl::loc_t* location, Dict* properties = nullptr); value::Dict* ToDict(); - Dict* properties; + Dict* properties{}; static constexpr uint32_t field_mask() { return ::error::FatalRuntime::field_mask() @@ -452,7 +452,7 @@ class Runtime { public: Runtime(BigStr* msg); BigStr* UserErrorString(); - BigStr* msg; + BigStr* msg{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(1, sizeof(Runtime)); @@ -466,11 +466,11 @@ class Decode { Decode(BigStr* msg, BigStr* s, int start_pos, int end_pos, int line_num); BigStr* Message(); BigStr* __str__(); - BigStr* msg; - BigStr* s; - int start_pos; - int end_pos; - int line_num; + BigStr* msg{}; + BigStr* s{}; + int start_pos{}; + int end_pos{}; + int line_num{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(2, sizeof(Decode)); @@ -483,7 +483,7 @@ class Encode { public: Encode(BigStr* msg); BigStr* Message(); - BigStr* msg; + BigStr* msg{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(1, sizeof(Encode)); diff --git a/prebuilt/frontend/args.mycpp.h b/prebuilt/frontend/args.mycpp.h index 5db44edaee..4a1fbbab89 100644 --- a/prebuilt/frontend/args.mycpp.h +++ b/prebuilt/frontend/args.mycpp.h @@ -60,8 +60,8 @@ hnode::Leaf* NewLeaf(BigStr* s, hnode_asdl::color_t e_color); class TraversalState { public: TraversalState(); - Dict* seen; - Dict* ref_count; + Dict* seen{}; + Dict* ref_count{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(2, sizeof(TraversalState)); @@ -91,8 +91,8 @@ class ColorOutput { void WriteRaw(Tuple2* raw); int NumChars(); Tuple2 GetRaw(); - mylib::Writer* f; - int num_chars; + mylib::Writer* f{}; + int num_chars{}; static constexpr uint32_t field_mask() { return maskbit(offsetof(ColorOutput, f)); @@ -170,7 +170,7 @@ class _PrettyPrinter { bool _PrintWholeArray(List* array, int prefix_len, format::ColorOutput* f, int indent); void _PrintRecord(hnode::Record* node, format::ColorOutput* f, int indent); void PrintNode(hnode_asdl::hnode_t* node, format::ColorOutput* f, int indent); - int max_col; + int max_col{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(0, sizeof(_PrettyPrinter)); @@ -198,12 +198,12 @@ class _Attributes { _Attributes(Dict* defaults); void SetTrue(BigStr* name); void Set(BigStr* name, value_asdl::value_t* val); - Dict* attrs; - List*>* opt_changes; - List*>* shopt_changes; - List* actions; - bool show_options; - bool saw_double_dash; + Dict* attrs{}; + List*>* opt_changes{}; + List*>* shopt_changes{}; + List* actions{}; + bool show_options{}; + bool saw_double_dash{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(4, sizeof(_Attributes)); @@ -226,10 +226,10 @@ class Reader { void Done(); syntax_asdl::loc_t* _FirstLocation(); syntax_asdl::loc_t* Location(); - List* argv; - List* locs; - int n; - int i; + List* argv{}; + List* locs{}; + int n{}; + int i{}; static constexpr ObjHeader obj_header() { return ObjHeader::ClassScanned(2, sizeof(Reader)); @@ -260,9 +260,9 @@ class _ArgAction : public ::args::_Action { virtual value_asdl::value_t* _Value(BigStr* arg, syntax_asdl::loc_t* location); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); - BigStr* name; - bool quit_parsing_flags; - List* valid; + BigStr* name{}; + bool quit_parsing_flags{}; + List* valid{}; static constexpr uint32_t field_mask() { return ::args::_Action::field_mask() @@ -330,7 +330,7 @@ class SetAttachedBool : public ::args::_Action { SetAttachedBool(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); - BigStr* name; + BigStr* name{}; static constexpr uint32_t field_mask() { return ::args::_Action::field_mask() @@ -349,7 +349,7 @@ class SetToTrue : public ::args::_Action { SetToTrue(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); - BigStr* name; + BigStr* name{}; static constexpr uint32_t field_mask() { return ::args::_Action::field_mask() @@ -368,7 +368,7 @@ class SetOption : public ::args::_Action { SetOption(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); - BigStr* name; + BigStr* name{}; static constexpr uint32_t field_mask() { return ::args::_Action::field_mask() @@ -388,8 +388,8 @@ class SetNamedOption : public ::args::_Action { void ArgName(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); - List* names; - bool shopt; + List* names{}; + bool shopt{}; static constexpr uint32_t field_mask() { return ::args::_Action::field_mask() @@ -408,7 +408,7 @@ class SetAction : public ::args::_Action { SetAction(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); - BigStr* name; + BigStr* name{}; static constexpr uint32_t field_mask() { return ::args::_Action::field_mask() @@ -428,7 +428,7 @@ class SetNamedAction : public ::args::_Action { void ArgName(BigStr* name); virtual bool OnMatch(BigStr* attached_arg, args::Reader* arg_r, args::_Attributes* out); - List* names; + List* names{}; static constexpr uint32_t field_mask() { return ::args::_Action::field_mask() From 5d93f2ced0c8040c48c01b4733f8910c012c8d65 Mon Sep 17 00:00:00 2001 From: meator Date: Sun, 15 Sep 2024 16:42:42 +0200 Subject: [PATCH 218/506] [build] Correct misleading comments in _build/oils.sh (#2075) --- build/ninja_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ninja_main.py b/build/ninja_main.py index aa70911940..73f247bcc5 100755 --- a/build/ninja_main.py +++ b/build/ninja_main.py @@ -105,8 +105,8 @@ def ShellFunctions(cc_sources, f, argv0): # Usage: # _build/oils.sh COMPILER? VARIANT? SKIP_REBUILD? # -# COMPILER: 'cxx' for system compiler, or 'clang' [default cxx] -# VARIANT: 'dbg' or 'opt' [default dbg] +# COMPILER: 'cxx' for system compiler, 'clang' or custom one [default cxx] +# VARIANT: 'dbg' or 'opt' [default opt] # SKIP_REBUILD: if non-empty, checks if the output exists before building . build/ninja-rules-cpp.sh From 32acc082243e91232b0a4f160fe0c5bbb234ef31 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 15 Sep 2024 10:40:19 -0400 Subject: [PATCH 219/506] [refactor] Use match.LooksLikeInteger() more consistently To define a language, we don't want to rely on mops.FromStr() which delegates to strtoll(). [osh/arith] Test for decimal integer first This improves the fib benchmark, but there's still a regression over the old code. --- builtin/process_osh.py | 5 +++-- frontend/args.py | 5 +++-- mycpp/mops.py | 25 +++++++++++++++++++++++++ osh/sh_expr_eval.py | 11 ++++++----- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/builtin/process_osh.py b/builtin/process_osh.py index db8ab6558d..e25c398982 100644 --- a/builtin/process_osh.py +++ b/builtin/process_osh.py @@ -23,6 +23,7 @@ from core import pyutil from core import vm from frontend import flag_util +from frontend import match from frontend import typed_args from mycpp import mops from mycpp import mylib @@ -524,9 +525,9 @@ def Run(self, cmd_val): # In C, RLIM_INFINITY is rlim_t limit = mops.FromC(RLIM_INFINITY) else: - try: + if match.LooksLikeInteger(s): big_int = mops.FromStr(s) - except ValueError as e: + else: raise error.Usage( "expected a number or 'unlimited', got %r" % s, s_loc) diff --git a/frontend/args.py b/frontend/args.py index eac806c231..35d277420a 100644 --- a/frontend/args.py +++ b/frontend/args.py @@ -57,6 +57,7 @@ from _devbuild.gen.value_asdl import (value, value_e, value_t) from core.error import e_usage +from frontend import match from mycpp import mops from mycpp.mylib import log, tagswitch, iteritems @@ -303,9 +304,9 @@ def __init__(self, name): def _Value(self, arg, location): # type: (str, loc_t) -> value_t - try: + if match.LooksLikeInteger(arg): i = mops.FromStr(arg) - except ValueError: + else: e_usage( 'expected integer after %s, got %r' % ('-' + self.name, arg), location) diff --git a/mycpp/mops.py b/mycpp/mops.py index 73687f0c2e..facee427e4 100644 --- a/mycpp/mops.py +++ b/mycpp/mops.py @@ -74,6 +74,31 @@ def ToHexLower(b): return '%x' % b.i +# Notes on FromStr() and recognizing integers +# +# - mops.FromStr should not use exceptions? That is consistent with mops.FromFloat +# - under the hood it uses StringToInt64, which uses strtoll +# - problem: we DO NOT want to rely on strtoll() to define a language, to +# reject user-facing strings - we want to use something like +# match.LooksLikeInteger() usually. This is part of our spec-driven +# philosophy. +# +# - a problem though is if we support 00, because sometimes that is OCTAL +# - int("00") is zero +# - match.LooksLikeInteger returns it + +# uses LooksLikeInteger and then FromStr() +# - YSH int() +# - printf builtin +# - YSH expression conversion + +# Uses only FromStr() +# - j8 - uses its own regex though +# - ulimit +# - trap - NON-NEGATIVE only +# - arg parser + + def FromStr(s, base=10): # type: (str, int) -> BigInt return BigInt(int(s, base)) diff --git a/osh/sh_expr_eval.py b/osh/sh_expr_eval.py index 0938b930ea..4131f030e3 100644 --- a/osh/sh_expr_eval.py +++ b/osh/sh_expr_eval.py @@ -304,6 +304,11 @@ def _MaybeParseInt(s, blame_loc): Returns the tuple (err, value) where err is true if this string is not an integer literal. """ + m = util.RegexSearch(consts.ARITH_INT_DEC_RE, s) + if m is not None: + # Normal base 10 integer. + return (False, mops.FromStr(m[1])) + m = util.RegexSearch(consts.ARITH_INT_HEX_RE, s) if m is not None: try: @@ -360,11 +365,7 @@ def _MaybeParseInt(s, blame_loc): mops.BigInt(digit)) return (False, integer) - m = util.RegexSearch(consts.ARITH_INT_DEC_RE, s) - if m is not None: - # Normal base 10 integer. - return (False, mops.FromStr(m[1])) - + # not an integer return (True, mops.BigInt(0)) From cb69363e6db5badd3429d5bac7815808b6266adc Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 15 Sep 2024 12:44:13 -0400 Subject: [PATCH 220/506] [osh refactor] Make integer parsing more consistent - Add a few tests for 'trap 0 foo' --- builtin/trap_osh.py | 18 +++++++++-------- frontend/consts.py | 4 ++-- mycpp/mops.py | 6 ++++++ osh/sh_expr_eval.py | 19 +++++++++--------- spec/builtin-trap.test.sh | 42 ++++++++++++++++++++++++++++++++++----- 5 files changed, 65 insertions(+), 24 deletions(-) diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index 77480f67bf..c3eb80be64 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -11,14 +11,14 @@ from core import dev from core import error from core import main_loop -from mycpp.mylib import log from core import pyos from core import vm from frontend import flag_util -from frontend import signal_def +from frontend import match from frontend import reader +from frontend import signal_def from mycpp import mylib -from mycpp.mylib import iteritems, print_stderr +from mycpp.mylib import iteritems, print_stderr, log from mycpp import mops from typing import Dict, List, Optional, TYPE_CHECKING @@ -155,12 +155,14 @@ def ThisProcessHasTraps(self): def _IsUnsignedInteger(s): # type: (str) -> bool - - try: - intval = mops.FromStr(s) - except ValueError: + if not match.LooksLikeInteger(s): return False - return not mops.Greater(mops.ZERO, intval) + + # Note: could simplify this by making match.LooksLikeUnsigned() + + # not (0 > s) is (s >= 0) + return not mops.Greater(mops.ZERO, mops.FromStr(s)) + def _GetSignalNumber(sig_spec): # type: (str) -> int diff --git a/frontend/consts.py b/frontend/consts.py index b3f90ed8e8..8796ce0990 100644 --- a/frontend/consts.py +++ b/frontend/consts.py @@ -360,9 +360,9 @@ def IfsEdge(state, ch): _ARITH_WS = '[ \t\r\n]*' -ARITH_INT_HEX_RE = '^' + _ARITH_WS + '0x([0-9A-Fa-f]+)' + _ARITH_WS + '$' -ARITH_INT_OCT_RE = '^' + _ARITH_WS + '0([0-7]+)' + _ARITH_WS + '$' ARITH_INT_DEC_RE = '^' + _ARITH_WS + '([1-9][0-9]*|0)' + _ARITH_WS + '$' +ARITH_INT_OCT_RE = '^' + _ARITH_WS + '0([0-7]+)' + _ARITH_WS + '$' +ARITH_INT_HEX_RE = '^' + _ARITH_WS + '0x([0-9A-Fa-f]+)' + _ARITH_WS + '$' ARITH_INT_ARB_RE = '^' + _ARITH_WS + '([1-9][0-9]*)#([0-9a-zA-Z@_]+)' + _ARITH_WS + '$' # Eggex equivalent: diff --git a/mycpp/mops.py b/mycpp/mops.py index facee427e4..41f6124193 100644 --- a/mycpp/mops.py +++ b/mycpp/mops.py @@ -76,6 +76,12 @@ def ToHexLower(b): # Notes on FromStr() and recognizing integers # +# 3 similar but DIFFERENT cases: +# +# 1. trap ' 42 ' x - unsigned, including 09, but not -1 +# 2. echo $(( x )) - 0123 is octal, but no -0123 because that's separate I think +# 3. int(), j8 - 077 is decimal +# # - mops.FromStr should not use exceptions? That is consistent with mops.FromFloat # - under the hood it uses StringToInt64, which uses strtoll # - problem: we DO NOT want to rely on strtoll() to define a language, to diff --git a/osh/sh_expr_eval.py b/osh/sh_expr_eval.py index 4131f030e3..965fcbfed2 100644 --- a/osh/sh_expr_eval.py +++ b/osh/sh_expr_eval.py @@ -302,12 +302,14 @@ def _MaybeParseInt(s, blame_loc): 42 -- decimal constant 64#z -- arbitrary base constant - Returns the tuple (err, value) where err is true if this string is not an integer literal. + Returns: + (True, value) when the string looks like an integer + (False, ...) when it doesn't """ m = util.RegexSearch(consts.ARITH_INT_DEC_RE, s) if m is not None: # Normal base 10 integer. - return (False, mops.FromStr(m[1])) + return (True, mops.FromStr(m[1])) m = util.RegexSearch(consts.ARITH_INT_HEX_RE, s) if m is not None: @@ -315,7 +317,7 @@ def _MaybeParseInt(s, blame_loc): integer = mops.FromStr(m[1], 16) except ValueError: e_strict('Invalid hex constant %r' % s, blame_loc) - return (False, integer) + return (True, integer) m = util.RegexSearch(consts.ARITH_INT_OCT_RE, s) if m is not None: @@ -323,7 +325,7 @@ def _MaybeParseInt(s, blame_loc): integer = mops.FromStr(s, 8) except ValueError: e_strict('Invalid octal constant %r' % s, blame_loc) - return (False, integer) + return (True, integer) m = util.RegexSearch(consts.ARITH_INT_ARB_RE, s) if m is not None: @@ -363,10 +365,9 @@ def _MaybeParseInt(s, blame_loc): #integer = integer * base + digit integer = mops.Add(mops.Mul(integer, mops.BigInt(base)), mops.BigInt(digit)) - return (False, integer) + return (True, integer) - # not an integer - return (True, mops.BigInt(0)) + return (False, mops.BigInt(0)) # not an integer class ArithEvaluator(object): @@ -407,8 +408,8 @@ def _StringToBigInt(self, s, blame_loc): bare word: variable quoted word: string (not done?) """ - err, i = _MaybeParseInt(s, blame_loc) - if not err: + ok, i = _MaybeParseInt(s, blame_loc) + if ok: return i # Doesn't look like an integer diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index 105bda1053..b5205d64a1 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -290,12 +290,44 @@ status=0 #### Remove trap with an unsigned integer -trap 'echo noprint' EXIT -trap 1 EXIT -echo printed +$SH -e -c ' +trap "echo noprint" EXIT +trap 0 EXIT +echo ok0 +' +echo + +$SH -e -c ' +trap "echo noprint" EXIT +trap " 42 " EXIT +echo ok42space +' +echo + +# corner case: sometimes 07 is treated as octal, but not here +$SH -e -c ' +trap "echo noprint" EXIT +trap 07 EXIT +echo ok07 +' +echo + +$SH -e -c ' +trap "echo trap-exit" EXIT +trap -1 EXIT +echo bad +' +if test $? -ne 0; then + echo failure +fi ## STDOUT: -printed -## END +ok0 + +ok42space +ok07 +trap-exit +failure +## END From aecb3683515801591640e25445da2ddb6061258e Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 15 Sep 2024 15:14:56 -0400 Subject: [PATCH 221/506] [test/unit] Fix tests --- osh/sh_expr_eval_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/osh/sh_expr_eval_test.py b/osh/sh_expr_eval_test.py index e3613825b9..def56aef7b 100755 --- a/osh/sh_expr_eval_test.py +++ b/osh/sh_expr_eval_test.py @@ -14,11 +14,11 @@ class ParsingTest(unittest.TestCase): def checkCases(self, cases): for s, expected in cases: try: - err, actual = sh_expr_eval._MaybeParseInt(s, loc.Missing) + ok, actual = sh_expr_eval._MaybeParseInt(s, loc.Missing) except error.Strict: - err = True + ok = False - if err: + if not ok: actual = None #print(expected and expected.i, actual and actual.i) From 32544bdfabcf94003e358722426f87c67c71600a Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 15 Sep 2024 15:46:18 -0400 Subject: [PATCH 222/506] [osh ysh] Distinguish between YSH numbers with _, and traditional numbers Fix bugs where these would crash: = int('5_2') And = '5_2' < 52 TODO: J8 Notation also needs 1_000_000, for ints and floats --- build/oil-defs/pyext/fastlex.c/methods.def | 3 ++- builtin/func_misc.py | 7 +++--- cpp/frontend_match.cc | 15 ++++++++----- cpp/frontend_match.h | 3 ++- frontend/lexer_def.py | 25 +++++++++++++--------- frontend/lexer_gen.py | 5 ++++- frontend/match.py | 20 ++++++++++++----- frontend/match_test.py | 7 ++++-- pyext/fastlex.c | 18 +++++++++++++--- pyext/fastlex.pyi | 3 ++- spec/ysh-convert.test.sh | 23 +++++++++++--------- spec/ysh-expr-compare.test.sh | 13 +++++++---- ysh/expr_eval.py | 22 ++++++++++--------- 13 files changed, 108 insertions(+), 56 deletions(-) diff --git a/build/oil-defs/pyext/fastlex.c/methods.def b/build/oil-defs/pyext/fastlex.c/methods.def index 9a828c674f..c86038c1e2 100644 --- a/build/oil-defs/pyext/fastlex.c/methods.def +++ b/build/oil-defs/pyext/fastlex.c/methods.def @@ -14,6 +14,7 @@ static PyMethodDef methods[] = { {"IsValidVarName", fastlex_IsValidVarName, METH_VARARGS}, {"ShouldHijack", fastlex_ShouldHijack, METH_VARARGS}, {"LooksLikeInteger", fastlex_LooksLikeInteger, METH_VARARGS}, - {"LooksLikeFloat", fastlex_LooksLikeFloat, METH_VARARGS}, + {"LooksLikeYshInt", fastlex_LooksLikeYshInt, METH_VARARGS}, + {"LooksLikeYshFloat", fastlex_LooksLikeYshFloat, METH_VARARGS}, {0}, }; diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 9273919b1f..c059a1b0af 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -234,11 +234,12 @@ def Call(self, rd): elif case(value_e.Str): val = cast(value.Str, UP_val) - if not match.LooksLikeInteger(val.s): + if not match.LooksLikeYshInt(val.s): raise error.Expr("Can't convert %s to Int" % val.s, rd.BlamePos()) - return value.Int(mops.FromStr(val.s)) + s = val.s.replace('_', '') + return value.Int(mops.FromStr(s)) raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str', rd.BlamePos()) @@ -267,7 +268,7 @@ def Call(self, rd): elif case(value_e.Str): val = cast(value.Str, UP_val) - if not match.LooksLikeFloat(val.s): + if not match.LooksLikeYshFloat(val.s): raise error.Expr('Cannot convert %s to Float' % val.s, rd.BlamePos()) diff --git a/cpp/frontend_match.cc b/cpp/frontend_match.cc index 25ccd17f7e..f73617bb9c 100644 --- a/cpp/frontend_match.cc +++ b/cpp/frontend_match.cc @@ -129,14 +129,19 @@ bool ShouldHijack(BigStr* s) { len(s)); } -bool LooksLikeFloat(BigStr* s) { - return ::LooksLikeFloat(reinterpret_cast(s->data_), - len(s)); -} - bool LooksLikeInteger(BigStr* s) { return ::LooksLikeInteger(reinterpret_cast(s->data_), len(s)); } +bool LooksLikeYshInt(BigStr* s) { + return ::LooksLikeYshInt(reinterpret_cast(s->data_), + len(s)); +} + +bool LooksLikeYshFloat(BigStr* s) { + return ::LooksLikeYshFloat(reinterpret_cast(s->data_), + len(s)); +} + } // namespace match diff --git a/cpp/frontend_match.h b/cpp/frontend_match.h index d797fbb5f4..2c89ef9712 100644 --- a/cpp/frontend_match.h +++ b/cpp/frontend_match.h @@ -78,8 +78,9 @@ Tuple2 MatchJsonStrToken(BigStr* s, int pos); bool IsValidVarName(BigStr* s); bool ShouldHijack(BigStr* s); bool CanOmitQuotes(BigStr* s); -bool LooksLikeFloat(BigStr* s); bool LooksLikeInteger(BigStr* s); +bool LooksLikeYshInt(BigStr* s); +bool LooksLikeYshFloat(BigStr* s); // StringToInt diff --git a/frontend/lexer_def.py b/frontend/lexer_def.py index 4b5ce6437d..fb5d146e90 100644 --- a/frontend/lexer_def.py +++ b/frontend/lexer_def.py @@ -955,23 +955,28 @@ def R(pat, tok_type): _WHITESPACE = r'[ \t\r\n]*' # ASCII whitespace doesn't have legacy \f \v +# Note: we often check match.LooksLikeInteger(s), call mops.FromStr(s), and +# ASSUME it will not throw ValueError +LOOKS_LIKE_INTEGER = _WHITESPACE + '-?[0-9]+' + _WHITESPACE + +# TODO: use for YSH comparison operators > >= < <= +# # Python allows 0 to be written 00 or 0_0_0, which is weird. But let's be # consistent, and avoid '00' turning into a float! -_DECIMAL_INT_RE = r'[0-9](_?[0-9])*' +_YSH_DECIMAL_INT_RE = r'[0-9](_?[0-9])*' -# Used for YSH comparison operators > >= < <= -LOOKS_LIKE_INTEGER = _WHITESPACE + '-?' + _DECIMAL_INT_RE + _WHITESPACE +LOOKS_LIKE_YSH_INT = _WHITESPACE + '-?' + _YSH_DECIMAL_INT_RE + _WHITESPACE -_FLOAT_RE = ( - _DECIMAL_INT_RE + +_YSH_FLOAT_RE = ( + _YSH_DECIMAL_INT_RE + # Unlike Python, exponent can't be like 42e5_000. There's no use because # 1e309 is already inf. Let's keep our code simple. - r'(\.' + _DECIMAL_INT_RE + ')?([eE][+\-]?[0-9]+)?') + r'(\.' + _YSH_DECIMAL_INT_RE + ')?([eE][+\-]?[0-9]+)?') -# Ditto, used for comparison operators +# Ditto, used for YSH comparison operators # Added optional Optional -? # Example: -3_000_000.000_001e12 -LOOKS_LIKE_FLOAT = _WHITESPACE + '-?' + _FLOAT_RE + _WHITESPACE +LOOKS_LIKE_YSH_FLOAT = _WHITESPACE + '-?' + _YSH_FLOAT_RE + _WHITESPACE # Python 3 float literals: @@ -1000,13 +1005,13 @@ def R(pat, tok_type): # octdigit ::= "0"..."7" # hexdigit ::= digit | "a"..."f" | "A"..."F" - R(_DECIMAL_INT_RE, Id.Expr_DecInt), + R(_YSH_DECIMAL_INT_RE, Id.Expr_DecInt), R(r'0[bB](_?[01])+', Id.Expr_BinInt), R(r'0[oO](_?[0-7])+', Id.Expr_OctInt), R(r'0[xX](_?[0-9a-fA-F])+', Id.Expr_HexInt), - R(_FLOAT_RE, Id.Expr_Float), + R(_YSH_FLOAT_RE, Id.Expr_Float), # These can be looked up as keywords separately, so you enforce that they have # space around them? diff --git a/frontend/lexer_gen.py b/frontend/lexer_gen.py index 252d0c8538..60b210b21c 100755 --- a/frontend/lexer_gen.py +++ b/frontend/lexer_gen.py @@ -444,7 +444,10 @@ def main(argv): TranslateRegexToPredicate(lexer_def.SHOULD_HIJACK_RE, 'ShouldHijack') TranslateRegexToPredicate(lexer_def.LOOKS_LIKE_INTEGER, 'LooksLikeInteger') - TranslateRegexToPredicate(lexer_def.LOOKS_LIKE_FLOAT, 'LooksLikeFloat') + TranslateRegexToPredicate(lexer_def.LOOKS_LIKE_YSH_INT, + 'LooksLikeYshInt') + TranslateRegexToPredicate(lexer_def.LOOKS_LIKE_YSH_FLOAT, + 'LooksLikeYshFloat') TranslateBracket('BracketUnary', TEST_UNARY_LOOKUP) TranslateBracket('BracketBinary', TEST_BINARY_LOOKUP) diff --git a/frontend/match.py b/frontend/match.py index 8a4b980215..cef063b02c 100644 --- a/frontend/match.py +++ b/frontend/match.py @@ -165,7 +165,8 @@ def _MatchJsonStrToken_Fast(line, start_pos): IsValidVarName = fastlex.IsValidVarName ShouldHijack = fastlex.ShouldHijack LooksLikeInteger = fastlex.LooksLikeInteger - LooksLikeFloat = fastlex.LooksLikeFloat + LooksLikeYshInt = fastlex.LooksLikeYshInt + LooksLikeYshFloat = fastlex.LooksLikeYshFloat else: OneToken = _MatchOshToken_Slow(lexer_def.LEXER_DEF) ECHO_MATCHER = _MatchTokenSlow(lexer_def.ECHO_E_DEF) @@ -194,19 +195,28 @@ def ShouldHijack(s): # type: (str) -> bool return bool(_SHOULD_HIJACK_RE.match(s)) + # + # Integer/float + # + _LOOKS_LIKE_INTEGER_RE = re.compile(lexer_def.LOOKS_LIKE_INTEGER + '$') # type: ignore def LooksLikeInteger(s): # type: (str) -> bool return bool(_LOOKS_LIKE_INTEGER_RE.match(s)) - _LOOKS_LIKE_FLOAT_RE = re.compile(lexer_def.LOOKS_LIKE_FLOAT + '$') # type: ignore - # yapf: enable + _LOOKS_LIKE_YSH_INT_RE = re.compile(lexer_def.LOOKS_LIKE_YSH_INT + '$') # type: ignore + def LooksLikeYshInt(s): + # type: (str) -> bool + return bool(_LOOKS_LIKE_YSH_INT_RE.match(s)) - def LooksLikeFloat(s): + _LOOKS_LIKE_YSH_FLOAT_RE = re.compile(lexer_def.LOOKS_LIKE_YSH_FLOAT + '$') # type: ignore + + def LooksLikeYshFloat(s): # type: (str) -> bool - return bool(_LOOKS_LIKE_FLOAT_RE.match(s)) + return bool(_LOOKS_LIKE_YSH_FLOAT_RE.match(s)) + # yapf: enable class SimpleLexer(object): diff --git a/frontend/match_test.py b/frontend/match_test.py index 207e4142b0..93c6b3c8f8 100755 --- a/frontend/match_test.py +++ b/frontend/match_test.py @@ -93,6 +93,9 @@ def testJ8StrLexer(self): _PrintTokens(lex) def testLooksLike(self): + self.assertEqual(False, match.LooksLikeInteger(' 3_000 ')) + self.assertEqual(False, match.LooksLikeInteger(' ')) + INTS = [ (False, ''), (False, 'foo'), @@ -110,7 +113,7 @@ def testLooksLike(self): ] for expected, s in INTS + MORE_INTS: - self.assertEqual(expected, match.LooksLikeInteger(s)) + self.assertEqual(expected, match.LooksLikeYshInt(s)) FLOATS = [ (True, '3.0'), @@ -121,7 +124,7 @@ def testLooksLike(self): ] for expected, s in INTS + FLOATS: # Use BOTH test cases - self.assertEqual(expected, match.LooksLikeFloat(s), s) + self.assertEqual(expected, match.LooksLikeYshFloat(s), s) if __name__ == '__main__': diff --git a/pyext/fastlex.c b/pyext/fastlex.c index c727d80889..68394e6aa7 100644 --- a/pyext/fastlex.c +++ b/pyext/fastlex.c @@ -302,14 +302,25 @@ fastlex_LooksLikeInteger(PyObject *self, PyObject *args) { } static PyObject * -fastlex_LooksLikeFloat(PyObject *self, PyObject *args) { +fastlex_LooksLikeYshInt(PyObject *self, PyObject *args) { unsigned char *name; int len; if (!PyArg_ParseTuple(args, "s#", &name, &len)) { return NULL; } - return PyBool_FromLong(LooksLikeFloat(name, len)); + return PyBool_FromLong(LooksLikeYshInt(name, len)); +} + +static PyObject * +fastlex_LooksLikeYshFloat(PyObject *self, PyObject *args) { + unsigned char *name; + int len; + + if (!PyArg_ParseTuple(args, "s#", &name, &len)) { + return NULL; + } + return PyBool_FromLong(LooksLikeYshFloat(name, len)); } #ifdef OVM_MAIN @@ -341,7 +352,8 @@ static PyMethodDef methods[] = { // Should we hijack this shebang line? {"ShouldHijack", fastlex_ShouldHijack, METH_VARARGS, ""}, {"LooksLikeInteger", fastlex_LooksLikeInteger, METH_VARARGS, ""}, - {"LooksLikeFloat", fastlex_LooksLikeFloat, METH_VARARGS, ""}, + {"LooksLikeYshInt", fastlex_LooksLikeYshInt, METH_VARARGS, ""}, + {"LooksLikeYshFloat", fastlex_LooksLikeYshFloat, METH_VARARGS, ""}, {NULL, NULL}, }; #endif diff --git a/pyext/fastlex.pyi b/pyext/fastlex.pyi index 0d512b3c5c..fc788cb4f0 100644 --- a/pyext/fastlex.pyi +++ b/pyext/fastlex.pyi @@ -3,7 +3,8 @@ from typing import Tuple def IsValidVarName(s: str) -> bool: ... def ShouldHijack(s: str) -> bool: ... def LooksLikeInteger(s: str) -> bool: ... -def LooksLikeFloat(s: str) -> bool: ... +def LooksLikeYshInt(s: str) -> bool: ... +def LooksLikeYshFloat(s: str) -> bool: ... def MatchOshToken(lex_mode_enum_id: int, line: str, start_pos: int) -> Tuple[int, int]: ... def MatchPS1Token(line: str, start_pos: int) -> Tuple[int, int]: ... diff --git a/spec/ysh-convert.test.sh b/spec/ysh-convert.test.sh index 6e42bf2cee..f07051114d 100644 --- a/spec/ysh-convert.test.sh +++ b/spec/ysh-convert.test.sh @@ -1,5 +1,3 @@ - - #### bool() conversion echo "$[bool(1234)]" echo "$[bool(0)]" @@ -47,17 +45,22 @@ echo "$[int(1.234)]" ## END #### int() more -var a = int("3") -var b = int("-35") -write $a $b +pp test_ (int("3")) +pp test_ (int("-35")) +pp test_ (int('5_6')) -var c = int("bad") -echo 'should not get here' +shopt -s ysh:upgrade + +try { + var c = int("bad") +} +echo code=$[_error.code] -## status: 3 ## STDOUT: -3 --35 +(Int) 3 +(Int) -35 +(Int) 56 +code=3 ## END #### float() conversion diff --git a/spec/ysh-expr-compare.test.sh b/spec/ysh-expr-compare.test.sh index a33a8fe38f..61e3df954c 100644 --- a/spec/ysh-expr-compare.test.sh +++ b/spec/ysh-expr-compare.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 0 +## oils_failures_allowed: 1 #### Exact equality with === and !== shopt -s ysh:all @@ -259,10 +259,15 @@ if (2 < '1') { #### Invalid String is an error shopt -s oil:upgrade -if ('3' < 'bar') { - echo no +try { + = '3' < 'bar' } -echo 'should not get here' +echo code=$[_error.code] + +try { + = '3' < '123_4' +} +echo code=$[_error.code] ## status: 3 ## STDOUT: diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 1fbd8cde47..79b30e4877 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -97,9 +97,9 @@ def _ConvertToInt(val, msg, blame_loc): elif case(value_e.Str): val = cast(value.Str, UP_val) - if match.LooksLikeInteger(val.s): - # TODO: Handle ValueError - return mops.FromStr(val.s) + if match.LooksLikeYshInt(val.s): + s = val.s.replace('_', '') + return mops.FromStr(s) raise error.TypeErr(val, msg, blame_loc) @@ -118,12 +118,14 @@ def _ConvertToNumber(val): elif case(value_e.Str): val = cast(value.Str, UP_val) - if match.LooksLikeInteger(val.s): - # TODO: Handle ValueError - return coerced_e.Int, mops.FromStr(val.s), -1.0 - if match.LooksLikeFloat(val.s): - return coerced_e.Float, mops.MINUS_ONE, float(val.s) + if match.LooksLikeYshInt(val.s): + s = val.s.replace('_', '') + return coerced_e.Int, mops.FromStr(s), -1.0 + + if match.LooksLikeYshFloat(val.s): + s = val.s.replace('_', '') + return coerced_e.Float, mops.MINUS_ONE, float(s) return coerced_e.Neither, mops.MINUS_ONE, -1.0 @@ -1046,8 +1048,8 @@ def _EvalRArrow(self, node, val): # - found in the properties, not in the prototype chain (not # sure if this error is common.) raise error.Expr( - "Mutating method %r not found on Obj prototype chain" % mut_name, - node.attr) + "Mutating method %r not found on Obj prototype chain" % + mut_name, node.attr) else: # Look up methods on builtin types # TODO: These should also be called M/append, M/erase, etc. From ba2a4c1f8e93155dd1a9deeabbe05abb701ec941 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 15 Sep 2024 20:37:24 -0400 Subject: [PATCH 223/506] [osh] Speed up shell number lexing by using an re2c lexer The libc engine is too slow! It slows down the Fibonacci benchmark A LOT. This passes all the new tests in osh/sh_expr_eval_test.py --- build/oil-defs/pyext/fastlex.c/methods.def | 1 + cpp/frontend_match.cc | 8 +++ cpp/frontend_match.h | 1 + frontend/consts.py | 13 ----- frontend/consts_gen.py | 8 --- frontend/id_kind_def.py | 2 + frontend/lexer_def.py | 13 ++++- frontend/lexer_gen.py | 1 + frontend/match.py | 8 +++ osh/sh_expr_eval.py | 64 ++++++++++++---------- osh/sh_expr_eval_test.py | 20 ++++++- pyext/fastlex.c | 26 +++++++++ pyext/fastlex.pyi | 1 + 13 files changed, 110 insertions(+), 56 deletions(-) diff --git a/build/oil-defs/pyext/fastlex.c/methods.def b/build/oil-defs/pyext/fastlex.c/methods.def index c86038c1e2..7e041130e4 100644 --- a/build/oil-defs/pyext/fastlex.c/methods.def +++ b/build/oil-defs/pyext/fastlex.c/methods.def @@ -11,6 +11,7 @@ static PyMethodDef methods[] = { {"MatchJ8LinesToken", fastlex_MatchJ8LinesToken, METH_VARARGS}, {"MatchJ8StrToken", fastlex_MatchJ8StrToken, METH_VARARGS}, {"MatchJsonStrToken", fastlex_MatchJsonStrToken, METH_VARARGS}, + {"MatchShNumberToken", fastlex_MatchShNumberToken, METH_VARARGS}, {"IsValidVarName", fastlex_IsValidVarName, METH_VARARGS}, {"ShouldHijack", fastlex_ShouldHijack, METH_VARARGS}, {"LooksLikeInteger", fastlex_LooksLikeInteger, METH_VARARGS}, diff --git a/cpp/frontend_match.cc b/cpp/frontend_match.cc index f73617bb9c..3a83dbe2b4 100644 --- a/cpp/frontend_match.cc +++ b/cpp/frontend_match.cc @@ -119,6 +119,14 @@ Tuple2 MatchJsonStrToken(BigStr* s, int pos) { return Tuple2(static_cast(id), end_pos); } +Tuple2 MatchShNumberToken(BigStr* s, int pos) { + int id; + int end_pos; + ::MatchShNumberToken(reinterpret_cast(s->data_), len(s), + pos, &id, &end_pos); + return Tuple2(static_cast(id), end_pos); +} + bool IsValidVarName(BigStr* s) { return ::IsValidVarName(reinterpret_cast(s->data_), len(s)); diff --git a/cpp/frontend_match.h b/cpp/frontend_match.h index 2c89ef9712..b73fa910a8 100644 --- a/cpp/frontend_match.h +++ b/cpp/frontend_match.h @@ -70,6 +70,7 @@ Tuple2 MatchJ8Token(BigStr* s, int pos); Tuple2 MatchJ8LinesToken(BigStr* s, int pos); Tuple2 MatchJ8StrToken(BigStr* s, int pos); Tuple2 MatchJsonStrToken(BigStr* s, int pos); +Tuple2 MatchShNumberToken(BigStr* s, int pos); // // Other Matching Functions diff --git a/frontend/consts.py b/frontend/consts.py index 8796ce0990..bb9e1a4079 100644 --- a/frontend/consts.py +++ b/frontend/consts.py @@ -352,19 +352,6 @@ def IfsEdge(state, ch): ASSIGN_ARG_RE = '^(' + lexer_def.VAR_NAME_RE + r')((=|\+=)(.*))?$' -# Patterns for validating integer constants in arithmetic substitutions. -# 0xAB -- hex constant -# 042 -- octal constant -# 42 -- decimal constant -# 64#z -- arbitrary base constant - -_ARITH_WS = '[ \t\r\n]*' - -ARITH_INT_DEC_RE = '^' + _ARITH_WS + '([1-9][0-9]*|0)' + _ARITH_WS + '$' -ARITH_INT_OCT_RE = '^' + _ARITH_WS + '0([0-7]+)' + _ARITH_WS + '$' -ARITH_INT_HEX_RE = '^' + _ARITH_WS + '0x([0-9A-Fa-f]+)' + _ARITH_WS + '$' -ARITH_INT_ARB_RE = '^' + _ARITH_WS + '([1-9][0-9]*)#([0-9a-zA-Z@_]+)' + _ARITH_WS + '$' - # Eggex equivalent: # # VarName = / diff --git a/frontend/consts_gen.py b/frontend/consts_gen.py index c17826269d..28fc048135 100755 --- a/frontend/consts_gen.py +++ b/frontend/consts_gen.py @@ -385,10 +385,6 @@ def out(fmt, *args): extern BigStr* ASSIGN_ARG_RE; extern BigStr* TEST_V_RE; -extern BigStr* ARITH_INT_HEX_RE; -extern BigStr* ARITH_INT_OCT_RE; -extern BigStr* ARITH_INT_DEC_RE; -extern BigStr* ARITH_INT_ARB_RE; } // namespace consts @@ -577,10 +573,6 @@ def _CString(s): GLOBAL_STRINGS = [ 'ASSIGN_ARG_RE', 'TEST_V_RE', - 'ARITH_INT_HEX_RE', - 'ARITH_INT_OCT_RE', - 'ARITH_INT_DEC_RE', - 'ARITH_INT_ARB_RE', ] for var_name in GLOBAL_STRINGS: out('GLOBAL_STR(%s, %s);', var_name, diff --git a/frontend/id_kind_def.py b/frontend/id_kind_def.py index 2380b841d6..21a9a8e523 100755 --- a/frontend/id_kind_def.py +++ b/frontend/id_kind_def.py @@ -707,6 +707,8 @@ def AddKinds(spec): 'Operator', ]) + spec.AddKind('ShNumber', ['Dec', 'Hex', 'Oct', 'BaseN']) + # Shared between [[ and test/[. _UNARY_STR_CHARS = 'zn' # -z -n diff --git a/frontend/lexer_def.py b/frontend/lexer_def.py index fb5d146e90..052770b623 100644 --- a/frontend/lexer_def.py +++ b/frontend/lexer_def.py @@ -679,6 +679,17 @@ def R(pat, tok_type): R(r'[^\\"\x00-\x1F]+', Id.Lit_Chars), ] +_WHITESPACE = r'[ \t\r\n]*' # ASCII whitespace doesn't have legacy \f \v + +SH_NUMBER_DEF = [ + R('0', Id.ShNumber_Dec), # not octal + R(r'[1-9][0-9]*', Id.ShNumber_Dec), + R(r'0[0-7]+', Id.ShNumber_Oct), + R(r'0x[0-9A-Fa-f]+', Id.ShNumber_Hex), + R(r'[1-9][0-9]*#[0-9a-zA-Z@_]+', Id.ShNumber_BaseN), + R(r'[^\0]', Id.Unknown_Tok), # any other char +] + OCTAL3_RE = r'\\[0-7]{1,3}' # https://www.gnu.org/software/bash/manual/html_node/Controlling-the-PromptEvaluator.html#Controlling-the-PromptEvaluator @@ -953,8 +964,6 @@ def R(pat, tok_type): R(r'[ \t\r]+', Id.Ignored_Space), ] -_WHITESPACE = r'[ \t\r\n]*' # ASCII whitespace doesn't have legacy \f \v - # Note: we often check match.LooksLikeInteger(s), call mops.FromStr(s), and # ASSUME it will not throw ValueError LOOKS_LIKE_INTEGER = _WHITESPACE + '-?[0-9]+' + _WHITESPACE diff --git a/frontend/lexer_gen.py b/frontend/lexer_gen.py index 60b210b21c..e5cee5fa51 100755 --- a/frontend/lexer_gen.py +++ b/frontend/lexer_gen.py @@ -439,6 +439,7 @@ def main(argv): TranslateSimpleLexer('MatchJ8LinesToken', lexer_def.J8_LINES_DEF) TranslateSimpleLexer('MatchJ8StrToken', lexer_def.J8_STR_DEF) TranslateSimpleLexer('MatchJsonStrToken', lexer_def.JSON_STR_DEF) + TranslateSimpleLexer('MatchShNumberToken', lexer_def.SH_NUMBER_DEF) TranslateRegexToPredicate(lexer_def.VAR_NAME_RE, 'IsValidVarName') TranslateRegexToPredicate(lexer_def.SHOULD_HIJACK_RE, 'ShouldHijack') diff --git a/frontend/match.py b/frontend/match.py index cef063b02c..3437f8d49f 100644 --- a/frontend/match.py +++ b/frontend/match.py @@ -149,6 +149,12 @@ def _MatchJsonStrToken_Fast(line, start_pos): return tok_type, end_pos +def _MatchShNumberToken_Fast(line, start_pos): + # type: (str, int) -> Tuple[Id_t, int] + tok_type, end_pos = fastlex.MatchShNumberToken(line, start_pos) + return tok_type, end_pos + + if fastlex: OneToken = _MatchOshToken_Fast ECHO_MATCHER = _MatchEchoToken_Fast @@ -161,6 +167,7 @@ def _MatchJsonStrToken_Fast(line, start_pos): MatchJ8LinesToken = _MatchJ8LinesToken_Fast MatchJ8StrToken = _MatchJ8StrToken_Fast MatchJsonStrToken = _MatchJsonStrToken_Fast + MatchShNumberToken = _MatchShNumberToken_Fast IsValidVarName = fastlex.IsValidVarName ShouldHijack = fastlex.ShouldHijack @@ -179,6 +186,7 @@ def _MatchJsonStrToken_Fast(line, start_pos): MatchJ8LinesToken = _MatchTokenSlow(lexer_def.J8_LINES_DEF) MatchJ8StrToken = _MatchTokenSlow(lexer_def.J8_STR_DEF) MatchJsonStrToken = _MatchTokenSlow(lexer_def.JSON_STR_DEF) + MatchShNumberToken = _MatchTokenSlow(lexer_def.SH_NUMBER_DEF) # Used by osh/cmd_parse.py to validate for loop name. Note it must be # anchored on the right. diff --git a/osh/sh_expr_eval.py b/osh/sh_expr_eval.py index 965fcbfed2..f681e09ff3 100644 --- a/osh/sh_expr_eval.py +++ b/osh/sh_expr_eval.py @@ -297,39 +297,39 @@ def ParseVarRef(self, ref_str, blame_tok): def _MaybeParseInt(s, blame_loc): # type: (str, loc_t) -> Tuple[bool, mops.BigInt] """ - 0xAB -- hex constant - 042 -- octal constant - 42 -- decimal constant - 64#z -- arbitrary base constant - Returns: (True, value) when the string looks like an integer (False, ...) when it doesn't + + Integer formats that are recognized: + 0xAB hex + 042 octal + 42 decimal + 64#z arbitrary base """ - m = util.RegexSearch(consts.ARITH_INT_DEC_RE, s) - if m is not None: + id_, pos = match.MatchShNumberToken(s, 0) # use re2c lexer + if pos != len(s): + # trailing data isn't allowed + return (False, mops.BigInt(0)) + + # Do conversions + + if id_ == Id.ShNumber_Dec: # Normal base 10 integer. - return (True, mops.FromStr(m[1])) + return (True, mops.FromStr(s)) - m = util.RegexSearch(consts.ARITH_INT_HEX_RE, s) - if m is not None: - try: - integer = mops.FromStr(m[1], 16) - except ValueError: - e_strict('Invalid hex constant %r' % s, blame_loc) - return (True, integer) + elif id_ == Id.ShNumber_Oct: + # 0123, offset by 1 + return (True, mops.FromStr(s[1:], 8)) - m = util.RegexSearch(consts.ARITH_INT_OCT_RE, s) - if m is not None: - try: - integer = mops.FromStr(s, 8) - except ValueError: - e_strict('Invalid octal constant %r' % s, blame_loc) - return (True, integer) + elif id_ == Id.ShNumber_Hex: + # 0xff, offset by 2 + return (True, mops.FromStr(s[2:], 16)) + + elif id_ == Id.ShNumber_BaseN: + b, digits = mylib.split_once(s, '#') + assert digits is not None, digits # assured by lexer - m = util.RegexSearch(consts.ARITH_INT_ARB_RE, s) - if m is not None: - b = m[1] try: base = int(b) # machine integer, not BigInt except ValueError: @@ -342,7 +342,6 @@ def _MaybeParseInt(s, blame_loc): e_strict('Base %d must be larger than 2' % base, blame_loc) integer = mops.ZERO - digits = m[2] for ch in digits: if IsLower(ch): digit = ord(ch) - ord('a') + 10 @@ -362,12 +361,15 @@ def _MaybeParseInt(s, blame_loc): e_strict('Digits %r out of range for base %d' % (digits, base), blame_loc) - #integer = integer * base + digit + # formula is: + # integer = integer * base + digit integer = mops.Add(mops.Mul(integer, mops.BigInt(base)), mops.BigInt(digit)) return (True, integer) - return (False, mops.BigInt(0)) # not an integer + else: + # Id.Unknown_Tok or Id.Eol_Tok + return (False, mops.BigInt(0)) # not an integer class ArithEvaluator(object): @@ -408,6 +410,8 @@ def _StringToBigInt(self, s, blame_loc): bare word: variable quoted word: string (not done?) """ + s = s.strip() + ok, i = _MaybeParseInt(s, blame_loc) if ok: return i @@ -416,7 +420,7 @@ def _StringToBigInt(self, s, blame_loc): # note: 'test' and '[' never evaluate recursively if self.parse_ctx is None: - if len(s.strip()) == 0 or match.IsValidVarName(s): + if len(s) == 0 or match.IsValidVarName(s): # x42 could evaluate to 0 e_strict("Invalid integer constant %r" % s, blame_loc) else: @@ -424,7 +428,7 @@ def _StringToBigInt(self, s, blame_loc): e_die("Invalid integer constant %r" % s, blame_loc) # Special case so we don't get EOF error - if len(s.strip()) == 0: + if len(s) == 0: return mops.ZERO # For compatibility: Try to parse it as an expression and evaluate it. diff --git a/osh/sh_expr_eval_test.py b/osh/sh_expr_eval_test.py index def56aef7b..0470edb639 100755 --- a/osh/sh_expr_eval_test.py +++ b/osh/sh_expr_eval_test.py @@ -4,25 +4,39 @@ import unittest from _devbuild.gen.syntax_asdl import loc +from _devbuild.gen.id_kind_asdl import Id_str from core import error +from frontend import match from mycpp import mops from osh import sh_expr_eval class ParsingTest(unittest.TestCase): + def testMatchFunction(self): + id_, pos = match.MatchShNumberToken('2#1010', 0) + if 0: + print('id = %r' % id_) + print('id = %r' % Id_str(id_)) + print('pos = %r' % pos) + def checkCases(self, cases): for s, expected in cases: + stripped = s.strip() # also done in caller try: - ok, actual = sh_expr_eval._MaybeParseInt(s, loc.Missing) + ok, actual = sh_expr_eval._MaybeParseInt(stripped, loc.Missing) except error.Strict: ok = False if not ok: actual = None - #print(expected and expected.i, actual and actual.i) - self.assertEqual(expected, actual) + if 0: + print('s %r' % s) + print('expected', expected and expected.i) + print('actual', actual and actual.i) + print() + self.assertEqual(actual, expected) def testDecimalConst(self): CASES = [ diff --git a/pyext/fastlex.c b/pyext/fastlex.c index 68394e6aa7..158b07ea2c 100644 --- a/pyext/fastlex.c +++ b/pyext/fastlex.c @@ -268,6 +268,30 @@ fastlex_MatchJsonStrToken(PyObject *self, PyObject *args) { return Py_BuildValue("(ii)", id, end_pos); } +static PyObject * +fastlex_MatchShNumberToken(PyObject *self, PyObject *args) { + unsigned char* line; + int line_len; + + int start_pos; + if (!PyArg_ParseTuple(args, "s#i", &line, &line_len, &start_pos)) { + return NULL; + } + + // Bounds checking. + if (start_pos > line_len) { + PyErr_Format(PyExc_ValueError, + "Invalid MatchShNumberToken call (start_pos = %d, line_len = %d)", + start_pos, line_len); + return NULL; + } + + int id; + int end_pos; + MatchShNumberToken(line, line_len, start_pos, &id, &end_pos); + return Py_BuildValue("(ii)", id, end_pos); +} + static PyObject * fastlex_IsValidVarName(PyObject *self, PyObject *args) { unsigned char *name; @@ -347,6 +371,8 @@ static PyMethodDef methods[] = { "(line, start_pos) -> (id, end_pos)."}, {"MatchJsonStrToken", fastlex_MatchJsonStrToken, METH_VARARGS, "(line, start_pos) -> (id, end_pos)."}, + {"MatchShNumberToken", fastlex_MatchShNumberToken, METH_VARARGS, + "(line, start_pos) -> (id, end_pos)."}, {"IsValidVarName", fastlex_IsValidVarName, METH_VARARGS, "Is it a valid var name?"}, // Should we hijack this shebang line? diff --git a/pyext/fastlex.pyi b/pyext/fastlex.pyi index fc788cb4f0..09cc4c9af6 100644 --- a/pyext/fastlex.pyi +++ b/pyext/fastlex.pyi @@ -16,5 +16,6 @@ def MatchJ8Token(line: str, start_pos: int) -> Tuple[int, int]: ... def MatchJ8LinesToken(line: str, start_pos: int) -> Tuple[int, int]: ... def MatchJ8StrToken(line: str, start_pos: int) -> Tuple[int, int]: ... def MatchJsonStrToken(line: str, start_pos: int) -> Tuple[int, int]: ... +def MatchShNumberToken(line: str, start_pos: int) -> Tuple[int, int]: ... def MatchOption(s: str) -> int: ... From 5aad12240ee0a86e5a6fd20f1d7c2cd719eca662 Mon Sep 17 00:00:00 2001 From: Andy Chu Date: Mon, 16 Sep 2024 23:25:55 -0400 Subject: [PATCH 224/506] [mycpp/README] Make note of C++17 dev dependency --- doc/ysh-tour.md | 2 +- mycpp/README.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/ysh-tour.md b/doc/ysh-tour.md index 2a658f2c10..09916649a0 100644 --- a/doc/ysh-tour.md +++ b/doc/ysh-tour.md @@ -170,7 +170,7 @@ can't be expressed): echo 'c:\Program Files\' # => c:\Program Files\ If you want C-style backslash **character escapes**, use a J8 string, which is -like JSON, but with single quotes:: +like JSON, but with single quotes: echo u' A is \u{41} \n line two, with backslash \\' # => diff --git a/mycpp/README.md b/mycpp/README.md index e5f79590a4..8639288362 100644 --- a/mycpp/README.md +++ b/mycpp/README.md @@ -47,6 +47,9 @@ dependencies. First install packages: # We need libssl-dev, libffi-dev, zlib1g-dev to bootstrap Python oil$ build/deps.sh install-ubuntu-packages +You'll also need a C++17 compiler for code generated by Souffle datalog, used +by mycpp, although Oils itself only requires C++11. + Then fetch data, like the Python 3.10 tarball and MyPy repo: oil$ build/deps.sh fetch From 55dccaf1c6de6ba3d72914717dec53e2a93e7e47 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 15 Sep 2024 21:44:24 -0400 Subject: [PATCH 225/506] [spec/builtin-trap] Failing test case for #1881 Trap without args This may require re-printing the AST. Or maybe we can just save the entire code string too. We may be able to attach the source.ArgvWord to the trap dict? Unrelated: comment about memset. --- mycpp/gc_alloc.h | 6 +-- spec/builtin-trap.test.sh | 79 +++++++++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/mycpp/gc_alloc.h b/mycpp/gc_alloc.h index 3797f4d03f..2981205e08 100644 --- a/mycpp/gc_alloc.h +++ b/mycpp/gc_alloc.h @@ -137,9 +137,9 @@ T* Alloc(Args&&... args) { #endif #endif void* obj = header->ObjectAddress(); - // TODO: now that mycpp generates code to initialize every field, we should - // get rid of this. I saw a failure in benchmarks/uftrace in Soil though. - // We may need to check the hand-written classes? + // Now that mycpp generates code to initialize every field, we should + // get rid of this. + // TODO: fix uftrace failure, maybe by upgrading, or working around memset(obj, 0, sizeof(T)); return new (obj) T(std::forward(args)...); } diff --git a/spec/builtin-trap.test.sh b/spec/builtin-trap.test.sh index b5205d64a1..5c476ec541 100644 --- a/spec/builtin-trap.test.sh +++ b/spec/builtin-trap.test.sh @@ -1,5 +1,5 @@ ## compare_shells: dash bash mksh ash -## oils_failures_allowed: 0 +## oils_failures_allowed: 1 # builtin-trap.test.sh @@ -11,6 +11,61 @@ ok hi ## END +#### Register invalid trap +trap 'foo' SIGINVALID +## status: 1 + +#### Remove invalid trap +trap - SIGINVALID +## status: 1 + +#### SIGINT and INT are aliases +trap - SIGINT +echo $? +trap - INT +echo $? +## STDOUT: +0 +0 +## END +## N-I dash STDOUT: +1 +0 +## END + +#### trap without args prints traps, like trap -p +case $SH in dash) exit ;; esac + +if false; then + # bash breaks the display across lines + trap "true +false" EXIT +fi + +$SH -c ' + +trap "true" EXIT + +echo status=$? +trap | grep EXIT +echo status=$? +' + +## STDOUT: +status=0 +trap -- 'true' EXIT +status=0 +## END + +## BUG mksh/ash STDOUT: +status=0 +status=1 +## END + +## N-I dash STDOUT: +## END + + #### trap 'echo hi' KILL (regression test, caught by smoosh suite) trap 'echo hi' 9 echo status=$? @@ -37,28 +92,6 @@ status=1 status=0 ## END -#### Register invalid trap -trap 'foo' SIGINVALID -## status: 1 - -#### Remove invalid trap -trap - SIGINVALID -## status: 1 - -#### SIGINT and INT are aliases -trap - SIGINT -echo $? -trap - INT -echo $? -## STDOUT: -0 -0 -## END -## N-I dash STDOUT: -1 -0 -## END - #### Invalid trap invocation trap 'foo' echo status=$? From 0ccf21cda82d1603a4f84291058b261b5f5bd95e Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 19 Sep 2024 21:04:19 -0400 Subject: [PATCH 226/506] [spec/ysh-bugs] Repro for issue #2078, pgen2 limit Also added a unit test. I tried to make this a bit more efficient in terms of allocations (before fixing the bug). Then I realized that PNodeAllocator is a GC object, so it can't allocate a std::vector in the constructor, because it would never get freed. GC objects can't have members that need destruction, like std::vector. --- cpp/NINJA_subgraph.py | 4 ++++ cpp/TEST.sh | 2 ++ cpp/pgen2_test.cc | 30 ++++++++++++++++++++++++++++++ spec/ysh-bugs.test.sh | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+) create mode 100644 cpp/pgen2_test.cc diff --git a/cpp/NINJA_subgraph.py b/cpp/NINJA_subgraph.py index a27874c903..b9b502ac75 100644 --- a/cpp/NINJA_subgraph.py +++ b/cpp/NINJA_subgraph.py @@ -195,6 +195,10 @@ def NinjaGraph(ru): '//frontend/syntax.asdl', ]) + ru.cc_binary('cpp/pgen2_test.cc', + deps=['//cpp/pgen2'], + matrix=ninja_lib.COMPILERS_VARIANTS) + ru.cc_library('//cpp/pylib', srcs=['cpp/pylib.cc'], deps=['//mycpp/runtime']) diff --git a/cpp/TEST.sh b/cpp/TEST.sh index 1be901426a..80312f74c5 100755 --- a/cpp/TEST.sh +++ b/cpp/TEST.sh @@ -65,6 +65,8 @@ unit() { run-one-test cpp/osh_test '' $variant + run-one-test cpp/pgen2_test '' $variant + run-one-test cpp/pylib_test '' $variant run-one-test cpp/stdlib_test '' $variant diff --git a/cpp/pgen2_test.cc b/cpp/pgen2_test.cc new file mode 100644 index 0000000000..65311ef181 --- /dev/null +++ b/cpp/pgen2_test.cc @@ -0,0 +1,30 @@ +#include "cpp/pgen2.h" + +#include "mycpp/runtime.h" +#include "vendor/greatest.h" + +TEST allocator_test() { +#if 0 + pnode::PNodeAllocator p; + for (int i = 0; i < 1000; ++i) { + p.NewPNode(1, nullptr); + } +#endif + + PASS(); +} + +GREATEST_MAIN_DEFS(); + +int main(int argc, char** argv) { + gHeap.Init(); + + GREATEST_MAIN_BEGIN(); + + RUN_TEST(allocator_test); + + gHeap.CleanProcessExit(); + + GREATEST_MAIN_END(); + return 0; +} diff --git a/spec/ysh-bugs.test.sh b/spec/ysh-bugs.test.sh index 12484e2788..2227d038f3 100644 --- a/spec/ysh-bugs.test.sh +++ b/spec/ysh-bugs.test.sh @@ -261,3 +261,35 @@ if false { ## END +#### crash due to arbitrary PNode limit - issue #2078 + +#!/usr/bin/env ysh +var DelegatedCompName = { + "llvm" : "x_project", + "rocprofiler_register" : "x_rocprofiler_register", + "roct_thunk_interface" : "x_roct", + "rocr_runtime" : "x_rocr", + "openmp" : "x_openmp", + "offload" : "x_offload", + "aomp_extras" : "x_extras", + "comgr" : "x_comgr", + "rocminfo" : "x_rocminfo", + "rocsmilib" : "x_rocm_smi_lib", + "amdsmi" : "x_amdsmi", + "flang_legacy" : "x_flang_legacy", + "pgmath" : "x_pgmath", + "flang" : "x_flang", + "flang_runtime" : "x_flang_runtime", + "hipcc" : "x_hipcc", + "hipamd" : "x_hipamd", + "rocm_dbgapi" : "x_rocdbgapi", + "rocgdb" : "x_rocgdb", + "roctracer" : "x_roctracer", + "rocprofiler" : "x_rocprofiler" +} + +echo $[len(DelegatedCompName)] + +## STDOUT: +21 +## END From f61cfd1d33d1dd2996eed26a8b37a22441785286 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 19 Sep 2024 22:49:39 -0400 Subject: [PATCH 227/506] [cpp] Raise arbitrary PNode limit Temporary workaround for issue #2078. The underlying issue is that std::vector invalidates pointers, so we probably have to use a different data structure. Add a repro with large dicts. --- cpp/pgen2.cc | 4 +++- cpp/pgen2_test.cc | 3 +-- test/bugs.sh | 11 +++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cpp/pgen2.cc b/cpp/pgen2.cc index 93a7e48b45..adc1b6615a 100644 --- a/cpp/pgen2.cc +++ b/cpp/pgen2.cc @@ -27,10 +27,12 @@ int PNode::NumChildren() { } PNodeAllocator::PNodeAllocator() : arena_(new std::vector()) { - arena_->reserve(512); + arena_->reserve(4096); } PNode* PNodeAllocator::NewPNode(int typ, syntax_asdl::Token* tok) { + // TODO: Remove arbitrary limit, probably by using something other than + // std::vector, which invalidates pointers on resize CHECK(arena_->size() < arena_->capacity()); arena_->emplace_back(typ, tok, nullptr); return arena_->data() + (arena_->size() - 1); diff --git a/cpp/pgen2_test.cc b/cpp/pgen2_test.cc index 65311ef181..ff8c266f10 100644 --- a/cpp/pgen2_test.cc +++ b/cpp/pgen2_test.cc @@ -4,12 +4,11 @@ #include "vendor/greatest.h" TEST allocator_test() { -#if 0 pnode::PNodeAllocator p; for (int i = 0; i < 1000; ++i) { p.NewPNode(1, nullptr); } -#endif + p.Clear(); PASS(); } diff --git a/test/bugs.sh b/test/bugs.sh index 2612504033..0ea3c7b7de 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -193,4 +193,15 @@ bug-1853() { $sh -c 'trap "echo hi" EXIT; $(which true); echo last' } +bug-2078() { + local n=${1:-150} + + { echo '= {' + for i in $(seq $n); do + echo '"key'$i'": "val"' + done + echo '}' + } | _bin/cxx-asan/ysh +} + "$@" From e042cfbc8b9f98c86cf8afa0b5eb16afab565ce0 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 21 Sep 2024 12:58:08 -0400 Subject: [PATCH 228/506] [cpp/pgen2] Use std::deque for the PNode arena Because unlike std::vector, pointers aren't invalidated on resize. Following the suggest from Julian Brown. This removes the limitation in issue #2078. My own arena, which is linked array, doesn't work! Let's see how much code size this adds, e.g. on metrics/preprocessed. --- cpp/pgen2.cc | 9 +++++---- cpp/pgen2.h | 3 ++- test/bugs.sh | 8 +++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cpp/pgen2.cc b/cpp/pgen2.cc index adc1b6615a..be68ea8cf7 100644 --- a/cpp/pgen2.cc +++ b/cpp/pgen2.cc @@ -26,16 +26,17 @@ int PNode::NumChildren() { return children.size(); } -PNodeAllocator::PNodeAllocator() : arena_(new std::vector()) { - arena_->reserve(4096); +PNodeAllocator::PNodeAllocator() : arena_(new std::deque()) { + //arena_->reserve(4096); } PNode* PNodeAllocator::NewPNode(int typ, syntax_asdl::Token* tok) { // TODO: Remove arbitrary limit, probably by using something other than // std::vector, which invalidates pointers on resize - CHECK(arena_->size() < arena_->capacity()); + //CHECK(arena_->size() < arena_->capacity()); arena_->emplace_back(typ, tok, nullptr); - return arena_->data() + (arena_->size() - 1); + //return arena_->data() + (arena_->size() - 1); + return &arena_->back(); } void PNodeAllocator::Clear() { diff --git a/cpp/pgen2.h b/cpp/pgen2.h index ac46bd0dff..f773dfdbcb 100644 --- a/cpp/pgen2.h +++ b/cpp/pgen2.h @@ -3,6 +3,7 @@ #ifndef CPP_PGEN2_H #define CPP_PGEN2_H +#include #include #include "_gen/frontend/syntax.asdl.h" @@ -77,7 +78,7 @@ class PNodeAllocator { private: // We put this on the heap so we can call its destructor from `Clear()`... - std::vector* arena_; + std::deque* arena_; }; } // namespace pnode diff --git a/test/bugs.sh b/test/bugs.sh index 0ea3c7b7de..3340c281f7 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -194,13 +194,15 @@ bug-1853() { } bug-2078() { - local n=${1:-150} + local n=${1:-160} - { echo '= {' + { echo 'var x = {' for i in $(seq $n); do echo '"key'$i'": "val"' done - echo '}' + echo '} + = x + = len(x)' } | _bin/cxx-asan/ysh } From 88d5a93c73aac864bb3fdb00a36534be628250e8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 21 Sep 2024 13:19:34 -0400 Subject: [PATCH 229/506] [cpp/pgen2 cleanup] Comments and testing --- cpp/pgen2.cc | 9 +++------ cpp/pgen2_test.cc | 13 ++++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cpp/pgen2.cc b/cpp/pgen2.cc index be68ea8cf7..eb9009e259 100644 --- a/cpp/pgen2.cc +++ b/cpp/pgen2.cc @@ -26,17 +26,14 @@ int PNode::NumChildren() { return children.size(); } +// TODO: It would be nicer to reuse the std::deque arena_ throughout the whole +// program. Rather than new/delete for parsing each YSH expression. PNodeAllocator::PNodeAllocator() : arena_(new std::deque()) { - //arena_->reserve(4096); } PNode* PNodeAllocator::NewPNode(int typ, syntax_asdl::Token* tok) { - // TODO: Remove arbitrary limit, probably by using something other than - // std::vector, which invalidates pointers on resize - //CHECK(arena_->size() < arena_->capacity()); arena_->emplace_back(typ, tok, nullptr); - //return arena_->data() + (arena_->size() - 1); - return &arena_->back(); + return &(arena_->back()); } void PNodeAllocator::Clear() { diff --git a/cpp/pgen2_test.cc b/cpp/pgen2_test.cc index ff8c266f10..807cf418ba 100644 --- a/cpp/pgen2_test.cc +++ b/cpp/pgen2_test.cc @@ -4,12 +4,15 @@ #include "vendor/greatest.h" TEST allocator_test() { - pnode::PNodeAllocator p; - for (int i = 0; i < 1000; ++i) { - p.NewPNode(1, nullptr); + for (int i = 0; i < 6000; i += 100) { + pnode::PNodeAllocator p; + log("Testing i = %d\n", i); + for (int j = 0; j < i; ++j) { + p.NewPNode(1, nullptr); + } + // TODO: it woudl be nicer to reuse the std::deque + p.Clear(); } - p.Clear(); - PASS(); } From ebee91222288b98528e84672f84d91a3bd92c81e Mon Sep 17 00:00:00 2001 From: Melvin Walls Date: Mon, 23 Sep 2024 01:59:34 -0400 Subject: [PATCH 230/506] [build, benchmarks] Add mycpp-souffle translator (#2076) This commit adds a new translator called `mycpp-souffle`. It translates files using mycpp with optimizations enabled. You can build the translation examples with souffle optimizations enabled with the same targets you normally would, but with `mycpp` replaced by `mycpp-souffle` in the suffix. For example, you build the intermediate C++ and the `cxx-asan` binary for `mycpp/examples/gc_stack_roots.py` respectively by running the following commands: ninja _gen/mycpp/examples/gc_stack_roots.mycpp-souffle.cc ninja _bin/cxx-asan/mycpp/examples/gc_stack_roots.mycpp-souffle You can also build OSH and YSH with this new variant. Their targets are named `_bin//mycpp-souffle/osh` and `_bin//mycpp-soufle/ysh` respectively. Currently, only the `cxx-opt`, `cxx-asan`, and `cxx-asan+gcalways` variants are enabled for the new translator. The binaries translated with vanilla mycpp are still available at their existing paths, e.g. `_bin/cxx-dbg/osh`. The translated C++ for the entire program with optimizations is named `_gen/bin/oils_for_unix.mycpp-souffle.cc`. --- benchmarks/autoconf.sh | 4 ++ benchmarks/common.sh | 3 + benchmarks/compute.sh | 2 +- benchmarks/gc.sh | 12 ++++ benchmarks/id.sh | 2 + benchmarks/osh-parser.sh | 38 +++++++---- benchmarks/osh-runtime.sh | 14 ++-- benchmarks/report.R | 27 +++++--- benchmarks/report_test.R | 4 ++ bin/NINJA_subgraph.py | 129 +++++++++++++++++++++---------------- build/native.sh | 15 ++++- build/ninja-rules-py.sh | 15 +++-- build/ninja_lib.py | 9 ++- build/ninja_main.py | 40 ++++++++---- devtools/release-native.sh | 5 +- devtools/test-oils.sh | 2 +- mycpp/NINJA_subgraph.py | 40 +++++++++--- mycpp/mycpp_main.py | 1 + mycpp/pass_state.py | 3 + soil/cpp-tarball.sh | 25 +++++-- yaks/NINJA_subgraph.py | 2 +- 21 files changed, 271 insertions(+), 121 deletions(-) diff --git a/benchmarks/autoconf.sh b/benchmarks/autoconf.sh index 5399cd6ac6..7461c36f99 100755 --- a/benchmarks/autoconf.sh +++ b/benchmarks/autoconf.sh @@ -44,6 +44,7 @@ cpython-configure-tasks() { for v in ${variants[@]}; do echo "${v}${TAB}_bin/cxx-$v/osh" done + echo "opt${TAB}_bin/cxx-opt/mycpp-souffle/osh" } cpython-setup() { @@ -145,6 +146,7 @@ shell-tasks() { echo "bash${TAB}bash" echo "dash${TAB}dash" echo "osh${TAB}$REPO_ROOT/_bin/cxx-opt/osh" + echo "osh${TAB}$REPO_ROOT/_bin/cxx-opt/mycpp-souffle/osh" } measure-syscalls() { @@ -524,6 +526,8 @@ fork-tasks() { # Hm this is noisy, but cxx-opt-sh does seem slower echo "osh${TAB}$REPO_ROOT/_bin/cxx-opt/osh" echo "osh${TAB}$REPO_ROOT/_bin/cxx-opt-sh/osh" + echo "osh${TAB}$REPO_ROOT/_bin/cxx-opt/mycpp-souffle/osh" + echo "osh${TAB}$REPO_ROOT/_bin/cxx-opt-sh/mycpp-souffle/osh" } measure-fork() { diff --git a/benchmarks/common.sh b/benchmarks/common.sh index 7f0b369c7f..1b292a21cf 100644 --- a/benchmarks/common.sh +++ b/benchmarks/common.sh @@ -25,11 +25,14 @@ OIL_VERSION=$(head -n 1 oil-version.txt) readonly BENCHMARK_DATA_OILS=$PWD/../benchmark-data/src/oils-for-unix-$OIL_VERSION readonly OSH_CPP_NINJA_BUILD=_bin/cxx-opt/osh +readonly OSH_SOUFFLE_CPP_NINJA_BUILD=_bin/cxx-opt/mycpp-souffle/osh readonly OSH_CPP_SH_BUILD=_bin/cxx-opt-sh/osh +readonly OSH_SOUFFLE_CPP_SH_BUILD=_bin/cxx-opt-sh/mycpp-souffle/osh readonly YSH_CPP_SH_BUILD=_bin/cxx-opt-sh/ysh readonly OSH_CPP_BENCHMARK_DATA=$BENCHMARK_DATA_OILS/$OSH_CPP_SH_BUILD +readonly OSH_SOUFFLE_CPP_BENCHMARK_DATA=$BENCHMARK_DATA_OILS/$OSH_SOUFFLE_CPP_SH_BUILD readonly YSH_CPP_BENCHMARK_DATA=$BENCHMARK_DATA_OILS/$YSH_CPP_SH_BUILD # diff --git a/benchmarks/compute.sh b/benchmarks/compute.sh index 65d751c184..e8efa1d4b0 100755 --- a/benchmarks/compute.sh +++ b/benchmarks/compute.sh @@ -411,7 +411,7 @@ soil-run() { mkdir -p $BASE_DIR # Test the one that's IN TREE, NOT in ../benchmark-data - local -a osh_bin=( $OSH_CPP_NINJA_BUILD _bin/cxx-opt+bumpleak/osh) + local -a osh_bin=( $OSH_CPP_NINJA_BUILD $OSH_SOUFFLE_CPP_NINJA_BUILD _bin/cxx-opt+bumpleak/osh) ninja "${osh_bin[@]}" local single_machine='no-host' diff --git a/benchmarks/gc.sh b/benchmarks/gc.sh index b3628b550d..2be2c57b9a 100755 --- a/benchmarks/gc.sh +++ b/benchmarks/gc.sh @@ -114,9 +114,13 @@ print-tasks() { # these have trivial GC stats "_bin/cxx-opt/osh${TAB}mut+alloc" "_bin/cxx-opt/osh${TAB}mut+alloc+free" + "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc" + "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free" # good GC stats "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc" "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit" + "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc" + "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit" ) if test -n "${TCMALLOC:-}"; then @@ -183,6 +187,11 @@ print-cachegrind-tasks() { "_bin/cxx-opt/osh${TAB}mut+alloc+free" "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc" "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit" + + "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc" + "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free" + "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc" + "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit" ) local id=0 @@ -389,10 +398,13 @@ build-binaries() { soil/cpp-tarball.sh build-like-ninja \ opt{,+bumpleak,+bumproot,+bumpsmall,+nopool} + OILS_TRANSLATOR=mycpp-souffle soil/cpp-tarball.sh build-like-ninja opt + else # Old Ninja build local -a bin=( _bin/cxx-opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}/osh ) + bin+=( _bin/cxx-opt/mycpp-souffle/osh ) if test -n "${TCMALLOC:-}"; then bin+=( _bin/cxx-opt+tcmalloc/osh ) diff --git a/benchmarks/id.sh b/benchmarks/id.sh index 47897b248d..94408ceb38 100755 --- a/benchmarks/id.sh +++ b/benchmarks/id.sh @@ -159,6 +159,8 @@ _shell-id-hash() { # For OSH file=$src/git-commit-hash.txt test -f $file && cat $file + # XXX: Include shell path to help distinguish between versions of OSH + echo $src return 0 } diff --git a/benchmarks/osh-parser.sh b/benchmarks/osh-parser.sh index 44f6a598bf..3980670b2d 100755 --- a/benchmarks/osh-parser.sh +++ b/benchmarks/osh-parser.sh @@ -60,12 +60,19 @@ parser-task() { local times_out="$out_dir/$host.$job_id.times.csv" local shell_name - shell_name=$(basename $sh_path) + case $sh_path in + _bin/*/mycpp-souffle/*) + shell_name=osh-native-souffle + ;; + *) + shell_name=$(basename $sh_path) + ;; + esac # Can't use array because of set -u bug!!! Only fixed in bash 4.4. extra_args='' case "$shell_name" in - osh|oils-for-unix.*) + osh*|oils-for-unix.*) extra_args='--ast-format none' ;; esac @@ -106,7 +113,14 @@ cachegrind-task() { mkdir -p $out_dir/$cachegrind_out_dir local shell_name - shell_name=$(basename $sh_path) + case $sh_path in + _bin/*/mycpp-souffle/*) + shell_name=osh-native-souffle + ;; + *) + shell_name=$(basename $sh_path) + ;; + esac local script_name script_name=$(basename $script_path) @@ -117,7 +131,7 @@ cachegrind-task() { # Can't use array because of set -u bug!!! Only fixed in bash 4.4. extra_args='' case "$shell_name" in - osh|oils-for-unix.*) + osh*|oils-for-unix.*) extra_args="--ast-format none" ;; esac @@ -202,7 +216,8 @@ measure() { local provenance=$1 local host_job_id=$2 local out_dir=${3:-$BASE_DIR/raw} - local osh_cpp=${4:-$OSH_CPP_BENCHMARK_DATA} + shift 3 + local -a osh_cpp=( "${@:-$OSH_CPP_BENCHMARK_DATA}" ) local times_out="$out_dir/$host_job_id.times.csv" local lines_out="$out_dir/$host_job_id.lines.csv" @@ -222,7 +237,7 @@ measure() { > $times_out local tasks=$BASE_DIR/tasks.txt - print-tasks $provenance "${SHELLS[@]}" $osh_cpp > $tasks + print-tasks $provenance "${SHELLS[@]}" "${osh_cpp[@]}" > $tasks # Run them all cat $tasks | xargs -n $NUM_TASK_COLS -- $0 parser-task $out_dir @@ -232,7 +247,8 @@ measure-cachegrind() { local provenance=$1 local host_job_id=$2 local out_dir=${3:-$BASE_DIR/raw} - local osh_cpp=${4:-$OSH_CPP_BENCHMARK_DATA} + shift 3 + local -a osh_cpp=( "${@:-$OSH_CPP_BENCHMARK_DATA}" ) local cachegrind_tsv="$out_dir/$host_job_id.cachegrind.tsv" local lines_out="$out_dir/$host_job_id.lines.tsv" @@ -257,7 +273,7 @@ measure-cachegrind() { # zsh weirdly forks during zsh -n, which complicates our cachegrind # measurement. So just ignore it. (This can be seen with # strace -e fork -f -- zsh -n $file) - print-tasks $provenance bash dash mksh $osh_cpp > $ctasks + print-tasks $provenance bash dash mksh "${osh_cpp[@]}" > $ctasks cat $ctasks | xargs -n $NUM_TASK_COLS -- $0 cachegrind-task $out_dir } @@ -474,7 +490,7 @@ soil-run() { rm -r -f $BASE_DIR mkdir -p $BASE_DIR - local -a osh_bin=( $OSH_CPP_NINJA_BUILD ) + local -a osh_bin=( $OSH_CPP_NINJA_BUILD $OSH_SOUFFLE_CPP_NINJA_BUILD ) ninja "${osh_bin[@]}" local single_machine='no-host' @@ -490,9 +506,9 @@ soil-run() { local provenance=_tmp/provenance.txt local host_job_id="$single_machine.$job_id" - measure $provenance $host_job_id '' $OSH_CPP_NINJA_BUILD + measure $provenance $host_job_id '' $OSH_CPP_NINJA_BUILD $OSH_SOUFFLE_CPP_NINJA_BUILD - measure-cachegrind $provenance $host_job_id '' $OSH_CPP_NINJA_BUILD + measure-cachegrind $provenance $host_job_id '' $OSH_CPP_NINJA_BUILD $OSH_SOUFFLE_CPP_NINJA_BUILD # TODO: R can use this TSV file cp -v _tmp/provenance.tsv $BASE_DIR/stage1/provenance.tsv diff --git a/benchmarks/osh-runtime.sh b/benchmarks/osh-runtime.sh index 9af40678c1..9686c8b27c 100755 --- a/benchmarks/osh-runtime.sh +++ b/benchmarks/osh-runtime.sh @@ -213,7 +213,8 @@ print-workloads() { print-tasks() { local host_name=$1 - local osh_native=$2 + shift 1 + local -a osh_native=( "$@" ) if test -n "${QUICKLY:-}"; then workloads=( @@ -226,7 +227,7 @@ print-tasks() { workloads=( "${ALL_WORKLOADS[@]}" ) fi - for sh_path in bash dash bin/osh $osh_native; do + for sh_path in bash dash bin/osh "${osh_native[@]}"; do for workload in "${workloads[@]}"; do tsv-row $host_name $sh_path $workload done @@ -311,9 +312,10 @@ measure() { ### For release and CI local host_name=$1 # 'no-host' or 'lenny' local raw_out_dir=$2 # _tmp/osh-runtime/$X or ../../benchmark-data/osh-runtime/$X - local osh_native=$3 # $OSH_CPP_NINJA_BUILD or $OSH_CPP_BENCHMARK_DATA + shift 2 + local -a osh_native=( "$@" ) # $OSH_CPP_NINJA_BUILD or $OSH_CPP_BENCHMARK_DATA, etc... - print-tasks "$host_name" "$osh_native" \ + print-tasks "$host_name" "${osh_native[@]}" \ | run-tasks-wrapper "$host_name" "$raw_out_dir" } @@ -492,7 +494,7 @@ soil-run() { extract # could add _bin/cxx-bumpleak/oils-for-unix, although sometimes it's slower - local -a osh_bin=( $OSH_CPP_NINJA_BUILD ) + local -a osh_bin=( $OSH_CPP_NINJA_BUILD $OSH_SOUFFLE_CPP_NINJA_BUILD ) ninja "${osh_bin[@]}" local single_machine='no-host' @@ -509,7 +511,7 @@ soil-run() { local raw_out_dir="$BASE_DIR/raw.$host_job_id" mkdir -p $raw_out_dir $BASE_DIR/stage1 - measure $single_machine $raw_out_dir $OSH_CPP_NINJA_BUILD + measure $single_machine $raw_out_dir $OSH_CPP_NINJA_BUILD $OSH_SOUFFLE_CPP_NINJA_BUILD # Trivial concatenation for 1 machine stage1 '' $single_machine diff --git a/benchmarks/report.R b/benchmarks/report.R index 48a1c8559d..45ffb9368e 100755 --- a/benchmarks/report.R +++ b/benchmarks/report.R @@ -65,6 +65,8 @@ GetOshLabel = function(shell_hash, prov_dir) { label = 'osh-ovm' } else if (length(grep('bin/osh', lines)) > 0) { label = 'osh-cpython' + } else if (length(grep('_bin/.*/mycpp-souffle/osh', lines)) > 0) { + label = 'osh-native-souffle' } else if (length(grep('_bin/.*/osh', lines)) > 0) { label = 'osh-native' } else { @@ -78,6 +80,8 @@ GetOshLabel = function(shell_hash, prov_dir) { opt_suffix1 = '_bin/cxx-opt/osh' opt_suffix2 = '_bin/cxx-opt-sh/osh' +opt_suffix3 = '_bin/cxx-opt/mycpp-souffle/osh' +opt_suffix4 = '_bin/cxx-opt-sh/mycpp-souffle/osh' ShellLabels = function(shell_name, shell_hash, num_hosts) { ### Given 2 vectors, return a vector of readable labels. @@ -104,6 +108,9 @@ ShellLabels = function(shell_name, shell_hash, num_hosts) { } else if (endsWith(sh, opt_suffix1) || endsWith(sh, opt_suffix2)) { label = 'opt/osh' + } else if (endsWith(sh, opt_suffix3) || endsWith(sh, opt_suffix4)) { + label = 'opt/osh-souffle' + } else if (endsWith(sh, '_bin/cxx-opt+bumpleak/osh')) { label = 'bumpleak/osh' @@ -128,6 +135,10 @@ ShellLabelFromPath = function(sh_path) { # the opt binary is osh-native label = 'osh-native' + } else if (endsWith(sh, opt_suffix3) || endsWith(sh, opt_suffix4)) { + # the opt binary is osh-native + label = 'osh-native-souffle' + } else if (endsWith(sh, '_bin/cxx-opt+bumpleak/osh')) { label = 'bumpleak/osh' @@ -303,7 +314,7 @@ ParserReport = function(in_dir, out_dir) { arrange(host_label, num_lines) %>% mutate(osh_to_bash_ratio = `osh-native` / bash) %>% select(c(host_label, bash, dash, mksh, zsh, - `osh-ovm`, `osh-cpython`, `osh-native`, + `osh-ovm`, `osh-cpython`, `osh-native`, `osh-native-souffle`, osh_to_bash_ratio, num_lines, filename, filename_HREF)) -> elapsed @@ -317,7 +328,7 @@ ParserReport = function(in_dir, out_dir) { spread(key = shell_label, value = lines_per_ms) %>% arrange(host_label, num_lines) %>% select(c(host_label, bash, dash, mksh, zsh, - `osh-ovm`, `osh-cpython`, `osh-native`, + `osh-ovm`, `osh-cpython`, `osh-native`, `osh-native-souffle`, num_lines, filename, filename_HREF)) -> rate @@ -331,7 +342,7 @@ ParserReport = function(in_dir, out_dir) { spread(key = shell_label, value = max_rss_MB) %>% arrange(host_label, num_lines) %>% select(c(host_label, bash, dash, mksh, zsh, - `osh-ovm`, `osh-cpython`, `osh-native`, + `osh-ovm`, `osh-cpython`, `osh-native`, `osh-native-souffle`, num_lines, filename, filename_HREF)) -> max_rss @@ -350,7 +361,7 @@ ParserReport = function(in_dir, out_dir) { select(-c(irefs)) %>% spread(key = shell_label, value = thousand_irefs_per_line) %>% arrange(num_lines) %>% - select(c(bash, dash, mksh, `osh-native`, + select(c(bash, dash, mksh, `osh-native`, `osh-native-souffle`, num_lines, filename, filename_HREF)) -> instructions @@ -537,7 +548,7 @@ RuntimeReport = function(in_dir, out_dir) { mutate(native_bash_ratio = `osh-native` / bash) %>% arrange(workload, host_name) %>% select(c(workload, host_name, - bash, dash, `osh-cpython`, `osh-native`, + bash, dash, `osh-cpython`, `osh-native`, `osh-native-souffle`, py_bash_ratio, native_bash_ratio)) -> elapsed @@ -553,7 +564,7 @@ RuntimeReport = function(in_dir, out_dir) { mutate(native_bash_ratio = `osh-native` / bash) %>% arrange(workload, host_name) %>% select(c(workload, host_name, - bash, dash, `osh-cpython`, `osh-native`, + bash, dash, `osh-cpython`, `osh-native`, `osh-native-souffle`, py_bash_ratio, native_bash_ratio)) -> page_faults @@ -568,7 +579,7 @@ RuntimeReport = function(in_dir, out_dir) { mutate(native_bash_ratio = `osh-native` / bash) %>% arrange(workload, host_name) %>% select(c(workload, host_name, - bash, dash, `osh-cpython`, `osh-native`, + bash, dash, `osh-cpython`, `osh-native`, `osh-native-souffle`, py_bash_ratio, native_bash_ratio)) -> max_rss @@ -610,7 +621,7 @@ RuntimeReport = function(in_dir, out_dir) { # milliseconds don't need decimal digit precision = ColumnPrecision(list(bash = 0, dash = 0, `osh-cpython` = 0, - `osh-native` = 0, py_bash_ratio = 2, + `osh-native` = 0, `osh-native-souffle` = 0, py_bash_ratio = 2, native_bash_ratio = 2)) writeTsv(elapsed, file.path(out_dir, 'elapsed'), precision) writeTsv(page_faults, file.path(out_dir, 'page_faults'), precision) diff --git a/benchmarks/report_test.R b/benchmarks/report_test.R index e897397dcc..92de25fe64 100755 --- a/benchmarks/report_test.R +++ b/benchmarks/report_test.R @@ -18,6 +18,10 @@ TestShellLabels = function() { label = ShellLabels(shell_name, shell_hash, 1) checkEquals('opt/osh', label) + shell_name = 'yy/zz/_bin/cxx-opt/mycpp-souffle/osh' + label = ShellLabels(shell_name, shell_hash, 1) + checkEquals('opt/osh-souffle', label) + shell_name = 'yy/zz/_bin/cxx-opt+bumpleak/osh' label = ShellLabels(shell_name, shell_hash, 1) checkEquals('bumpleak/osh', label) diff --git a/bin/NINJA_subgraph.py b/bin/NINJA_subgraph.py index 3b0f3172fd..a6079afaaf 100644 --- a/bin/NINJA_subgraph.py +++ b/bin/NINJA_subgraph.py @@ -67,60 +67,75 @@ def NinjaGraph(ru): # for main_name in ('osh_eval', 'oils_for_unix'): - with open('_build/NINJA/bin.%s/translate.txt' % main_name) as f: - deps = [line.strip() for line in f] - - prefix = '_gen/bin/%s.mycpp' % main_name - outputs = [prefix + '.cc', prefix + '.h'] - n.build(outputs, - 'gen-oils-for-unix', - deps, - implicit=['_bin/shwrap/mycpp_main', RULES_PY], - variables=[('out_prefix', prefix), ('main_name', main_name), - ('preamble', 'cpp/preamble.h')]) - - if main_name == 'oils_for_unix': - # The main program! - bin_path = 'oils-for-unix' - symlinks = ['osh', 'ysh'] - else: - symlinks = [] - bin_path = None # use default - - ru.cc_binary( - '_gen/bin/%s.mycpp.cc' % main_name, - bin_path=bin_path, - symlinks=symlinks, - preprocessed=True, - matrix=(ninja_lib.COMPILERS_VARIANTS + ninja_lib.GC_PERF_VARIANTS + - ninja_lib.OTHER_VARIANTS), - deps=[ - '//bin/text_files', - '//cpp/core', - '//cpp/data_lang', - '//cpp/fanos', - '//cpp/libc', - '//cpp/osh', - '//cpp/pgen2', - '//cpp/pylib', - '//cpp/stdlib', - '//cpp/frontend_flag_spec', - '//cpp/frontend_match', - '//cpp/frontend_pyreadline', - '//data_lang/nil8.asdl', - '//display/pretty.asdl', - '//frontend/arg_types', - '//frontend/consts', - '//frontend/help_meta', - '//frontend/id_kind.asdl', - '//frontend/option.asdl', - '//frontend/signal', - '//frontend/syntax.asdl', - '//frontend/types.asdl', - '//core/optview', - '//core/runtime.asdl', - '//core/value.asdl', - '//osh/arith_parse', - '//ysh/grammar', - '//mycpp/runtime', - ]) + for translator in ('mycpp', 'mycpp-souffle'): + with open('_build/NINJA/bin.%s/translate.txt' % main_name) as f: + deps = [line.strip() for line in f] + + prefix = '_gen/bin/%s.%s' % (main_name, translator) + outputs = [prefix + '.cc', prefix + '.h'] + + variables = [ + ('out_prefix', prefix), + ('main_name', main_name), + ('translator', translator), + ('preamble', 'cpp/preamble.h'), + ] + if translator == 'mycpp-souffle': + variables.append(('extra_mycpp_opts', '--minimize-stack-roots')) + + n.build(outputs, + 'gen-oils-for-unix', + deps, + implicit=['_bin/shwrap/mycpp_main', RULES_PY], + variables=variables) + + if main_name == 'oils_for_unix': + # The main program! + if translator == 'mycpp-souffle': + bin_path = '%s/oils-for-unix' % translator + else: + # Keep the default mycpp build at the original location to + # avoid breaking benchmarks and tests. + bin_path = 'oils-for-unix' + symlinks = ['osh', 'ysh'] + else: + symlinks = [] + bin_path = None # use default + + ru.cc_binary( + '_gen/bin/%s.%s.cc' % (main_name, translator), + bin_path=bin_path, + symlinks=symlinks, + preprocessed=True, + matrix=(ninja_lib.COMPILERS_VARIANTS + ninja_lib.GC_PERF_VARIANTS + + ninja_lib.OTHER_VARIANTS), + deps=[ + '//bin/text_files', + '//cpp/core', + '//cpp/data_lang', + '//cpp/fanos', + '//cpp/libc', + '//cpp/osh', + '//cpp/pgen2', + '//cpp/pylib', + '//cpp/stdlib', + '//cpp/frontend_flag_spec', + '//cpp/frontend_match', + '//cpp/frontend_pyreadline', + '//data_lang/nil8.asdl', + '//display/pretty.asdl', + '//frontend/arg_types', + '//frontend/consts', + '//frontend/help_meta', + '//frontend/id_kind.asdl', + '//frontend/option.asdl', + '//frontend/signal', + '//frontend/syntax.asdl', + '//frontend/types.asdl', + '//core/optview', + '//core/runtime.asdl', + '//core/value.asdl', + '//osh/arith_parse', + '//ysh/grammar', + '//mycpp/runtime', + ]) diff --git a/build/native.sh b/build/native.sh index 89458cd9b4..ab8aeb5668 100755 --- a/build/native.sh +++ b/build/native.sh @@ -19,13 +19,22 @@ source build/common.sh # log # - TODO: do this in the Soil 'cpp' task tarball-demo() { + translator=${1:-mycpp} mkdir -p _bin ./configure - time _build/oils.sh '' '' SKIP_REBUILD - - local bin=_bin/cxx-opt-sh/oils-for-unix.stripped + time _build/oils.sh '' '' $translator SKIP_REBUILD + + local bin + case $translator in + mycpp) + bin=_bin/cxx-opt-sh/oils-for-unix.stripped + ;; + *) + bin=_bin/cxx-opt-sh/$translator/oils-for-unix.stripped + ;; + esac ls -l $bin diff --git a/build/ninja-rules-py.sh b/build/ninja-rules-py.sh index 77aa841a24..5e27c34cba 100755 --- a/build/ninja-rules-py.sh +++ b/build/ninja-rules-py.sh @@ -64,12 +64,14 @@ EOF gen-oils-for-unix() { local main_name=$1 - local out_prefix=$2 - local preamble=$3 - shift 3 # rest are inputs + local translator=$2 + local out_prefix=$3 + local preamble=$4 + local mycpp_opts=$5 + shift 5 # rest are inputs # Put it in _build/tmp so it's not in the tarball - local tmp=_build/tmp + local tmp=_build/tmp/$translator mkdir -p $tmp local raw_cc=$tmp/${main_name}_raw.cc @@ -82,13 +84,14 @@ gen-oils-for-unix() { _bin/shwrap/mycpp_main $mypypath $raw_cc \ --header-out $raw_header \ + $mycpp_opts \ ${EXTRA_MYCPP_ARGS:-} \ "$@" # oils_for_unix -> OILS_FOR_UNIX_MYCPP_H' local guard=${main_name^^}_MYCPP_H - { echo "// $main_name.mycpp.h: translated from Python by mycpp" + { echo "// $main_name.h: translated from Python by mycpp" echo echo "#ifndef $guard" echo "#define $guard" @@ -100,7 +103,7 @@ gen-oils-for-unix() { } > $header_out { cat < Date: Sun, 22 Sep 2024 09:19:04 -0400 Subject: [PATCH 231/506] [doc] Design skeleton for doc processing This is part of "maximalist YSH" - it's not clear if we can get this done. But I think it fits within the YSH language! --- build/doc.sh | 3 +- doc/index.md | 7 +- doc/ysh-doc-processing.md | 135 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 doc/ysh-doc-processing.md diff --git a/build/doc.sh b/build/doc.sh index 9b9d96ab51..d58ae6f745 100755 --- a/build/doc.sh +++ b/build/doc.sh @@ -100,9 +100,10 @@ readonly MARKDOWN_DOCS=( qtt j8-notation # Protocol - byo pretty-printing stream-table-process + byo + ysh-doc-processing lib-osh diff --git a/doc/index.md b/doc/index.md index e88ee8c9c4..5e9a48b63c 100644 --- a/doc/index.md +++ b/doc/index.md @@ -97,7 +97,12 @@ Features: - [Guide to YSH Error Handling](ysh-error.html) - [Guide to Procs and Funcs](proc-func.html) - [Block Literals](block-literals.html) † -- [Streams, Tables, Processes - awk, R, xargs](stream-table-process.html) † + +Designs for "Maximalist YSH": + +- [Streams, Tables, and Processes - awk, R, xargs](stream-table-process.html) † +- [Document Processing in YSH - Notation, Query, Templating](ysh-doc-processing.html) † + Crosscutting design issues: diff --git a/doc/ysh-doc-processing.md b/doc/ysh-doc-processing.md new file mode 100644 index 0000000000..7e57897265 --- /dev/null +++ b/doc/ysh-doc-processing.md @@ -0,0 +1,135 @@ +--- +in_progress: yes +default_highlighter: oils-sh +--- + +Doc Processing in YSH - Notation, Query, Templating +==================================================== + +This is a slogan for "maximalist YSH" design: + +*Documents, Objects, and Tables - HTML, JSON, and CSV* † + +This design doc is about the first part - **documents** and document processing. + +† from a paper about the C# language + +
+
+ +## Intro + +Let's sketch a design for 3 aspects of doc processing: + +1. HTM8 Notation - A **subset** of HTML5 meant for easy implementation, with + regular languages. + - It's part of J8 Notation (although it does not use J8 strings, like JSON8 + and TSV8 do.) + - It's very important to understand that this is HTM8, not HTML8! +1. A subset of CSS for querying +1. Templating in the Markaby style (a bit like Lisp, but unlike JSX templates) + +The basic goal is to write ad hod HTML processors. + +YSH programs should loosely follow the style of the DOM API in web browsers, +e.g. `document.querySelectorAll('table#mytable')` and the doc fragments it +returns. + +Note that the DOM API is not available in node.js or Deno by default, much less +any alternative lightweight JavaScript runtimes. + +I believe we can write include something that's simpler, and just as powerful, +in YSH. + +## Use Cases for HTML Processing + +These will help people get an idea. + +1. making Oils cross-ref.html + - query and replacement +1. table language - md-ul-table + - query and replacement + - many tables to make here +1. safe HTML subset, e.g. for publishing user results on continuous build + - well I think I want to encode the policy, like + - query + +Design goals: + +- Simple format that can be re-implemented anywhere + - a few re2c expressions +- Fast + - re2c uses C + - Few allocations +- much simpler than an entire browser engine + +## Operations + +- doc('

') - validates it and creates a value.Obj +- docQuery(mydoc, '#element') - does a simple search + +Constructors: + + doc { # prints valid HT8 + p { + echo 'hi' + } + p { + 'hi' # I think I want to turn on this auto-quote feature + } + raw 'bold' + } + +And then + + doc (&mydoc) { # captures the output, and creates a value.Obj + p { + 'hi' # I think I want to turn on this auto-quote feature + "hi $x" + } + } + +This is the same as the table constructor + +Module: + + source $LIB_YSH/doc.ysh + + doc (&d) { + } + doc { + } + doc('

') + + This can have both __invoke__ and __call__ + + var results = d.query('#a') + + # The doc could be __invoke__ ? + d query '#a' { + } + + doc query (d, '#a') { + for result in (results) { + echo hi + } + } + + # we create (old, new) pairs? + # this is performs an operation like: + # d.outerHTML = outerHTML + var d = d.replace(pairs) + + +Safe HTML subset + + d query (tags= :|a p div h1 h2 h3|) { + case (_frag.tag) { + a { + # get a list of all attributes + var attrs = _frag.getAttributes() + } + } + } + +If you want to take user HTML, then you first use an HTML5 -> HT8 converter. From d5014e3dfb69bf80fb73ab7b0f5aacaffeab0c1b Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 24 Sep 2024 15:18:14 -0400 Subject: [PATCH 232/506] [mycpp] Comment out 'sync' workaround to see what happens --- mycpp/pass_state.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mycpp/pass_state.py b/mycpp/pass_state.py index d5d6cb5706..ec0d7780db 100644 --- a/mycpp/pass_state.py +++ b/mycpp/pass_state.py @@ -568,9 +568,11 @@ def ComputeMinimalStackRoots(cfgs: dict[str, ControlFlowGraph], that can be queried by cppgen_pass. """ DumpControlFlowGraphs(cfgs, facts_dir=facts_dir) - # The facts files can be pretty large. Sync them first to avoid reading - # truncated files from the solver. - subprocess.run('sync {}/*.facts'.format(facts_dir), shell=True) + + # Work around bug of reading truncated files from the solver? + # Could this be a ninja race condition, with a missing dependency? + # subprocess.run('sync {}/*.facts'.format(facts_dir), shell=True) + subprocess.check_call([ '_bin/datalog/dataflow', '-F', From ac405f6ce1710f877a8e7bc0ae4645768cef375f Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 24 Sep 2024 15:26:41 -0400 Subject: [PATCH 233/506] [test/spec] Test cases for control flow within block Feedback from Julian Brown Findings: - spec/builtin-eval-source - all shells except mksh are consistent - ysh blocks differ than OSH strings, which is bad - spec/ysh-control-flow - cd builtin and proc-that-runs-block are consistent - good - proc-that-evals a string is also consistent, but it may not have enough testing Need to look into the latter issue more. This is related to issue #2039. --- spec/builtin-eval-source.test.sh | 124 ++++++++++++++++++--- spec/ysh-blocks.test.sh | 51 --------- spec/ysh-control-flow.test.sh | 180 +++++++++++++++++++++++++++++++ test/spec.sh | 4 + 4 files changed, 294 insertions(+), 65 deletions(-) create mode 100644 spec/ysh-control-flow.test.sh diff --git a/spec/builtin-eval-source.test.sh b/spec/builtin-eval-source.test.sh index eb0a002759..a69108a4e6 100644 --- a/spec/builtin-eval-source.test.sh +++ b/spec/builtin-eval-source.test.sh @@ -1,4 +1,5 @@ ## compare_shells: dash bash-4.4 mksh zsh +## oils_failures_allowed: 1 #### Eval eval "a=3" @@ -35,6 +36,115 @@ echo $? 127 ## END +#### eval string with 'break continue return error' + +set -e + +sh_func_that_evals() { + local code_str=$1 + for i in 1 2; do + echo $i + eval "$code_str" + done + echo 'end func' +} + +for code_str in break continue return false; do + echo "--- $code_str" + sh_func_that_evals "$code_str" +done +echo status=$? + +## status: 1 +## STDOUT: +--- break +1 +end func +--- continue +1 +2 +end func +--- return +1 +--- false +1 +## END + +## BUG mksh STDOUT: +--- break +1 +2 +end func +--- continue +1 +2 +end func +--- return +1 +--- false +1 +## END + +#### eval YSH block with 'break continue return error' +case $SH in dash|bash*|mksh|zsh) exit ;; esac + +shopt -s ysh:all + +proc proc_that_evals(; ; ;b) { + for i in 1 2; do + echo $i + eval (b) + done + echo 'end func' +} + +var cases = [ + ['break', ^(break)], + ['continue', ^(continue)], + ['return', ^(return)], + ['false', ^(false)], +] + +for test_case in (cases) { + var code_str, block = test_case + echo "--- $code_str" + proc_that_evals (; ; block) +} +echo status=$? + +## status: 1 +## STDOUT: +--- break +1 +end func +--- continue +1 +2 +end func +--- return +1 +--- false +1 +## END + +## N-I dash/bash/mksh/zsh status: 0 +## N-I dash/bash/mksh/zsh STDOUT: +## END + +#### exit within eval (regression) +eval 'exit 42' +echo 'should not get here' +## stdout-json: "" +## status: 42 + +#### exit within source (regression) +cd $TMP +echo 'exit 42' > lib.sh +. ./lib.sh +echo 'should not get here' +## stdout-json: "" +## status: 42 + #### Source lib=$TMP/spec-test-lib.sh echo 'LIBVAR=libvar' > $lib @@ -226,20 +336,6 @@ rm dir/cmd path ## END -#### exit within eval (regression) -eval 'exit 42' -echo 'should not get here' -## stdout-json: "" -## status: 42 - -#### exit within source (regression) -cd $TMP -echo 'exit 42' > lib.sh -. ./lib.sh -echo 'should not get here' -## stdout-json: "" -## status: 42 - #### source doesn't crash when targeting a directory cd $TMP mkdir -p dir diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh index 79dd229c10..518227c1e1 100644 --- a/spec/ysh-blocks.test.sh +++ b/spec/ysh-blocks.test.sh @@ -40,57 +40,6 @@ cd { echo $PWD } /tmp ## END -#### cd with block: fatal error in block -shopt -s ysh:all -cd / { - echo one - false - echo two -} -## status: 1 -## STDOUT: -one -## END - - -#### cd with block: return in block -shopt -s oil:all -f() { - cd / { - echo one - return - echo two - } - echo 'end func' -} -f -## STDOUT: -one -end func -## END - -#### cd with block: break in block -shopt -s oil:all -f() { - cd / { - echo one - for i in 1 2; do - echo $i - break # break out of loop - done - - break # break out of block isn't valid - echo two - } - echo end func -} -f -## status: 1 -## STDOUT: -one -1 -## END - #### cd with block exits with status 0 shopt -s ysh:all cd / { diff --git a/spec/ysh-control-flow.test.sh b/spec/ysh-control-flow.test.sh new file mode 100644 index 0000000000..da59dca09b --- /dev/null +++ b/spec/ysh-control-flow.test.sh @@ -0,0 +1,180 @@ + + +#### cd builtin: fatal error in block +shopt -s ysh:all +cd / { + echo one + false + echo two +} +## status: 1 +## STDOUT: +one +## END + + +#### cd builtin: return in block +shopt -s ysh:all +f() { + cd / { + echo one + return + echo two + } + echo 'end func' +} +f +## STDOUT: +one +end func +## END + +#### cd builtin: break in block +shopt -s ysh:all +f() { + cd / { + echo one + for i in 1 2; do + echo $i + break # break out of loop + done + + break # break out of block isn't valid + echo two + } + echo end func +} +f +## status: 1 +## STDOUT: +one +1 +## END + +#### proc eval block: fatal error +shopt -s ysh:all + +proc proc-that-runs-block (; ; ; b) { + eval (b) +} +proc-that-runs-block { + echo one + false + echo two +} +## status: 1 +## STDOUT: +one +## END + +#### proc eval block: return +shopt -s ysh:all + +proc proc-that-runs-block (; ; ; b) { + eval (b) +} + +f() { + proc-that-runs-block { + echo one + return + echo two + } + echo 'end func' +} +f +## STDOUT: +one +end func +## END + +#### proc eval block: break in block +shopt -s ysh:all + +proc proc-that-runs-block (; ; ; b) { + eval (b) +} + +f() { + proc-that-runs-block { + echo one + for i in 1 2; do + echo $i + break # break out of loop + done + + break # break out of block isn't valid + echo two + } + echo end func +} +f +## status: 1 +## STDOUT: +one +1 +## END + +#### proc eval string: fatal error +shopt -s ysh:all + +proc proc-that-evals (s) { + eval $s +} +proc-that-evals ' + echo one + false + echo two +' +## status: 1 +## STDOUT: +one +## END + +#### proc eval string: return +shopt -s ysh:all + +proc proc-that-evals (s) { + eval $s +} + +f() { + proc-that-evals ' + echo one + return + echo two + ' + echo 'end func' +} +f +## STDOUT: +one +end func +## END + +#### proc eval string: break +shopt -s ysh:all + +proc proc-that-evals (s) { + eval $s +} + +f() { + proc-that-evals ' + echo one + for i in 1 2; do + echo $i + break # break out of loop + done + + break # break out of string is not valid + echo two + ' + echo end func +} +f +## status: 1 +## STDOUT: +one +1 +## END diff --git a/test/spec.sh b/test/spec.sh index c1bd9c99d0..84dfe980db 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -745,6 +745,10 @@ ysh-blocks() { run-file ysh-blocks "$@" } +ysh-control-flow() { + run-file ysh-control-flow "$@" +} + ysh-bugs() { run-file ysh-bugs "$@" } From 5b3fd5c0bc1c56798d16a9376971702f3b0c3f3e Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 24 Sep 2024 22:00:29 -0400 Subject: [PATCH 234/506] [ysh] Allow control flow out of block arguments Don't handle 'return break continue false' in a special way. This addresses issue #2039, reported by Julian Brown. I may still want to look into remaining inconsistencies bewtween: eval $mystr eval (cmd) It might make sense to have eval-command (cmd) --- builtin/meta_osh.py | 12 ++++- osh/cmd_eval.py | 12 +---- spec/builtin-eval-source.test.sh | 2 +- spec/ysh-blocks.test.sh | 87 ++++---------------------------- spec/ysh-control-flow.test.sh | 4 +- 5 files changed, 27 insertions(+), 90 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index 4c8c50eb01..bffa8a7f29 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -64,7 +64,17 @@ def __init__( def RunTyped(self, cmd_val): # type: (cmd_value.Argv) -> int - """For eval (mycmd)""" + """For eval (mycmd) + + Note: this doesn't have the exact same interface as main_loop.Batch(). + I wonder if it's better to have + + var cmd = parseCommand(s) + var expr = parseExpr(s) + + eval-command (cmd) or eval-block (b) + = evalExpr(expr) + """ rd = typed_args.ReaderForProc(cmd_val) cmd = rd.PosCommand() dollar0 = rd.NamedStr("dollar0", None) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index f6ad542e69..42cd5bdc99 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -2107,17 +2107,7 @@ def EvalCommand(self, block): (Should those be more like eval 'mystring'?) """ - status = 0 - try: - status = self._Execute(block) # can raise FatalRuntimeError, etc. - except vm.IntControlFlow as e: # A block is more like a function. - # return in a block - if e.IsReturn(): - status = e.StatusCode() - else: - e_die('Unexpected control flow in block', e.token) - - return status + return self._Execute(block) # can raise FatalRuntimeError, etc. def RunTrapsOnExit(self, mut_status): # type: (IntParamBox) -> None diff --git a/spec/builtin-eval-source.test.sh b/spec/builtin-eval-source.test.sh index a69108a4e6..48e6df6695 100644 --- a/spec/builtin-eval-source.test.sh +++ b/spec/builtin-eval-source.test.sh @@ -1,5 +1,5 @@ ## compare_shells: dash bash-4.4 mksh zsh -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 #### Eval eval "a=3" diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh index 518227c1e1..e88b420f5a 100644 --- a/spec/ysh-blocks.test.sh +++ b/spec/ysh-blocks.test.sh @@ -40,19 +40,22 @@ cd { echo $PWD } /tmp ## END -#### cd with block exits with status 0 +#### cd passed block with return 1 shopt -s ysh:all -cd / { - echo block - # This return value is ignored. - # Or maybe this should be a runtime error? - return 1 +f() { + cd / { + echo block + return 1 + echo 'not reached' + } } -echo status=$? +f +echo 'not reached' + +## status: 1 ## STDOUT: block -status=0 ## END #### block doesn't have its own scope @@ -151,74 +154,6 @@ builtin / command / ## END - -#### Consistency: Control Flow and Blocks -shopt --set parse_brace - -# "Invalid control flow at top level" -eval ' - cd / { - echo cd - break - } -' -echo cd no loop $? - -# warning: "Unexpected control flow in block" (strict_control_flow) -eval ' -while true { - cd / { - echo cd - break - } -} -' -echo cd loop $? - -eval ' -while true { - shopt --unset errexit { - echo shopt - continue - } -} -' -echo shopt continue $? - -eval ' -while true { - shvar FOO=foo { - echo shvar - continue - } -} -' -echo shvar continue $? - - -eval ' -while true { - try { - echo try - break - } -} -' -echo try break $? - -## STDOUT: -cd -cd no loop 0 -cd -cd loop 1 -shopt -shopt continue 1 -shvar -shvar continue 1 -try -try break 1 -## END - #### Consistency: Exit Status and Blocks shopt --set parse_brace diff --git a/spec/ysh-control-flow.test.sh b/spec/ysh-control-flow.test.sh index da59dca09b..87ccebb6ad 100644 --- a/spec/ysh-control-flow.test.sh +++ b/spec/ysh-control-flow.test.sh @@ -21,12 +21,12 @@ f() { return echo two } + # not reached, because we're turning out of f echo 'end func' } f ## STDOUT: one -end func ## END #### cd builtin: break in block @@ -80,6 +80,7 @@ f() { return echo two } + # this is reached because we're returning of proc-that-runs-block echo 'end func' } f @@ -144,6 +145,7 @@ f() { return echo two ' + # this is reached because we're returning of proc-that-evals echo 'end func' } f From 50f7f6943fce2f69edefc2d61b9f93cb9e4ef5a3 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 25 Sep 2024 20:37:14 -0400 Subject: [PATCH 235/506] [builtin-func] Implement parseCommand() Based on a use case from Julian Brown. We'll have call io->evalToDict() And also var expr = parseExpr(s) To go with evalExpr(). Although we might want to change it to: io.evalExpr() If it can run $(echo hi) and so forth. --- builtin/func_misc.py | 49 -------------- builtin/func_reflect.py | 119 ++++++++++++++++++++++++++++++++++ builtin/meta_osh.py | 2 +- builtin/method_io.py | 8 ++- builtin/printf_osh.py | 2 +- builtin/trap_osh.py | 2 +- core/shell.py | 38 ++++++----- display/ui.py | 14 ++-- doc/ref/chap-builtin-func.md | 14 ++++ doc/ref/toc-ysh.md | 19 +++--- frontend/syntax.asdl | 4 +- osh/sh_expr_eval.py | 2 +- spec/ysh-builtin-eval.test.sh | 28 +++++++- 13 files changed, 212 insertions(+), 89 deletions(-) create mode 100644 builtin/func_reflect.py diff --git a/builtin/func_misc.py b/builtin/func_misc.py index c059a1b0af..134c7db5ba 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -4,12 +4,10 @@ """ from __future__ import print_function -from _devbuild.gen.runtime_asdl import (scope_e) from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj) from core import error from core import num -from core import state from display import pp_value from display import ui from core import vm @@ -19,7 +17,6 @@ from mycpp import mops from mycpp import mylib from mycpp.mylib import NewDict, iteritems, log, tagswitch -from ysh import expr_eval from ysh import val_ops from typing import TYPE_CHECKING, Dict, List, Optional, cast @@ -495,52 +492,6 @@ def Call(self, rd): return value.List(l) -class Shvar_get(vm._Callable): - """Look up with dynamic scope.""" - - def __init__(self, mem): - # type: (state.Mem) -> None - vm._Callable.__init__(self) - self.mem = mem - - def Call(self, rd): - # type: (typed_args.Reader) -> value_t - name = rd.PosStr() - rd.Done() - return state.DynamicGetVar(self.mem, name, scope_e.Dynamic) - - -class GetVar(vm._Callable): - """Look up normal scoping rules.""" - - def __init__(self, mem): - # type: (state.Mem) -> None - vm._Callable.__init__(self) - self.mem = mem - - def Call(self, rd): - # type: (typed_args.Reader) -> value_t - name = rd.PosStr() - rd.Done() - return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal) - - -class EvalExpr(vm._Callable): - - def __init__(self, expr_ev): - # type: (expr_eval.ExprEvaluator) -> None - self.expr_ev = expr_ev - - def Call(self, rd): - # type: (typed_args.Reader) -> value_t - lazy = rd.PosExpr() - rd.Done() - - result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken()) - - return result - - class ToJson8(vm._Callable): def __init__(self, is_j8): diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py new file mode 100644 index 0000000000..db1b2e8d4e --- /dev/null +++ b/builtin/func_reflect.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python2 +""" +func_reflect.py - Functions for reflecting on Oils code - OSH or YSH. +""" +from __future__ import print_function + +from _devbuild.gen.runtime_asdl import (scope_e) +from _devbuild.gen.syntax_asdl import source +from _devbuild.gen.value_asdl import (value, value_t) + +from core import alloc +from core import error +from core import main_loop +from core import state +from core import vm +from frontend import reader +from frontend import typed_args +from mycpp.mylib import log +from ysh import expr_eval + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from frontend import parse_lib + from display import ui + +_ = log + + +class Shvar_get(vm._Callable): + """Look up with dynamic scope.""" + + def __init__(self, mem): + # type: (state.Mem) -> None + vm._Callable.__init__(self) + self.mem = mem + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + name = rd.PosStr() + rd.Done() + return state.DynamicGetVar(self.mem, name, scope_e.Dynamic) + + +class GetVar(vm._Callable): + """Look up normal scoping rules.""" + + def __init__(self, mem): + # type: (state.Mem) -> None + vm._Callable.__init__(self) + self.mem = mem + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + name = rd.PosStr() + rd.Done() + return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal) + + +class ParseCommand(vm._Callable): + + def __init__(self, parse_ctx, errfmt): + # type: (parse_lib.ParseContext, ui.ErrorFormatter) -> None + self.parse_ctx = parse_ctx + self.errfmt = errfmt + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + code_str = rd.PosStr() + rd.Done() + + line_reader = reader.StringLineReader(code_str, self.parse_ctx.arena) + c_parser = self.parse_ctx.MakeOshParser(line_reader) + + # TODO: it would be nice to point to the location of the expression + # argument + src = source.Dynamic('parseCommand()', rd.LeftParenToken()) + with alloc.ctx_SourceCode(self.parse_ctx.arena, src): + try: + cmd = main_loop.ParseWholeFile(c_parser) + except error.Parse as e: + # This prints the location + self.errfmt.PrettyPrintError(e) + + # TODO: add inner location info to this structured error + raise error.Structured(3, "Syntax error in parseCommand()", + rd.LeftParenToken()) + + return value.Command(cmd) + + +class ParseExpr(vm._Callable): + + def __init__(self, parse_ctx, errfmt): + # type: (parse_lib.ParseContext, ui.ErrorFormatter) -> None + self.parse_ctx = parse_ctx + self.errfmt = errfmt + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + code_str = rd.PosStr() + rd.Done() + + return value.Null + + +class EvalExpr(vm._Callable): + + def __init__(self, expr_ev): + # type: (expr_eval.ExprEvaluator) -> None + self.expr_ev = expr_ev + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + lazy = rd.PosExpr() + rd.Done() + + result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken()) + + return result diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index bffa8a7f29..a8e38f0012 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -116,7 +116,7 @@ def Run(self, cmd_val): line_reader = reader.StringLineReader(code_str, self.arena) c_parser = self.parse_ctx.MakeOshParser(line_reader) - src = source.ArgvWord('eval', eval_loc) + src = source.Dynamic('eval arg', eval_loc) with dev.ctx_Tracer(self.tracer, 'eval', None): with alloc.ctx_SourceCode(self.arena, src): return main_loop.Batch(self.cmd_ev, diff --git a/builtin/method_io.py b/builtin/method_io.py index bae6bfdf8e..665a3433c5 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -16,6 +16,9 @@ _ = log +EVAL_NULL = 1 +EVAL_DICT = 2 + class Eval(vm._Callable): """ @@ -29,9 +32,10 @@ class Eval(vm._Callable): The CALLER must handle errors. """ - def __init__(self, cmd_ev): - # type: (cmd_eval.CommandEvaluator) -> None + def __init__(self, cmd_ev, which): + # type: (cmd_eval.CommandEvaluator, int) -> None self.cmd_ev = cmd_ev + self.which = which def Call(self, rd): # type: (typed_args.Reader) -> value_t diff --git a/builtin/printf_osh.py b/builtin/printf_osh.py index 7c7c441ba0..2d7e15c18c 100644 --- a/builtin/printf_osh.py +++ b/builtin/printf_osh.py @@ -510,7 +510,7 @@ def Run(self, cmd_val): parser = _FormatStringParser(lexer) with alloc.ctx_SourceCode(arena, - source.ArgvWord('printf', fmt_loc)): + source.Dynamic('printf arg', fmt_loc)): try: parts = parser.Parse() except error.Parse as e: diff --git a/builtin/trap_osh.py b/builtin/trap_osh.py index c3eb80be64..f2d6cc701f 100644 --- a/builtin/trap_osh.py +++ b/builtin/trap_osh.py @@ -216,7 +216,7 @@ def _ParseTrapCode(self, code_str): c_parser = self.parse_ctx.MakeOshParser(line_reader) # TODO: the SPID should be passed through argv. - src = source.ArgvWord('trap', loc.Missing) + src = source.Dynamic('trap arg', loc.Missing) with alloc.ctx_SourceCode(self.arena, src): try: node = main_loop.ParseWholeFile(c_parser) diff --git a/core/shell.py b/core/shell.py index cee29984a4..63e5555a3a 100644 --- a/core/shell.py +++ b/core/shell.py @@ -61,6 +61,7 @@ from builtin import func_eggex from builtin import func_hay from builtin import func_misc +from builtin import func_reflect from builtin import method_dict from builtin import method_io @@ -566,20 +567,24 @@ def Main( # PromptEvaluator rendering is needed in non-interactive shells for @P. prompt_ev = prompt.Evaluator(lang, version_str, parse_ctx, mem) - io_methods = { - 'promptVal': value.BuiltinFunc(method_io.PromptVal(prompt_ev)), + io_methods = {} # type: Dict[str, value_t] + io_methods['promptVal'] = value.BuiltinFunc(method_io.PromptVal(prompt_ev)) - # The M/ prefix means it's io->eval() - 'M/eval': value.BuiltinFunc(method_io.Eval(cmd_ev)), + # The M/ prefix means it's io->eval() + io_methods['M/eval'] = value.BuiltinFunc( + method_io.Eval(cmd_ev, method_io.EVAL_NULL)) + io_methods['M/evalToDict'] = value.BuiltinFunc( + method_io.Eval(cmd_ev, method_io.EVAL_DICT)) - # Identical to command sub - 'captureStdout': value.BuiltinFunc(method_io.CaptureStdout(shell_ex)), + # Identical to command sub + io_methods['captureStdout'] = value.BuiltinFunc( + method_io.CaptureStdout(shell_ex)) + + # TODO: + io_methods['time'] = value.BuiltinFunc(method_io.Time()) + io_methods['strftime'] = value.BuiltinFunc(method_io.Strftime()) + io_methods['glob'] = None - # TODO: - 'time': value.BuiltinFunc(method_io.Time()), - 'strftime': value.BuiltinFunc(method_io.Strftime()), - 'glob': None, - } # type: Dict[str, value_t] io_props = {'stdin': value.Stdin} # type: Dict[str, value_t] io_obj = Obj(Obj(None, io_methods), io_props) @@ -857,7 +862,13 @@ def Main( mem)) _SetGlobalFunc(mem, '_end', func_eggex.MatchFunc(func_eggex.E, None, mem)) - _SetGlobalFunc(mem, 'evalExpr', func_misc.EvalExpr(expr_ev)) + _SetGlobalFunc(mem, 'parseCommand', + func_reflect.ParseCommand(parse_ctx, errfmt)) + _SetGlobalFunc(mem, 'parseExpr', func_reflect.ParseExpr(parse_ctx, errfmt)) + _SetGlobalFunc(mem, 'evalExpr', func_reflect.EvalExpr(expr_ev)) + + _SetGlobalFunc(mem, 'shvarGet', func_reflect.Shvar_get(mem)) + _SetGlobalFunc(mem, 'getVar', func_reflect.GetVar(mem)) _SetGlobalFunc(mem, 'Object', func_misc.Object()) _SetGlobalFunc(mem, 'prototype', func_misc.Prototype()) @@ -890,9 +901,6 @@ def Main( _SetGlobalFunc(mem, 'maybe', func_misc.Maybe()) _SetGlobalFunc(mem, 'glob', func_misc.Glob(globber)) - _SetGlobalFunc(mem, 'shvarGet', func_misc.Shvar_get(mem)) - _SetGlobalFunc(mem, 'getVar', func_misc.GetVar(mem)) - # Serialize _SetGlobalFunc(mem, 'toJson8', func_misc.ToJson8(True)) _SetGlobalFunc(mem, 'toJson', func_misc.ToJson8(False)) diff --git a/display/ui.py b/display/ui.py index ad5503d497..3c4be39a8c 100644 --- a/display/ui.py +++ b/display/ui.py @@ -188,22 +188,22 @@ def GetLineSourceString(line, quote_filename=False): if quote_filename: s = j8_lite.EncodeString(s, unquoted_ok=True) - elif case(source_e.ArgvWord): - src = cast(source.ArgvWord, UP_src) + elif case(source_e.Dynamic): + src = cast(source.Dynamic, UP_src) # Note: _PrintWithLocation() uses this more specifically # TODO: check loc.Missing; otherwise get Token from loc_t, then line blame_tok = location.TokenFor(src.location) if blame_tok is None: - s = '[ %s word at ? ]' % src.what + s = '[ %s at ? ]' % src.what else: line = blame_tok.line line_num = line.line_num outer_source = GetLineSourceString( line, quote_filename=quote_filename) - s = '[ %s word at line %d of %s ]' % (src.what, line_num, - outer_source) + s = '[ %s at line %d of %s ]' % (src.what, line_num, + outer_source) elif case(source_e.Variable): src = cast(source.Variable, UP_src) @@ -316,8 +316,8 @@ def _PrintWithLocation(prefix, msg, blame_loc, show_code): # We overwrite it with the original token. _PrintCodeExcerpt(line2, orig_col + lbracket_col, 1, f) - elif case(source_e.ArgvWord): - src = cast(source.ArgvWord, UP_src) + elif case(source_e.Dynamic): + src = cast(source.Dynamic, UP_src) # Special case for eval, unset, printf -v, etc. # Show errors: diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index 8991e78f4d..5a1a194b12 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -392,6 +392,20 @@ scope" rule.) If the variable isn't defined, `getVar()` returns `null`. So there's no way to distinguish an undefined variable from one that's `null`. +### `parseCommand()` + +Given a code string, parse it as a command (with the current parse options). + +Returns a `value.Command` instance. + +### `parseExpr()` + +TODO: + +Given a code string, parse it as an expression. + +Returns a `value.Expr` instance. + ### `evalExpr()` Given a an expression quotation, evaluate it and return its value: diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 382a98d3b4..fc6acab265 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -42,25 +42,25 @@ error handling, and more. [Atom Types] Null Bool [Number Types] Int Float [Str] X find() replace() - trim() trimStart() trimEnd() + trim() trimStart() trimEnd() startsWith() endsWith() upper() lower() search() leftMatch() - [List] List/append() pop() extend() indexOf() - X insert() X remove() reverse() - [Dict] keys() values() get() erase() + [List] List/append() pop() extend() indexOf() + X insert() X remove() reverse() + [Dict] keys() values() get() erase() X inc() X accum() [Range] [Eggex] - [Match] group() start() end() + [Match] group() start() end() X groups() X groupDict() [Place] setValue() [Code Types] Expr Command BuiltinFunc BoundFunc -X [Func] name() location() toJson() -X [Proc] name() location() toJson() +X [Func] name() location() toJson() +X [Proc] name() location() toJson() X [Module] name() filename() - [IO] eval() captureStdout() + [IO] eval() evalToDict() captureStdout() promptVal() X time() X strftime() X glob() @@ -85,7 +85,8 @@ X [Module] name() filename() toJson8() fromJson8() X [J8 Decode] J8.Bool() J8.Int() ... [Pattern] _group() _start() _end() - [Introspection] shvarGet() getVar() evalExpr() + [Introspection] shvarGet() getVar() + parseCommand() X parseExpr() evalExpr() [Hay Config] parseHay() evalHay() X [Hashing] sha1dc() sha256() ``` diff --git a/frontend/syntax.asdl b/frontend/syntax.asdl index 1784c74664..09c001f23c 100644 --- a/frontend/syntax.asdl +++ b/frontend/syntax.asdl @@ -53,8 +53,8 @@ module syntax | SourcedFile(str path, loc location) # code parsed from a word - # used for 'eval', 'trap', 'printf', 'complete -W', etc. - | ArgvWord(str what, loc location) + # used for 'eval', 'trap', 'printf', 'complete -W', parseCommand() + | Dynamic(str what, loc location) # code parsed from the value of a variable # used for $PS1 $PROMPT_COMMAND diff --git a/osh/sh_expr_eval.py b/osh/sh_expr_eval.py index f681e09ff3..be93ac5b8e 100644 --- a/osh/sh_expr_eval.py +++ b/osh/sh_expr_eval.py @@ -234,7 +234,7 @@ def ParseLValue(self, s, location): a_parser = self.parse_ctx.MakeArithParser(s) with alloc.ctx_SourceCode(self.arena, - source.ArgvWord('dynamic LHS', location)): + source.Dynamic('dynamic LHS', location)): try: anode = a_parser.Parse() except error.Parse as e: diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index adbf22b11b..09256b2f2f 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 1 +## oils_failures_allowed: 2 #### Eval does not take a literal block - can restore this later @@ -337,3 +337,29 @@ one one (Dict) {"code":1} ## END + + +#### parseCommand then io.evalToDict() + +var cmd = parseCommand('var x = 42; echo hi; var y = 99') + +pp test_ (cmd) +#pp asdl_ (cmd) + +var d = io->evalToDict(cmd) + +pp test_ (d) + +## STDOUT: +## END + +#### parseCommand with syntax error + +try { + var cmd = parseCommand('echo >') +} +pp test_ (_error) + +## STDOUT: +(Dict) {"code":3,"message":"Syntax error in parseCommand()"} +## END From cd9d5cec2a85ee68695652427887469077c3860e Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 25 Sep 2024 22:31:36 -0400 Subject: [PATCH 236/506] [ysh] Some progress on io->evalToDict() I realized that we need a __builtins__ module like Python But the module will be a value.Obj, which has a Dict[str, value_t] It doesn't need to be a Dict[str, Cell], because we don't need flags like - readonly - everything is readonly - export - nameref - the -i flag, if we ever implement it etc. YSH values don't need any of those concepts --- builtin/method_io.py | 55 +++++++++++++++++++++++++++++++---- core/shell.py | 2 +- core/state.py | 11 +++++-- doc/ref/chap-builtin-func.md | 6 ++-- doc/ref/chap-type-method.md | 13 ++++++++- doc/ref/toc-ysh.md | 4 +-- spec/ysh-builtin-eval.test.sh | 4 +++ 7 files changed, 79 insertions(+), 16 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index 665a3433c5..979f9ab0d9 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -6,7 +6,7 @@ from core import error from core import num from core import vm -from mycpp.mylib import log +from mycpp.mylib import iteritems, log, NewDict from osh import prompt from typing import Dict, TYPE_CHECKING @@ -24,14 +24,17 @@ class Eval(vm._Callable): """ These are similar: - var c = ^(echo hi) + var cmd = ^(echo hi) + call io->eval(cmd) - eval (c) - call _io->eval(c) + Also give the top namespace + + call io->evalToDict(cmd) + + TODO: remove eval (c) The CALLER must handle errors. """ - def __init__(self, cmd_ev, which): # type: (cmd_eval.CommandEvaluator, int) -> None self.cmd_ev = cmd_ev @@ -45,7 +48,34 @@ def Call(self, rd): # errors can arise from false' and 'exit' unused_status = self.cmd_ev.EvalCommand(cmd) - return value.Null + + if self.which == EVAL_NULL: + return value.Null + + elif self.which == EVAL_DICT: + block_attrs = self.cmd_ev.mem.TopNamespace() + + # Copied from builtin/hay_ysh.py + # Hay should be rewritten with YSH reflection primitives. + # + # Hay pushes a temp frame. + # TODO: + + attrs = NewDict() # type: Dict[str, value_t] + for name, cell in iteritems(block_attrs): + #log('name %r', name) + #log('cell %r', cell) + + # User can hide variables with _ suffix + # e.g. for i_ in foo bar { echo $i_ } + if name.endswith('_'): + continue + + attrs[name] = cell.val + return value.Dict(attrs) + + else: + raise AssertionError() class CaptureStdout(vm._Callable): @@ -103,6 +133,8 @@ def Call(self, rd): return value.Str(self.prompt_ev.PromptVal(what)) +# TODO: Implement these + class Time(vm._Callable): def __init__(self): @@ -123,3 +155,14 @@ def __init__(self): def Call(self, rd): # type: (typed_args.Reader) -> value_t return value.Null + + +class Glob(vm._Callable): + + def __init__(self): + # type: () -> None + pass + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + return value.Null diff --git a/core/shell.py b/core/shell.py index 63e5555a3a..59e3b3d435 100644 --- a/core/shell.py +++ b/core/shell.py @@ -583,7 +583,7 @@ def Main( # TODO: io_methods['time'] = value.BuiltinFunc(method_io.Time()) io_methods['strftime'] = value.BuiltinFunc(method_io.Strftime()) - io_methods['glob'] = None + io_methods['glob'] = value.BuiltinFunc(method_io.Glob()) io_props = {'stdin': value.Stdin} # type: Dict[str, value_t] io_obj = Obj(Obj(None, io_methods), io_props) diff --git a/core/state.py b/core/state.py index 008e24e977..f4b2b535a2 100644 --- a/core/state.py +++ b/core/state.py @@ -1737,6 +1737,9 @@ def SetPlace(self, place, val, blame_loc): def SetLocalName(self, lval, val): # type: (LeftName, value_t) -> None + """ + Set a name in the local scope - used for func/proc param binding, etc. + """ # Equivalent to # self._ResolveNameOnly(lval.name, scope_e.LocalOnly) @@ -1754,7 +1757,6 @@ def SetLocalName(self, lval, val): def SetNamed(self, lval, val, which_scopes, flags=0): # type: (LeftName, value_t, scope_t, int) -> None - if flags & SetNameref or flags & ClearNameref: # declare -n ref=x # refers to the ref itself cell, name_map = self._ResolveNameOnly(lval.name, which_scopes) @@ -2130,6 +2132,7 @@ def GetValue(self, name, which_scopes=scope_e.Shopt): if cell: return cell.val + # TODO: Can look in the builtins module, which is a value.Obj return value.Undef def GetCell(self, name, which_scopes=scope_e.Shopt): @@ -2532,8 +2535,10 @@ def DynamicGetVar(mem, name, which_scopes): def GetString(mem, name): # type: (Mem, str) -> str - """Wrapper around GetValue(). Check that HOME, PWD, OLDPWD, etc. are - strings. bash doesn't have these errors because ${array} is ${array[0]}. + """Wrapper around GetValue(). + + Check that HOME, PWD, OLDPWD, etc. are strings. bash doesn't have these + errors because ${array} is ${array[0]}. TODO: We could also check this when you're storing variables? """ diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index 5a1a194b12..623d2a2ca8 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -396,7 +396,7 @@ distinguish an undefined variable from one that's `null`. Given a code string, parse it as a command (with the current parse options). -Returns a `value.Command` instance. +Returns a `value.Command` instance, or raises an error. ### `parseExpr()` @@ -404,7 +404,7 @@ TODO: Given a code string, parse it as an expression. -Returns a `value.Expr` instance. +Returns a `value.Expr` instance, or raises an error. ### `evalExpr()` @@ -415,6 +415,8 @@ Given a an expression quotation, evaluate it and return its value: $ = evalExpr(expr) 3 + + ## Hay Config ### parseHay() diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 5ce3319e89..5c9500d27e 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -549,7 +549,7 @@ A module is a file with YSH code. Evaluate a command, and return `null`. var c = ^(echo hi) - call _io->eval(c) + call io->eval(c) It's like like the `eval` builtin, and meant to be used in pure functions. @@ -560,6 +560,17 @@ shell VM. Though this runs in the same VM, not a new one. --> +### evalToDict() + +The `evalToDict()` method is like the `eval()` method, but it also returns a +Dict of bindings. + +TODO: + +- Does it push a new frame? Or is this a new module? + - I think we have to change the lookup rules +- Move functions like `len()` to their own `__builtin__` module? + ### captureStdout() Capture stdout of a command a string. diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index fc6acab265..c885c5c03c 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -59,11 +59,9 @@ error handling, and more. BuiltinFunc BoundFunc X [Func] name() location() toJson() X [Proc] name() location() toJson() -X [Module] name() filename() [IO] eval() evalToDict() captureStdout() promptVal() - X time() X strftime() - X glob() + X time() X strftime() X glob() ```

diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 09256b2f2f..f17a75c201 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -342,6 +342,7 @@ one #### parseCommand then io.evalToDict() var cmd = parseCommand('var x = 42; echo hi; var y = 99') +#var cmd = parseCommand('echo hi') pp test_ (cmd) #pp asdl_ (cmd) @@ -351,6 +352,9 @@ var d = io->evalToDict(cmd) pp test_ (d) ## STDOUT: + +hi +(Dict) ## END #### parseCommand with syntax error From 086561915df1cc5f765469aa5099f26f7be66974 Mon Sep 17 00:00:00 2001 From: Melvin Walls Date: Thu, 26 Sep 2024 00:06:44 -0400 Subject: [PATCH 237/506] [mycpp] Use a temp dir for souffle facts and outputs (#2079) * always write all souffle fact files * the temp dir can be overriden with MYCPP_SOUFFLE_DIR --- mycpp/mycpp_main.py | 11 +++++++++- mycpp/pass_state.py | 50 ++++++++++++++++++++++++++------------------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/mycpp/mycpp_main.py b/mycpp/mycpp_main.py index 6c15351b05..82ab6916f3 100755 --- a/mycpp/mycpp_main.py +++ b/mycpp/mycpp_main.py @@ -7,6 +7,7 @@ import optparse import os import sys +import tempfile from typing import List, Optional, Tuple @@ -370,7 +371,15 @@ def main(argv): log('\tmycpp pass: DATAFLOW') stack_roots = None if opts.minimize_stack_roots: - stack_roots = pass_state.ComputeMinimalStackRoots(cfgs) + # souffle_dir contains two subdirectories. + # facts: TSV files for the souffle inputs generated by mycpp + # outputs: TSV files for the solver's output relations + souffle_dir = os.getenv('MYCPP_SOUFFLE_DIR', None) + if souffle_dir is None: + tmp_dir = tempfile.TemporaryDirectory() + souffle_dir = tmp_dir.name + stack_roots = pass_state.ComputeMinimalStackRoots(cfgs, + souffle_dir=souffle_dir) else: pass_state.DumpControlFlowGraphs(cfgs) diff --git a/mycpp/pass_state.py b/mycpp/pass_state.py index ec0d7780db..dc6683f42b 100644 --- a/mycpp/pass_state.py +++ b/mycpp/pass_state.py @@ -170,7 +170,8 @@ class Fact(object): def __init__(self) -> None: pass - def name(self) -> str: + @staticmethod + def name() -> str: raise NotImplementedError() def Generate(self, func: str, statement: int) -> str: @@ -182,7 +183,8 @@ class FunctionCall(Fact): def __init__(self, callee: str) -> None: self.callee = callee - def name(self) -> str: + @staticmethod + def name() -> str: return 'call' def Generate(self, func: str, statement: int) -> str: @@ -198,7 +200,8 @@ def __init__(self, ref: SymbolPath, obj: str) -> None: self.ref = ref self.obj = obj - def name(self) -> str: + @staticmethod + def name() -> str: return 'assign' def Generate(self, func: str, statement: int) -> str: @@ -216,7 +219,8 @@ def __init__(self, lhs: SymbolPath, rhs: SymbolPath) -> None: self.lhs = lhs self.rhs = rhs - def name(self) -> str: + @staticmethod + def name() -> str: return 'assign' def Generate(self, func: str, statement: int) -> str: @@ -250,7 +254,8 @@ class Use(Fact): def __init__(self, ref: SymbolPath) -> None: self.ref = ref - def name(self) -> str: + @staticmethod + def name() -> str: return 'use' def Generate(self, func: str, statement: int) -> str: @@ -269,7 +274,8 @@ def __init__(self, ref: SymbolPath, callee: SymbolPath, self.callee = callee self.arg_pos = arg_pos - def name(self) -> str: + @staticmethod + def name() -> str: return 'bind' def Generate(self, func: str, statement: int) -> str: @@ -538,8 +544,16 @@ def DumpControlFlowGraphs(cfgs: dict[str, ControlFlowGraph], directory as text files that can be consumed by datalog. """ edge_facts = '{}/cf_edge.facts'.format(facts_dir) - fact_files = {} + os.makedirs(facts_dir, exist_ok=True) + # Open files for all facts that we might emit even if we don't end up having + # anything to write to them. Souffle will complain if it can't find the file + # for anything marked as an input. + fact_files = { + fact_type.name(): + open('{}/{}.facts'.format(facts_dir, fact_type.name()), 'w') + for fact_type in Fact.__subclasses__() + } with open(edge_facts, 'w') as cfg_f: for func, cfg in sorted(cfgs.items()): joined = join_name(func, delim='.') @@ -548,12 +562,7 @@ def DumpControlFlowGraphs(cfgs: dict[str, ControlFlowGraph], for statement, facts in sorted(cfg.facts.items()): for fact in facts: # already sorted temporally - fact_f = fact_files.get(fact.name()) - if not fact_f: - fact_f = open( - '{}/{}.facts'.format(facts_dir, fact.name()), 'w') - fact_files[fact.name()] = fact_f - + fact_f = fact_files[fact.name()] fact_f.write(fact.Generate(joined, statement)) for f in fact_files.values(): @@ -561,28 +570,27 @@ def DumpControlFlowGraphs(cfgs: dict[str, ControlFlowGraph], def ComputeMinimalStackRoots(cfgs: dict[str, ControlFlowGraph], - facts_dir: str = '_tmp/mycpp-facts', - souffle_output_dir: str = '_tmp') -> StackRoots: + souffle_dir: str = '_tmp') -> StackRoots: """ Run the the souffle stack roots solver and translate its output in a format that can be queried by cppgen_pass. """ + facts_dir = '{}/facts'.format(souffle_dir) + os.makedirs(facts_dir) + output_dir = '{}/outputs'.format(souffle_dir) + os.makedirs(output_dir) DumpControlFlowGraphs(cfgs, facts_dir=facts_dir) - # Work around bug of reading truncated files from the solver? - # Could this be a ninja race condition, with a missing dependency? - # subprocess.run('sync {}/*.facts'.format(facts_dir), shell=True) - subprocess.check_call([ '_bin/datalog/dataflow', '-F', facts_dir, '-D', - souffle_output_dir, + output_dir, ]) tuples: set[tuple[SymbolPath, SymbolPath]] = set({}) - with open('{}/stack_root_vars.tsv'.format(souffle_output_dir), + with open('{}/stack_root_vars.tsv'.format(output_dir), 'r') as roots_f: pat = re.compile(r'\$(.*)\((.*), (.*)\)') for line in roots_f: From 8c8fb44fd35ef95a48d29abec43b071d191d9c47 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 26 Sep 2024 01:20:30 -0400 Subject: [PATCH 238/506] [spec/ysh-builtin-eval] Test cases for design of io->evalToDict() which will be used in Hay and the Dict function. --- core/state.py | 58 +++++++++++++++++++++++- core/value.asdl | 14 +++--- osh/cmd_eval.py | 4 +- spec/ysh-builtin-eval.test.sh | 83 ++++++++++++++++++++++++++++++++++- 4 files changed, 147 insertions(+), 12 deletions(-) diff --git a/core/state.py b/core/state.py index f4b2b535a2..19c532e162 100644 --- a/core/state.py +++ b/core/state.py @@ -1127,6 +1127,62 @@ def _MakeArgvCell(argv): return Cell(False, False, False, value.List(items)) +class ctx_FrontFrame(object): + """ + For use by io->evalToDict(), which is a primitive used for Hay and the Dict + proc + + var mutated = 'm' + var shadowed = 's' + + Dict (&d) { + shadowed = 42 + mutated = 'new' # this is equivalent to var mutated + + setvar mutated = 'new' + } + echo $shadowed # restored to 's' + echo $mutated # new + + Or maybe we disallow the setvar lookup? + """ + + def __init__(self, mem, out_dict): + # type: (Mem, Dict[str, value_t]) -> None + self.rear_frame = mem.var_stack[-1] + + # __rear__ gets a lookup rule + self.front_frame = NewDict() # type: Dict[str, Cell] + self.front_frame['__rear__'] = Cell(False, False, False, + value.Frame(self.rear_frame)) + + mem.var_stack[-1] = self.front_frame + + self.mem = mem + self.out_dict = out_dict + + def __enter__(self): + # type: () -> None + pass + + def __exit__(self, type, value, traceback): + # type: (Any, Any, Any) -> None + + for name, cell in iteritems(self.front_frame): + #log('name %r', name) + #log('cell %r', cell) + + # User can hide variables with _ suffix + # e.g. for i_ in foo bar { echo $i_ } + if name.endswith('_'): + continue + + self.out_dict[name] = cell.val + + # Restore + self.mem.var_stack[-1] = self.rear_frame + + class ctx_Eval(object): """Push temporary set of variables, $0, $1, $2, etc.""" @@ -1987,7 +2043,7 @@ def GetValue(self, name, which_scopes=scope_e.Shopt): with str_switch(name) as case: # "Registers" - if case('_status'): + if case('_status'): # deprecated in favor of _error.code return num.ToBig(self.TryStatus()) elif case('_error'): diff --git a/core/value.asdl b/core/value.asdl index 901c39a6d6..b222e39b46 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -134,11 +134,9 @@ module value # The frame MUST be lower on the stack at the time of use. | Place(y_lvalue lval, Dict[str, Cell] frame) - # TODO: Remove this, could be value.Obj - # for Flags/flag and Flags/arg? - # for json read/write ? - # Possibly unify Hay and modules/namespaces - | Module(Dict[str, value] defs) + # for io->evalToDict(), which uses ctx_FrontFrame(), which is distinct from + # ctx_Eval() + | Frame(Dict[str, Cell] bindings) # callable is vm._Callable. # TODO: ASDL needs some kind of "extern" to declare vm._Callable and @@ -154,11 +152,13 @@ module value # different @ARGV. | Proc(str name, Token name_tok, proc_sig sig, command body, - ProcDefaults? defaults, bool sh_compat) + ProcDefaults? defaults, bool sh_compat, + # module is where "global" lookups happen + Dict[str, Cell]? module_) - # module may be a frame where defined | Func(str name, Func parsed, List[value] pos_defaults, Dict[str, value] named_defaults, + # module is where "global" lookups happen Dict[str, Cell]? module_) # for i in (1:n) { echo $i } # both ends are required diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 42cd5bdc99..02f8c39141 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1301,7 +1301,7 @@ def _DoShFunction(self, node): "Function %s was already defined (redefine_proc_func)" % node.name, node.name_tok) sh_func = value.Proc(node.name, node.name_tok, proc_sig.Open, - node.body, None, True) + node.body, None, True, None) self.procs.SetShFunc(node.name, sh_func) def _DoProc(self, node): @@ -1321,7 +1321,7 @@ def _DoProc(self, node): # no dynamic scope proc = value.Proc(proc_name, node.name, node.sig, node.body, - proc_defaults, False) + proc_defaults, False, None) self.procs.SetProc(proc_name, proc) def _DoFunc(self, node): diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index f17a75c201..410e2ee87a 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 2 +## oils_failures_allowed: 4 #### Eval does not take a literal block - can restore this later @@ -338,8 +338,24 @@ one (Dict) {"code":1} ## END +#### io->evalToDict() - local and global -#### parseCommand then io.evalToDict() +# in the global frame +var d = io->evalToDict(^(var foo = 42; var bar = 'zz';)) +#pp test_ (d) + +# Same thing in a local frame +proc p (dummy) { + var d = io->evalToDict(^(var foo = 42; var bar = 'zz';)) + pp test_ (d) +} +p dummy + +## STDOUT: +## END + + +#### parseCommand then io->evalToDict() - in global scope var cmd = parseCommand('var x = 42; echo hi; var y = 99') #var cmd = parseCommand('echo hi') @@ -347,6 +363,7 @@ var cmd = parseCommand('var x = 42; echo hi; var y = 99') pp test_ (cmd) #pp asdl_ (cmd) +# problems: env var leakage var d = io->evalToDict(cmd) pp test_ (d) @@ -367,3 +384,65 @@ pp test_ (_error) ## STDOUT: (Dict) {"code":3,"message":"Syntax error in parseCommand()"} ## END + + +#### Dict (&d) { } function - local scope with __pframe__ + +# pframe is a read-only parent frame +# +# I guess we have a value.Frame() wrapper then? Why not ... + +proc Dict ( ; out; ; block) { + # Leakage: ARGV, out, block + # So we have to create a __pframe__ + + var d = io->evalToDict(block) + call out->setValue(d) +} + +# it can read f + +var myglobal = 'global' +var k = 'k-shadowed' +var k2 = 'k2-shadowed' + +Dict (&d) { + var k = 'k' + setvar k = 'k2' + + # is this in the dict? + setvar k2 = 'z' # this is in the dict! It'slocal to! + + # do we allow this? + setvar myglobal = 'global' +} + +pp test_ (d) += d + +# restored to the shadowed values +echo $k +echo $k2 + + +## STDOUT: +## END + +#### bindings created shvar persist, which is different than evalToDict() + +var a = 'a' +shvar IFS=: a='b' { + echo a=$a + inner=z + var inner2 = 'z' +} +echo a=$a +echo inner=$inner +echo inner2=$inner2 + +## STDOUT: +a=b +a=a +inner=z +inner2=z +## END From 96c111808bb0a4e5619ec3c86349a31090d59332 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 26 Sep 2024 14:33:41 -0400 Subject: [PATCH 239/506] [mycpp] Fix bug with members that are initialized by NewDict() This was tickled by ctx_FrontFrame() - which is not used yet --- mycpp/cppgen_pass.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/mycpp/cppgen_pass.py b/mycpp/cppgen_pass.py index 92b1490867..c91f75a1cc 100644 --- a/mycpp/cppgen_pass.py +++ b/mycpp/cppgen_pass.py @@ -1499,6 +1499,14 @@ def _IteratorImpl(self, o, lval, rval_type): self.def_write(';\n') self.def_write_ind('%s %s(&%s);\n', c_type, lval.name, iter_buf[0]) + def _MaybeAddMember(self, lval, current_member_vars): + if isinstance(lval.expr, NameExpr) and lval.expr.name == 'self': + #log(' lval.name %s', lval.name) + lval_type = self.types[lval] + c_type = GetCType(lval_type) + is_managed = CTypeIsManaged(c_type) + current_member_vars[lval.name] = (lval_type, c_type, is_managed) + def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: # Declare constant strings. They have to be at the top level. if self.decl and self.indent == 0 and len(o.lvalues) == 1: @@ -1578,6 +1586,10 @@ def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: if callee.name == 'NewDict': self._AssignNewDictImpl(lval) + + # Bug fix: self.front_frame = NewDict() needs to register member + if isinstance(lval, MemberExpr): + self._MaybeAddMember(lval, self.current_member_vars) return if callee.name == 'cast': @@ -1628,14 +1640,7 @@ def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: # HACK for WordParser: also include Reset(). We could change them # all up front but I kinda like this. - if (isinstance(lval.expr, NameExpr) and - lval.expr.name == 'self'): - #log(' lval.name %s', lval.name) - lval_type = self.types[lval] - c_type = GetCType(lval_type) - is_managed = CTypeIsManaged(c_type) - self.current_member_vars[lval.name] = (lval_type, c_type, - is_managed) + self._MaybeAddMember(lval, self.current_member_vars) return if isinstance(lval, IndexExpr): # a[x] = 1 From 640ac4014395fdd9f1619422569d786317491a0c Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 26 Sep 2024 16:26:59 -0400 Subject: [PATCH 240/506] [test/unit] Fix build --- core/completion_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/completion_test.py b/core/completion_test.py index c752473bdc..067474fa63 100755 --- a/core/completion_test.py +++ b/core/completion_test.py @@ -204,7 +204,7 @@ def testShellFuncExecution(self): arena=arena) node = c_parser.ParseLogicalLine() proc = value.Proc(node.name, node.name_tok, proc_sig.Open, node.body, - [], True) + [], True, None) cmd_ev = test_lib.InitCommandEvaluator(arena=arena) From b6b292fb420c166b7e0355ad3d510a6b59e75cf4 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 26 Sep 2024 19:02:21 -0400 Subject: [PATCH 241/506] [core/state refactor] name_map -> var_frame We use 'frame' in other places I also added value.Frame, which holds a Dict[str, Cell] --- builtin/method_io.py | 32 ++++------- core/state.py | 99 ++++++++++++++++++++--------------- core/value.asdl | 3 +- spec/ysh-builtin-eval.test.sh | 11 ++-- 4 files changed, 76 insertions(+), 69 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index 979f9ab0d9..164dffddc1 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -5,6 +5,7 @@ from core import error from core import num +from core import state from core import vm from mycpp.mylib import iteritems, log, NewDict from osh import prompt @@ -35,6 +36,7 @@ class Eval(vm._Callable): The CALLER must handle errors. """ + def __init__(self, cmd_ev, which): # type: (cmd_eval.CommandEvaluator, int) -> None self.cmd_ev = cmd_ev @@ -46,33 +48,16 @@ def Call(self, rd): cmd = rd.PosCommand() rd.Done() # no more args - # errors can arise from false' and 'exit' - unused_status = self.cmd_ev.EvalCommand(cmd) - if self.which == EVAL_NULL: + # errors can arise from false' and 'exit' + unused_status = self.cmd_ev.EvalCommand(cmd) return value.Null elif self.which == EVAL_DICT: - block_attrs = self.cmd_ev.mem.TopNamespace() - - # Copied from builtin/hay_ysh.py - # Hay should be rewritten with YSH reflection primitives. - # - # Hay pushes a temp frame. - # TODO: - - attrs = NewDict() # type: Dict[str, value_t] - for name, cell in iteritems(block_attrs): - #log('name %r', name) - #log('cell %r', cell) - - # User can hide variables with _ suffix - # e.g. for i_ in foo bar { echo $i_ } - if name.endswith('_'): - continue - - attrs[name] = cell.val - return value.Dict(attrs) + bindings = NewDict() # type: Dict[str, value_t] + with state.ctx_FrontFrame(self.cmd_ev.mem, bindings): + unused_status = self.cmd_ev.EvalCommand(cmd) + return value.Dict(bindings) else: raise AssertionError() @@ -135,6 +120,7 @@ def Call(self, rd): # TODO: Implement these + class Time(vm._Callable): def __init__(self): diff --git a/core/state.py b/core/state.py index 19c532e162..ce80bf57e9 100644 --- a/core/state.py +++ b/core/state.py @@ -854,7 +854,7 @@ def _InitDefaults(mem): # ' \t\n' SetGlobalString(mem, 'IFS', split.DEFAULT_IFS) - # NOTE: Should we put these in a name_map for Oil? + # NOTE: Should we put these in a var_frame for Oil? SetGlobalString(mem, 'UID', str(posix.getuid())) SetGlobalString(mem, 'EUID', str(posix.geteuid())) SetGlobalString(mem, 'PPID', str(posix.getppid())) @@ -1643,6 +1643,24 @@ def GetSpecialVar(self, op_id): # Named Vars # + def _ResolveInFrame(self, frame, name): + # type: (Dict[str, Cell], str) -> Optional[Tuple[Cell, Dict[str, Cell]]] + """ + Look in the __rear__ frame + """ + cell = frame.get(name) + if cell: + return cell, frame + + rear_val = frame.get('__rear__').val # ctx_FrontFrame() sets this + if rear_val and rear_val.tag() == value_e.Frame: + frame = cast(value.Frame, rear_val).frame + cell = frame.get(name) + if cell: + return cell, frame + + return None + def _ResolveNameOnly(self, name, which_scopes): # type: (str, scope_t) -> Tuple[Optional[Cell], Dict[str, Cell]] """Helper for getting and setting variable. @@ -1650,35 +1668,35 @@ def _ResolveNameOnly(self, name, which_scopes): Returns: cell: The cell corresponding to looking up 'name' with the given mode, or None if it's not found. - name_map: The name_map it should be set to or deleted from. + var_frame: The frame it should be set to or deleted from. """ if which_scopes == scope_e.Dynamic: for i in xrange(len(self.var_stack) - 1, -1, -1): - name_map = self.var_stack[i] - if name in name_map: - cell = name_map[name] - return cell, name_map + var_frame = self.var_stack[i] + if name in var_frame: + cell = var_frame[name] + return cell, var_frame no_cell = None # type: Optional[Cell] - return no_cell, self.var_stack[0] # set in global name_map + return no_cell, self.var_stack[0] # set in global var_frame if which_scopes == scope_e.LocalOnly: - name_map = self.var_stack[-1] - return name_map.get(name), name_map + var_frame = self.var_stack[-1] + return var_frame.get(name), var_frame if which_scopes == scope_e.GlobalOnly: - name_map = self.var_stack[0] - return name_map.get(name), name_map + var_frame = self.var_stack[0] + return var_frame.get(name), var_frame if which_scopes == scope_e.LocalOrGlobal: # Local - name_map = self.var_stack[-1] - cell = name_map.get(name) + var_frame = self.var_stack[-1] + cell = var_frame.get(name) if cell: - return cell, name_map + return cell, var_frame # Global - name_map = self.var_stack[0] - return name_map.get(name), name_map + var_frame = self.var_stack[0] + return var_frame.get(name), var_frame raise AssertionError() @@ -1693,10 +1711,10 @@ def _ResolveNameOrRef( Resolving namerefs does RECURSIVE calls. """ - cell, name_map = self._ResolveNameOnly(name, which_scopes) + cell, var_frame = self._ResolveNameOnly(name, which_scopes) if cell is None or not cell.nameref: - return cell, name_map, name # not a nameref + return cell, var_frame, name # not a nameref val = cell.val UP_val = val @@ -1708,7 +1726,7 @@ def _ResolveNameOrRef( if self.exec_opts.strict_nameref(): e_die('nameref %r is undefined' % name) else: - return cell, name_map, name # fallback + return cell, var_frame, name # fallback elif case(value_e.Str): val = cast(value.Str, UP_val) @@ -1729,7 +1747,7 @@ def _ResolveNameOrRef( # Bash has this odd behavior of clearing the nameref bit when # ref=#invalid#. strict_nameref avoids it. cell.nameref = False - return cell, name_map, name # fallback + return cell, var_frame, name # fallback # Check for circular namerefs. if ref_trail is None: @@ -1740,10 +1758,9 @@ def _ResolveNameOrRef( ref_trail.append(new_name) # 'declare -n' uses dynamic scope. - cell, name_map, cell_name = self._ResolveNameOrRef(new_name, - scope_e.Dynamic, - ref_trail=ref_trail) - return cell, name_map, cell_name + cell, var_frame, cell_name = self._ResolveNameOrRef( + new_name, scope_e.Dynamic, ref_trail=ref_trail) + return cell, var_frame, cell_name def IsBashAssoc(self, name): # type: (str) -> bool @@ -1799,8 +1816,8 @@ def SetLocalName(self, lval, val): # Equivalent to # self._ResolveNameOnly(lval.name, scope_e.LocalOnly) - name_map = self.var_stack[-1] - cell = name_map.get(lval.name) + var_frame = self.var_stack[-1] + cell = var_frame.get(lval.name) if cell: if cell.readonly: @@ -1809,13 +1826,13 @@ def SetLocalName(self, lval, val): cell.val = val # Mutate value_t else: cell = Cell(False, False, False, val) - name_map[lval.name] = cell + var_frame[lval.name] = cell def SetNamed(self, lval, val, which_scopes, flags=0): # type: (LeftName, value_t, scope_t, int) -> None if flags & SetNameref or flags & ClearNameref: # declare -n ref=x # refers to the ref itself - cell, name_map = self._ResolveNameOnly(lval.name, which_scopes) + cell, var_frame = self._ResolveNameOnly(lval.name, which_scopes) cell_name = lval.name else: # ref=x # mutates THROUGH the reference @@ -1826,7 +1843,7 @@ def SetNamed(self, lval, val, which_scopes, flags=0): # BracedVarSub # 3. Turn BracedVarSub into an sh_lvalue, and call # self.unsafe_arith.SetValue() wrapper with ref_trail - cell, name_map, cell_name = self._ResolveNameOrRef( + cell, var_frame, cell_name = self._ResolveNameOrRef( lval.name, which_scopes) if cell: @@ -1863,7 +1880,7 @@ def SetNamed(self, lval, val, which_scopes, flags=0): cell = Cell(bool(flags & SetExport), bool(flags & SetReadOnly), bool(flags & SetNameref), val) - name_map[cell_name] = cell + var_frame[cell_name] = cell # Maintain invariant that only strings and undefined cells can be # exported. @@ -1929,10 +1946,10 @@ def SetValue(self, lval, val, which_scopes, flags=0): # bash/mksh have annoying behavior of letting you do LHS assignment to # Undef, which then turns into an INDEXED array. (Undef means that set # -o nounset fails.) - cell, name_map, _ = self._ResolveNameOrRef( + cell, var_frame, _ = self._ResolveNameOrRef( lval.name, which_scopes) if not cell: - self._BindNewArrayWithEntry(name_map, lval, rval, flags) + self._BindNewArrayWithEntry(var_frame, lval, rval, flags) return if cell.readonly: @@ -1942,7 +1959,7 @@ def SetValue(self, lval, val, which_scopes, flags=0): # undef[0]=y is allowed with tagswitch(UP_cell_val) as case2: if case2(value_e.Undef): - self._BindNewArrayWithEntry(name_map, lval, rval, + self._BindNewArrayWithEntry(var_frame, lval, rval, flags) return @@ -1991,7 +2008,7 @@ def SetValue(self, lval, val, which_scopes, flags=0): left_loc = lval.blame_loc - cell, name_map, _ = self._ResolveNameOrRef( + cell, var_frame, _ = self._ResolveNameOrRef( lval.name, which_scopes) if cell.readonly: e_die("Can't assign to readonly associative array", @@ -2006,9 +2023,9 @@ def SetValue(self, lval, val, which_scopes, flags=0): else: raise AssertionError(lval.tag()) - def _BindNewArrayWithEntry(self, name_map, lval, val, flags): + def _BindNewArrayWithEntry(self, var_frame, lval, val, flags): # type: (Dict[str, Cell], sh_lvalue.Indexed, value.Str, int) -> None - """Fill 'name_map' with a new indexed array entry.""" + """Fill 'var_frame' with a new indexed array entry.""" no_str = None # type: Optional[str] items = [no_str] * lval.index items.append(val.s) @@ -2016,7 +2033,7 @@ def _BindNewArrayWithEntry(self, name_map, lval, val, flags): # arrays can't be exported; can't have BashAssoc flag readonly = bool(flags & SetReadOnly) - name_map[lval.name] = Cell(False, readonly, False, new_value) + var_frame[lval.name] = Cell(False, readonly, False, new_value) def InternalSetGlobal(self, name, new_val): # type: (str, value_t) -> None @@ -2232,7 +2249,7 @@ def Unset(self, lval, which_scopes): if which_scopes == scope_e.Shopt: which_scopes = self.ScopesForWriting() - cell, name_map, cell_name = self._ResolveNameOrRef( + cell, var_frame, cell_name = self._ResolveNameOrRef( var_name, which_scopes) if not cell: return False # 'unset' builtin falls back on functions @@ -2243,10 +2260,10 @@ def Unset(self, lval, which_scopes): if case(sh_lvalue_e.Var): # unset x # Make variables in higher scopes visible. # example: test/spec.sh builtin-vars -r 24 (ble.sh) - mylib.dict_erase(name_map, cell_name) + mylib.dict_erase(var_frame, cell_name) # alternative that some shells use: - # name_map[cell_name].val = value.Undef + # var_frame[cell_name].val = value.Undef # cell.exported = False # This should never happen because we do recursive lookups of namerefs. @@ -2322,7 +2339,7 @@ def ClearFlag(self, name, flag): We don't use SetValue() because even if rval is None, it will make an Undef value in a scope. """ - cell, name_map = self._ResolveNameOnly(name, self.ScopesForReading()) + cell, var_frame = self._ResolveNameOnly(name, self.ScopesForReading()) if cell: if flag & ClearExport: cell.exported = False diff --git a/core/value.asdl b/core/value.asdl index b222e39b46..b1d38dc3e4 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -136,7 +136,8 @@ module value # for io->evalToDict(), which uses ctx_FrontFrame(), which is distinct from # ctx_Eval() - | Frame(Dict[str, Cell] bindings) + # TODO: ASDL should let us "collapse" this Dict directly into value_t + | Frame(Dict[str, Cell] frame) # callable is vm._Callable. # TODO: ASDL needs some kind of "extern" to declare vm._Callable and diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 410e2ee87a..7a2adcb991 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -340,21 +340,24 @@ one #### io->evalToDict() - local and global +var g = 'global' + # in the global frame -var d = io->evalToDict(^(var foo = 42; var bar = 'zz';)) -#pp test_ (d) +var d = io->evalToDict(^(var foo = 42; var bar = g;)) +pp test_ (d) # Same thing in a local frame proc p (dummy) { - var d = io->evalToDict(^(var foo = 42; var bar = 'zz';)) + var d = io->evalToDict(^(var foo = 42; var bar = g;)) pp test_ (d) } p dummy ## STDOUT: +(Dict) {"foo":42,"bar":"zz"} +(Dict) {"foo":42,"bar":"zz"} ## END - #### parseCommand then io->evalToDict() - in global scope var cmd = parseCommand('var x = 42; echo hi; var y = 99') From 01e4d7b4b1c2342c190285737922ecb3012d200a Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 26 Sep 2024 19:20:30 -0400 Subject: [PATCH 242/506] [ysh] Implement the front/rear frame idea for io->evalToDict() This lets us to the following: var outer = 3 Dict (&d) { # Dict uses io->evalToDict() a = 42 b = 99 + outer # reference rear frame } # the front frame as a dict pp (d) # => {a: 42, b: 102} The rear frame is differen than the frame "below", which is dynamic scope. This slows down variable lookups a bit, in theory. Let's see the measurements in practice. --- builtin/method_io.py | 2 +- core/state.py | 52 +++++++++++++++++++++-------------- spec/ysh-builtin-eval.test.sh | 19 +++++++++---- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index 164dffddc1..4ed8eafead 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -7,7 +7,7 @@ from core import num from core import state from core import vm -from mycpp.mylib import iteritems, log, NewDict +from mycpp.mylib import log, NewDict from osh import prompt from typing import Dict, TYPE_CHECKING diff --git a/core/state.py b/core/state.py index ce80bf57e9..938cfbc7c2 100644 --- a/core/state.py +++ b/core/state.py @@ -1643,23 +1643,25 @@ def GetSpecialVar(self, op_id): # Named Vars # - def _ResolveInFrame(self, frame, name): - # type: (Dict[str, Cell], str) -> Optional[Tuple[Cell, Dict[str, Cell]]] + def _FrameLookup(self, frame, name): + # type: (Dict[str, Cell], str) -> Tuple[Optional[Cell], Dict[str, Cell]] """ - Look in the __rear__ frame + Look in the frame itself, then the __rear__ frame if it exists """ cell = frame.get(name) if cell: return cell, frame - rear_val = frame.get('__rear__').val # ctx_FrontFrame() sets this - if rear_val and rear_val.tag() == value_e.Frame: - frame = cast(value.Frame, rear_val).frame - cell = frame.get(name) - if cell: - return cell, frame + rear_cell = frame.get('__rear__') # ctx_FrontFrame() sets this + if rear_cell: + rear_val = rear_cell.val + if rear_val and rear_val.tag() == value_e.Frame: + frame = cast(value.Frame, rear_val).frame + cell = frame.get(name) + if cell: + return cell, frame - return None + return None, None def _ResolveNameOnly(self, name, which_scopes): # type: (str, scope_t) -> Tuple[Optional[Cell], Dict[str, Cell]] @@ -1673,30 +1675,40 @@ def _ResolveNameOnly(self, name, which_scopes): if which_scopes == scope_e.Dynamic: for i in xrange(len(self.var_stack) - 1, -1, -1): var_frame = self.var_stack[i] - if name in var_frame: - cell = var_frame[name] - return cell, var_frame - no_cell = None # type: Optional[Cell] - return no_cell, self.var_stack[0] # set in global var_frame + cell, result_frame = self._FrameLookup(var_frame, name) + if cell: + return cell, result_frame + return None, self.var_stack[0] # set in global var_frame if which_scopes == scope_e.LocalOnly: var_frame = self.var_stack[-1] - return var_frame.get(name), var_frame + cell, result_frame = self._FrameLookup(var_frame, name) + if cell: + return cell, result_frame + return None, var_frame if which_scopes == scope_e.GlobalOnly: var_frame = self.var_stack[0] - return var_frame.get(name), var_frame + cell, result_frame = self._FrameLookup(var_frame, name) + if cell: + return cell, result_frame + + return None, var_frame if which_scopes == scope_e.LocalOrGlobal: # Local var_frame = self.var_stack[-1] - cell = var_frame.get(name) + cell, result_frame = self._FrameLookup(var_frame, name) if cell: - return cell, var_frame + return cell, result_frame # Global var_frame = self.var_stack[0] - return var_frame.get(name), var_frame + cell, result_frame = self._FrameLookup(var_frame, name) + if cell: + return cell, result_frame + + return None, var_frame raise AssertionError() diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 7a2adcb991..19f6ec6d89 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 4 +## oils_failures_allowed: 3 #### Eval does not take a literal block - can restore this later @@ -347,15 +347,22 @@ var d = io->evalToDict(^(var foo = 42; var bar = g;)) pp test_ (d) # Same thing in a local frame -proc p (dummy) { - var d = io->evalToDict(^(var foo = 42; var bar = g;)) +proc p (myparam) { + var mylocal = 'local' + var cmd = ^( + var foo = 42 + var g = "-$g" + var p = "-$myparam" + var L = "-$mylocal" + ) + var d = io->evalToDict(cmd) pp test_ (d) } -p dummy +p param ## STDOUT: -(Dict) {"foo":42,"bar":"zz"} -(Dict) {"foo":42,"bar":"zz"} +(Dict) {"foo":42,"bar":"global"} +(Dict) {"foo":42,"g":"-global","p":"-param","L":"-local"} ## END #### parseCommand then io->evalToDict() - in global scope From 0cebaa911f042f976274d65b808bf86ad8ccaffc Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 26 Sep 2024 20:45:02 -0400 Subject: [PATCH 243/506] [core/state refactor] Extract free function And reduce List copies in Procs::GetNames() --- core/state.py | 62 ++++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/core/state.py b/core/state.py index 938cfbc7c2..ab7d7c172f 100644 --- a/core/state.py +++ b/core/state.py @@ -1242,6 +1242,27 @@ def _Pop(self): self.mem.SetNamed(lval, old_val, scope_e.LocalOnly) +def _FrameLookup(frame, name): + # type: (Dict[str, Cell], str) -> Tuple[Optional[Cell], Dict[str, Cell]] + """ + Look in the frame itself, then the __rear__ frame if it exists + """ + cell = frame.get(name) + if cell: + return cell, frame + + rear_cell = frame.get('__rear__') # ctx_FrontFrame() sets this + if rear_cell: + rear_val = rear_cell.val + if rear_val and rear_val.tag() == value_e.Frame: + frame = cast(value.Frame, rear_val).frame + cell = frame.get(name) + if cell: + return cell, frame + + return None, None + + class Mem(object): """For storing variables. @@ -1643,26 +1664,6 @@ def GetSpecialVar(self, op_id): # Named Vars # - def _FrameLookup(self, frame, name): - # type: (Dict[str, Cell], str) -> Tuple[Optional[Cell], Dict[str, Cell]] - """ - Look in the frame itself, then the __rear__ frame if it exists - """ - cell = frame.get(name) - if cell: - return cell, frame - - rear_cell = frame.get('__rear__') # ctx_FrontFrame() sets this - if rear_cell: - rear_val = rear_cell.val - if rear_val and rear_val.tag() == value_e.Frame: - frame = cast(value.Frame, rear_val).frame - cell = frame.get(name) - if cell: - return cell, frame - - return None, None - def _ResolveNameOnly(self, name, which_scopes): # type: (str, scope_t) -> Tuple[Optional[Cell], Dict[str, Cell]] """Helper for getting and setting variable. @@ -1675,21 +1676,21 @@ def _ResolveNameOnly(self, name, which_scopes): if which_scopes == scope_e.Dynamic: for i in xrange(len(self.var_stack) - 1, -1, -1): var_frame = self.var_stack[i] - cell, result_frame = self._FrameLookup(var_frame, name) + cell, result_frame = _FrameLookup(var_frame, name) if cell: return cell, result_frame return None, self.var_stack[0] # set in global var_frame if which_scopes == scope_e.LocalOnly: var_frame = self.var_stack[-1] - cell, result_frame = self._FrameLookup(var_frame, name) + cell, result_frame = _FrameLookup(var_frame, name) if cell: return cell, result_frame return None, var_frame if which_scopes == scope_e.GlobalOnly: var_frame = self.var_stack[0] - cell, result_frame = self._FrameLookup(var_frame, name) + cell, result_frame = _FrameLookup(var_frame, name) if cell: return cell, result_frame @@ -1698,13 +1699,13 @@ def _ResolveNameOnly(self, name, which_scopes): if which_scopes == scope_e.LocalOrGlobal: # Local var_frame = self.var_stack[-1] - cell, result_frame = self._FrameLookup(var_frame, name) + cell, result_frame = _FrameLookup(var_frame, name) if cell: return cell, result_frame # Global var_frame = self.var_stack[0] - cell, result_frame = self._FrameLookup(var_frame, name) + cell, result_frame = _FrameLookup(var_frame, name) if cell: return cell, result_frame @@ -2508,15 +2509,16 @@ def Del(self, to_del): def GetNames(self): # type: () -> List[str] """Returns a *sorted* list of all proc names""" - names = list(self.sh_funcs.keys()) + names = self.sh_funcs.keys() - vars = self.mem.var_stack[0] - for name in vars: - cell = vars[name] + var_frame = self.mem.var_stack[0] + for name in var_frame: + cell = var_frame[name] if cell.val.tag() == value_e.Proc: names.append(name) - return sorted(names) + names.sort() + return names # From 8e175027896a4d83ddc014fa08c598bfedf239ce Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 26 Sep 2024 21:48:47 -0400 Subject: [PATCH 244/506] [test/spec] setvar within Dict (&d) is a bit odd Add some test cases and explanation Also add spec/ysh-module, for 2 things I want to change - YSH builtins go in a separate namespace - env vars aren't in YSH by default --- builtin/io_ysh.py | 5 ++ core/state.py | 3 +- spec/ysh-builtin-eval.test.sh | 104 +++++++++++++++++++++++++++++----- spec/ysh-module.test.sh | 27 +++++++++ test/spec.sh | 4 ++ 5 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 spec/ysh-module.test.sh diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 0f8aa4a5c2..38806e1075 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -190,6 +190,11 @@ def Run(self, cmd_val): return 0 + if action == 'frame_vars_': # Print names in current frame, for testing + top = self.mem.var_stack[-1] + print('\tframe_vars_: %s' % ' '.join(top.keys())) + return 0 + if action == 'gc-stats_': print('TODO') return 0 diff --git a/core/state.py b/core/state.py index ab7d7c172f..d683fd1653 100644 --- a/core/state.py +++ b/core/state.py @@ -1258,7 +1258,8 @@ def _FrameLookup(frame, name): frame = cast(value.Frame, rear_val).frame cell = frame.get(name) if cell: - return cell, frame + #return cell, frame + return cell, None return None, None diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 19f6ec6d89..d99058d735 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 3 +## oils_failures_allowed: 4 #### Eval does not take a literal block - can restore this later @@ -367,13 +367,13 @@ p param #### parseCommand then io->evalToDict() - in global scope -var cmd = parseCommand('var x = 42; echo hi; var y = 99') +var g = 'global' +var cmd = parseCommand('var x = 42; echo hi; var y = g') #var cmd = parseCommand('echo hi') pp test_ (cmd) #pp asdl_ (cmd) -# problems: env var leakage var d = io->evalToDict(cmd) pp test_ (d) @@ -381,7 +381,7 @@ pp test_ (d) ## STDOUT: hi -(Dict) +(Dict) {"x":42,"y":"global"} ## END #### parseCommand with syntax error @@ -396,16 +396,13 @@ pp test_ (_error) ## END -#### Dict (&d) { } function - local scope with __pframe__ +#### Dict (&d) { ... } converts frame to dict # pframe is a read-only parent frame # # I guess we have a value.Frame() wrapper then? Why not ... proc Dict ( ; out; ; block) { - # Leakage: ARGV, out, block - # So we have to create a __pframe__ - var d = io->evalToDict(block) call out->setValue(d) } @@ -417,22 +414,99 @@ var k = 'k-shadowed' var k2 = 'k2-shadowed' Dict (&d) { - var k = 'k' - setvar k = 'k2' + var k = 'k-block' + setvar k = 'k-block-mutated' - # is this in the dict? - setvar k2 = 'z' # this is in the dict! It'slocal to! + # this is confusing + # because it doesn't find it in the local stack frame + # it doesn't have 'var without setvar' bug + setvar k2 = 'k2-block' # this is in the dict! It'slocal to! + setvar k3 = 'k3' # do we allow this? setvar myglobal = 'global' } pp test_ (d) -= d # restored to the shadowed values -echo $k -echo $k2 +echo k=$k +echo k2=$k2 + +proc p { + Dict (&d) { + var k = 'k-proc' + setvar k = 'k-proc-mutated' + + # is this in the dict? + setvar k2 = 'k2-proc' # this is in the dict! It'slocal to! + } +} + +## STDOUT: +## END + +#### Dict (&d) and setvar + +proc Dict ( ; out; ; block) { + var d = io->evalToDict(block) + + echo 'proc Dict frame after evalToDict' + pp frame_vars_ + + echo "Dict outer=$outer" + #echo "Dict outer2=$outer2" + call out->setValue(d) +} + +var outer = 'xx' + +Dict (&d) { + # new variable in the front frame + outer2 = 'outer2' + + #var v = 'v' + #setvar v = 'v-mutated' + + # hm setvar is local ONLY, so it does NOT find the 'outer' + # because we're inside Dict! Gah + # + # Do we want to say there's no matching 'var', instead of mutating locally? + # + # And also plain io->eval() should be able to mutate outer... + setvar outer = 'zz' + + setvar not_declared = 'yy' + + echo 'inside Dict block' + pp frame_vars_ +} + +pp test_ (d) +echo after outer=$outer + +echo 'after Dict' +pp frame_vars_ + +## STDOUT: +## END + + +#### Dict (&d) and setglobal + +proc Dict ( ; out; ; block) { + var d = io->evalToDict(block) + call out->setValue(d) +} + +var outer = 'xx' + +Dict (&d) { + setglobal outer = 'zz' +} + +pp test_ (d) +echo outer=$outer ## STDOUT: diff --git a/spec/ysh-module.test.sh b/spec/ysh-module.test.sh new file mode 100644 index 0000000000..65d3fe9893 --- /dev/null +++ b/spec/ysh-module.test.sh @@ -0,0 +1,27 @@ +## our_shell: ysh +## oils_failures_allowed: 2 + +#### global frame doesn't contain builtins like len(), dict() + +try { + pp frame_vars_ | grep -o len +} +pp test_ (_pipeline_status) + +## STDOUT: +(List) [0,1] +## END + +#### global frame doesn't contain env vars + +try { + pp frame_vars_ | grep -o TMP +} +pp test_ (_pipeline_status) + + +## STDOUT: +(List) [0,1] +## END + + diff --git a/test/spec.sh b/test/spec.sh index 84dfe980db..fcf2364cac 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -834,6 +834,10 @@ ysh-method-io() { run-file ysh-method-io "$@" } +ysh-module() { + run-file ysh-module "$@" +} + ysh-object() { run-file ysh-object "$@" } From 14e91b3b763b9a46005eb9e62249e7f5ab998499 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 26 Sep 2024 23:56:07 -0400 Subject: [PATCH 245/506] [demo] Test out long prompt with plain Python module Calling raw_input() doesn't exhibit the bug - it erases the prompt correctly Related to issue #2081 --- demo/cpython/readline_mod.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/demo/cpython/readline_mod.py b/demo/cpython/readline_mod.py index ba12f34d64..1dd1630f75 100755 --- a/demo/cpython/readline_mod.py +++ b/demo/cpython/readline_mod.py @@ -15,11 +15,15 @@ def main(argv): + try: + prompt_str = argv[1] + except IndexError: + prompt_str = '! ' import os readline.parse_and_bind("tab: complete") print('PID %d' % os.getpid()) while True: - x = raw_input('! ') + x = raw_input(prompt_str) print(x) From 62b57aae2c28e33589fc592cc12869e33c79d7f8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 00:19:08 -0400 Subject: [PATCH 246/506] [test] More repro for Ctrl-R history bug This is issue #2081 It's due to GNU readline horizontal-scroll-mode Unrelated: cases in spec/builtin-eval-test, with pp frame_vars_ --- builtin/io_ysh.py | 2 +- frontend/py_readline.py | 13 +++++++++++++ spec/ysh-builtin-eval.test.sh | 25 +++++++++++++++---------- test/bugs.sh | 4 ++++ 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 38806e1075..95a71bdc4d 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -192,7 +192,7 @@ def Run(self, cmd_val): if action == 'frame_vars_': # Print names in current frame, for testing top = self.mem.var_stack[-1] - print('\tframe_vars_: %s' % ' '.join(top.keys())) + print(' [frame_vars_] %s' % ' '.join(top.keys())) return 0 if action == 'gc-stats_': diff --git a/frontend/py_readline.py b/frontend/py_readline.py index 9bc0437aba..dc092bf547 100644 --- a/frontend/py_readline.py +++ b/frontend/py_readline.py @@ -102,3 +102,16 @@ def MaybeGetReadline(): return Readline() return None + + +if __name__ == '__main__': + import sys + line_input = MaybeGetReadline() + try: + prompt_str = sys.argv[1] + except IndexError: + prompt_str = '! ' + + while True: + x = line_input.prompt_input(prompt_str) + print(x) diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index d99058d735..a299fe2e1a 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 4 +## oils_failures_allowed: 3 #### Eval does not take a literal block - can restore this later @@ -398,10 +398,6 @@ pp test_ (_error) #### Dict (&d) { ... } converts frame to dict -# pframe is a read-only parent frame -# -# I guess we have a value.Frame() wrapper then? Why not ... - proc Dict ( ; out; ; block) { var d = io->evalToDict(block) call out->setValue(d) @@ -420,7 +416,7 @@ Dict (&d) { # this is confusing # because it doesn't find it in the local stack frame # it doesn't have 'var without setvar' bug - setvar k2 = 'k2-block' # this is in the dict! It'slocal to! + setvar k2 = 'k2-block' # global, so not checked setvar k3 = 'k3' # do we allow this? @@ -439,7 +435,7 @@ proc p { setvar k = 'k-proc-mutated' # is this in the dict? - setvar k2 = 'k2-proc' # this is in the dict! It'slocal to! + setvar k2 = 'k2-proc' # local, so it's checked } } @@ -499,17 +495,26 @@ proc Dict ( ; out; ; block) { call out->setValue(d) } -var outer = 'xx' +var g = 'xx' Dict (&d) { - setglobal outer = 'zz' + setglobal g = 'zz' + + a = 42 + pp frame_vars_ } +echo pp test_ (d) -echo outer=$outer +echo g=$g +#pp frame_vars_ ## STDOUT: + [frame_vars_] __rear__ a + +(Dict) {"a":42} +g=zz ## END #### bindings created shvar persist, which is different than evalToDict() diff --git a/test/bugs.sh b/test/bugs.sh index 3340c281f7..b400ab3e24 100755 --- a/test/bugs.sh +++ b/test/bugs.sh @@ -206,4 +206,8 @@ bug-2078() { } | _bin/cxx-asan/ysh } +py-readline() { + PYTHONPATH=.:vendor python2 frontend/py_readline.py "$@" +} + "$@" From 1cf4068bc8db9b52f0598570c8979bdf558fd437 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 02:05:43 -0400 Subject: [PATCH 247/506] [types] Fix build --- frontend/py_readline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/py_readline.py b/frontend/py_readline.py index dc092bf547..1339e6d35e 100644 --- a/frontend/py_readline.py +++ b/frontend/py_readline.py @@ -106,12 +106,12 @@ def MaybeGetReadline(): if __name__ == '__main__': import sys - line_input = MaybeGetReadline() + readline = MaybeGetReadline() try: prompt_str = sys.argv[1] except IndexError: prompt_str = '! ' while True: - x = line_input.prompt_input(prompt_str) + x = readline.prompt_input(prompt_str) print(x) From 6447c47c5051bfc692de38d4c203423ca19d6294 Mon Sep 17 00:00:00 2001 From: Aidan <46799759+PossiblyAShrub@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:54:52 -0600 Subject: [PATCH 248/506] [ysh] Define procs in the current scope, not the global one (#2077) Still needs a spec test --- core/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state.py b/core/state.py index d683fd1653..e087434d08 100644 --- a/core/state.py +++ b/core/state.py @@ -2480,7 +2480,7 @@ def __init__(self, mem): def SetProc(self, name, proc): # type: (str, value.Proc) -> None - self.mem.var_stack[0][name] = Cell(False, False, False, proc) + self.mem.var_stack[-1][name] = Cell(False, False, False, proc) def SetShFunc(self, name, proc): # type: (str, value.Proc) -> None From 4bb4510a4d63b5fd9e9d838490c71010a002e00f Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 11:26:10 -0400 Subject: [PATCH 249/506] [spec/ysh-proc-meta] Different ways to create procs dynamically 1. shell eval builtin 2. io->eval() 3. eval () or io->eval() with vars={out_dict: {}} 4. io->evalToDict() 5. reflection via __invoke__ - no parsing, not implemented Inspired by discussion / use cases from Zulip: https://oilshell.zulipchat.com/#narrow/stream/384942-language-design/topic/Metaprogramming.20in.20ysh.2C.20sort.20of It also relates to Hay, which is the "straggler" I identified. --- spec/ysh-proc-meta.test.sh | 178 +++++++++++++++++++++++++++++++++++++ spec/ysh-proc.test.sh | 25 ++++++ test/spec.sh | 4 + 3 files changed, 207 insertions(+) create mode 100644 spec/ysh-proc-meta.test.sh diff --git a/spec/ysh-proc-meta.test.sh b/spec/ysh-proc-meta.test.sh new file mode 100644 index 0000000000..ba25858554 --- /dev/null +++ b/spec/ysh-proc-meta.test.sh @@ -0,0 +1,178 @@ +## oils_failures_allowed: 1 +## our_shell: ysh + +# dynamically generate procs + +#### with eval builtin command, in global scope + +for param in a b { + eval """ + proc echo_$param(prefix) { + echo \$prefix $param + } + """ +} + +echo_a prefix +echo_b prefix + +## STDOUT: +prefix a +prefix b +## END + +#### with eval builtin command, in local scope + +proc p { + for param in a b { + eval """ + proc echo_$param(prefix) { + echo \$prefix $param + } + """ + } + + echo_a prefix + echo_b prefix +} + +p + +echo_a prefix # not available here! + +## status: 127 +## STDOUT: +prefix a +prefix b +## END + +#### with parseCommand() then io->eval(), in local scope + +proc p { + var result = {} + for param in a b { + var s = """ + proc echo_$param(prefix) { + echo \$prefix $param + } + """ + var cmd = parseCommand(s) + call io->eval(cmd) + } + + echo_a prefix + echo_b prefix +} + +p + +echo_a prefix + +## status: 127 +## STDOUT: +prefix a +prefix b +## END + +#### with parseCommand() then eval vars={out_dict: {}} + +# This could take the place of evalToDict()? But evalToDict() is useful in +# Hay? + +func genProcs() { + var vars = {out_dict: {}} + for param in a b { + var s = """ + proc echo_$param(prefix) { + echo \$prefix $param + } + setvar out_dict.echo_$param = echo_$param + """ + var cmd = parseCommand(s) + + # TODO: io->eval() should support vars=vars + #call io->eval(cmd) + eval (cmd, vars=vars) + } + return (vars.out_dict) +} + +var procs = genProcs() + +var my_echo_a = procs.echo_a +var my_echo_b = procs.echo_b + +my_echo_a prefix +my_echo_b prefix + +## STDOUT: +prefix a +prefix b +## END + +#### with evalToDict() + +func genProcs() { + var result = {} + for param in a b { + var s = """ + # This is defined locally + proc echo_$param(prefix) { + echo \$prefix $param + } + if false { + = echo_$param + var a = 42 + pp frame_vars_ + } + """ + var cmd = parseCommand(s) + + var d = io->evalToDict(cmd) + + # accumulate + setvar result["echo_$param"] = d["echo_$param"] + } + return (result) +} + +var procs = genProcs() + +var my_echo_a = procs.echo_a +var my_echo_b = procs.echo_b + +my_echo_a prefix +my_echo_b prefix + +## STDOUT: +prefix a +prefix b +## END + + +#### with runtime REFLECTION via __invoke__ - no parsing + +# self is the first typed arg +proc p (prefix; self) { + echo $prefix $[self.param] +} + +# p is invoked with "self", which has self.param +var methods = Object(null, {__invoke__: p}) + +var procs = {} +for param in a b { + setvar procs["echo_$param"] = Object(methods, {param: param}) +} + +var my_echo_a = procs.echo_a +var my_echo_b = procs.echo_b + +# Maybe show an error if this is not value.Obj? +my_echo_a prefix +my_echo_b prefix + +## STDOUT: +prefix a +prefix b +## END diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 3b996e2234..a611aca66b 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -556,3 +556,28 @@ foo bar foo ## END + + +#### procs are defined in local scope +shopt -s ysh:upgrade + +proc gen-proc { + eval 'proc localproc { echo hi }' + pp frame_vars_ + +} + +gen-proc + +# can't suppress 'grep' failure +if false { + try { + pp frame_vars_ | grep localproc + } + pp test_ (_pipeline_status) + #pp test_ (PIPESTATUS) +} + +## STDOUT: + [frame_vars_] ARGV localproc +## END diff --git a/test/spec.sh b/test/spec.sh index fcf2364cac..0eccceab62 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -882,6 +882,10 @@ ysh-proc() { run-file ysh-proc "$@" } +ysh-proc-meta() { + run-file ysh-proc-meta "$@" +} + ysh-regex() { run-file ysh-regex "$@" } From 9ff1b50de56c63d94951bdd5fa74e132a58d959c Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 13:30:19 -0400 Subject: [PATCH 250/506] [ysh] Move builtins like len() out of global namespace And into its own __builtins__ module, similar to Python (both 2 and 3) This prepares for modules with namespaces. It also makes it easier to write tests with 'pp frame_vars_' --- core/shell.py | 106 +++++++++++++++++++--------------------- core/state.py | 20 +++++++- spec/ysh-module.test.sh | 32 +++++++++++- spec/ysh-scope.test.sh | 24 ++++----- 4 files changed, 113 insertions(+), 69 deletions(-) diff --git a/core/shell.py b/core/shell.py index 59e3b3d435..1a90f65124 100644 --- a/core/shell.py +++ b/core/shell.py @@ -201,13 +201,10 @@ def OnChange(self, opt0_array, opt_name, b): return True -def _SetGlobalFunc(mem, name, func): +def _AddBuiltinFunc(mem, name, func): # type: (state.Mem, str, vm._Callable) -> None assert isinstance(func, vm._Callable), func - - # Note: no location info for builtin functions? - mem.SetNamed(location.LName(name), value.BuiltinFunc(func), - scope_e.GlobalOnly) + mem.AddBuiltin(name, value.BuiltinFunc(func)) def InitAssignmentBuiltins( @@ -848,74 +845,73 @@ def Main( eval_hay = func_hay.EvalHay(hay_state, mutable_opts, mem, cmd_ev) hay_func = func_hay.HayFunc(hay_state) - _SetGlobalFunc(mem, 'parseHay', parse_hay) - _SetGlobalFunc(mem, 'evalHay', eval_hay) - _SetGlobalFunc(mem, '_hay', hay_func) + _AddBuiltinFunc(mem, 'parseHay', parse_hay) + _AddBuiltinFunc(mem, 'evalHay', eval_hay) + _AddBuiltinFunc(mem, '_hay', hay_func) - _SetGlobalFunc(mem, 'len', func_misc.Len()) - _SetGlobalFunc(mem, 'type', func_misc.Type()) + _AddBuiltinFunc(mem, 'len', func_misc.Len()) + _AddBuiltinFunc(mem, 'type', func_misc.Type()) g = func_eggex.MatchFunc(func_eggex.G, expr_ev, mem) - _SetGlobalFunc(mem, '_group', g) - _SetGlobalFunc(mem, '_match', g) # TODO: remove this backward compat alias - _SetGlobalFunc(mem, '_start', func_eggex.MatchFunc(func_eggex.S, None, - mem)) - _SetGlobalFunc(mem, '_end', func_eggex.MatchFunc(func_eggex.E, None, mem)) - - _SetGlobalFunc(mem, 'parseCommand', - func_reflect.ParseCommand(parse_ctx, errfmt)) - _SetGlobalFunc(mem, 'parseExpr', func_reflect.ParseExpr(parse_ctx, errfmt)) - _SetGlobalFunc(mem, 'evalExpr', func_reflect.EvalExpr(expr_ev)) - - _SetGlobalFunc(mem, 'shvarGet', func_reflect.Shvar_get(mem)) - _SetGlobalFunc(mem, 'getVar', func_reflect.GetVar(mem)) - - _SetGlobalFunc(mem, 'Object', func_misc.Object()) - _SetGlobalFunc(mem, 'prototype', func_misc.Prototype()) - _SetGlobalFunc(mem, 'propView', func_misc.PropView()) + _AddBuiltinFunc(mem, '_group', g) + _AddBuiltinFunc(mem, '_match', + g) # TODO: remove this backward compat alias + _AddBuiltinFunc(mem, '_start', + func_eggex.MatchFunc(func_eggex.S, None, mem)) + _AddBuiltinFunc(mem, '_end', func_eggex.MatchFunc(func_eggex.E, None, mem)) + + _AddBuiltinFunc(mem, 'parseCommand', + func_reflect.ParseCommand(parse_ctx, errfmt)) + _AddBuiltinFunc(mem, 'parseExpr', + func_reflect.ParseExpr(parse_ctx, errfmt)) + _AddBuiltinFunc(mem, 'evalExpr', func_reflect.EvalExpr(expr_ev)) + + _AddBuiltinFunc(mem, 'shvarGet', func_reflect.Shvar_get(mem)) + _AddBuiltinFunc(mem, 'getVar', func_reflect.GetVar(mem)) + + _AddBuiltinFunc(mem, 'Object', func_misc.Object()) + _AddBuiltinFunc(mem, 'prototype', func_misc.Prototype()) + _AddBuiltinFunc(mem, 'propView', func_misc.PropView()) # type conversions - _SetGlobalFunc(mem, 'bool', func_misc.Bool()) - _SetGlobalFunc(mem, 'int', func_misc.Int()) - _SetGlobalFunc(mem, 'float', func_misc.Float()) - _SetGlobalFunc(mem, 'str', func_misc.Str_()) - _SetGlobalFunc(mem, 'list', func_misc.List_()) - _SetGlobalFunc(mem, 'dict', func_misc.DictFunc()) - - _SetGlobalFunc(mem, 'runes', func_misc.Runes()) - _SetGlobalFunc(mem, 'encodeRunes', func_misc.EncodeRunes()) - _SetGlobalFunc(mem, 'bytes', func_misc.Bytes()) - _SetGlobalFunc(mem, 'encodeBytes', func_misc.EncodeBytes()) + _AddBuiltinFunc(mem, 'bool', func_misc.Bool()) + _AddBuiltinFunc(mem, 'int', func_misc.Int()) + _AddBuiltinFunc(mem, 'float', func_misc.Float()) + _AddBuiltinFunc(mem, 'str', func_misc.Str_()) + _AddBuiltinFunc(mem, 'list', func_misc.List_()) + _AddBuiltinFunc(mem, 'dict', func_misc.DictFunc()) + + _AddBuiltinFunc(mem, 'runes', func_misc.Runes()) + _AddBuiltinFunc(mem, 'encodeRunes', func_misc.EncodeRunes()) + _AddBuiltinFunc(mem, 'bytes', func_misc.Bytes()) + _AddBuiltinFunc(mem, 'encodeBytes', func_misc.EncodeBytes()) # Str - #_SetGlobalFunc(mem, 'strcmp', None) + #_AddBuiltinFunc(mem, 'strcmp', None) # TODO: This should be Python style splitting - _SetGlobalFunc(mem, 'split', func_misc.Split(splitter)) - _SetGlobalFunc(mem, 'shSplit', func_misc.Split(splitter)) + _AddBuiltinFunc(mem, 'split', func_misc.Split(splitter)) + _AddBuiltinFunc(mem, 'shSplit', func_misc.Split(splitter)) # Float - _SetGlobalFunc(mem, 'floatsEqual', func_misc.FloatsEqual()) + _AddBuiltinFunc(mem, 'floatsEqual', func_misc.FloatsEqual()) # List - _SetGlobalFunc(mem, 'join', func_misc.Join()) - _SetGlobalFunc(mem, 'maybe', func_misc.Maybe()) - _SetGlobalFunc(mem, 'glob', func_misc.Glob(globber)) + _AddBuiltinFunc(mem, 'join', func_misc.Join()) + _AddBuiltinFunc(mem, 'maybe', func_misc.Maybe()) + _AddBuiltinFunc(mem, 'glob', func_misc.Glob(globber)) # Serialize - _SetGlobalFunc(mem, 'toJson8', func_misc.ToJson8(True)) - _SetGlobalFunc(mem, 'toJson', func_misc.ToJson8(False)) + _AddBuiltinFunc(mem, 'toJson8', func_misc.ToJson8(True)) + _AddBuiltinFunc(mem, 'toJson', func_misc.ToJson8(False)) - _SetGlobalFunc(mem, 'fromJson8', func_misc.FromJson8(True)) - _SetGlobalFunc(mem, 'fromJson', func_misc.FromJson8(False)) + _AddBuiltinFunc(mem, 'fromJson8', func_misc.FromJson8(True)) + _AddBuiltinFunc(mem, 'fromJson', func_misc.FromJson8(False)) # Demos - _SetGlobalFunc(mem, '_a2sp', func_misc.BashArrayToSparse()) - _SetGlobalFunc(mem, '_opsp', func_misc.SparseOp()) - - # TODO: 'io' can be in the builtin module, and then hidden in functions - mem.SetNamed(location.LName('io'), io_obj, scope_e.GlobalOnly) + _AddBuiltinFunc(mem, '_a2sp', func_misc.BashArrayToSparse()) + _AddBuiltinFunc(mem, '_opsp', func_misc.SparseOp()) - #mem.SetNamed(location.LName('stdin'), value.Stdin, scope_e.GlobalOnly) + mem.AddBuiltin('io', io_obj) # # Is the shell interactive? diff --git a/core/state.py b/core/state.py index e087434d08..ce52332224 100644 --- a/core/state.py +++ b/core/state.py @@ -16,7 +16,7 @@ from _devbuild.gen.syntax_asdl import (loc, loc_t, Token, debug_frame, debug_frame_e, debug_frame_t) from _devbuild.gen.types_asdl import opt_group_i -from _devbuild.gen.value_asdl import (value, value_e, value_t, sh_lvalue, +from _devbuild.gen.value_asdl import (value, value_e, value_t, Obj, sh_lvalue, sh_lvalue_e, sh_lvalue_t, LeftName, y_lvalue_e, regex_match, regex_match_e, regex_match_t, RegexMatch) @@ -48,6 +48,7 @@ if TYPE_CHECKING: from _devbuild.gen.option_asdl import option_t from core import alloc + from core import vm from osh import sh_expr_eval _ = log @@ -1336,6 +1337,13 @@ def __init__(self, dollar0, argv, arena, debug_stack): # For the ctx builtin self.ctx_stack = [] # type: List[Dict[str, value_t]] + self.builtins = NewDict() # type: Dict[str, value_t] + + # Note: Python 2 and 3 have __builtins__ + # This is just for inspection + builtins_module = Obj(None, self.builtins) + frame['__builtins__'] = Cell(False, False, False, builtins_module) + def __repr__(self): # type: () -> str parts = [] # type: List[str] @@ -1347,6 +1355,10 @@ def __repr__(self): parts.append('>') return '\n'.join(parts) + '\n' + def AddBuiltin(self, name, val): + # type: (str, value_t) -> None + self.builtins[name] = val + def SetPwd(self, pwd): # type: (str) -> None """Used by builtins.""" @@ -2219,6 +2231,10 @@ def GetValue(self, name, which_scopes=scope_e.Shopt): if cell: return cell.val + builtin_val = self.builtins.get(name) + if builtin_val: + return builtin_val + # TODO: Can look in the builtins module, which is a value.Obj return value.Undef @@ -2231,6 +2247,8 @@ def GetCell(self, name, which_scopes=scope_e.Shopt): - declare -p - ${x@a} - to test of 'TZ' is exported in printf? Why? + + Note: consulting __builtins__ doesn't see necessary for any of these """ if which_scopes == scope_e.Shopt: which_scopes = self.ScopesForReading() diff --git a/spec/ysh-module.test.sh b/spec/ysh-module.test.sh index 65d3fe9893..9a9e6249f3 100644 --- a/spec/ysh-module.test.sh +++ b/spec/ysh-module.test.sh @@ -1,15 +1,27 @@ ## our_shell: ysh -## oils_failures_allowed: 2 +## oils_failures_allowed: 1 -#### global frame doesn't contain builtins like len(), dict() +#### global frame doesn't contain builtins like len(), dict(), io try { pp frame_vars_ | grep -o len } pp test_ (_pipeline_status) +try { + pp frame_vars_ | grep -o dict +} +pp test_ (_pipeline_status) + +try { + pp frame_vars_ | grep -o -w io +} +pp test_ (_pipeline_status) + ## STDOUT: (List) [0,1] +(List) [0,1] +(List) [0,1] ## END #### global frame doesn't contain env vars @@ -25,3 +37,19 @@ pp test_ (_pipeline_status) ## END + +#### __builtins__ module + +var b = len(propView(__builtins__)) + +# more than 30 builtins +assert [b > 30] + +var mylist = :| a b | + +setvar len = 4 # overwrite +setvar len = __builtins__.len(mylist) +assert [2 === len] + +## STDOUT: +## END diff --git a/spec/ysh-scope.test.sh b/spec/ysh-scope.test.sh index b9e3ce6cda..901b91c1a4 100644 --- a/spec/ysh-scope.test.sh +++ b/spec/ysh-scope.test.sh @@ -206,19 +206,21 @@ x= ## END #### declare -p respects it -__g=G + +___g=G + show-vars() { - local __x=X - declare -p | grep '__' + local ___x=X + declare -p | grep '___' echo status=$? echo - - declare -p __y | grep '__' + declare -p ___y | grep '___' echo status=$? } demo() { - local __y=Y + local ___y=Y show-vars echo --- @@ -229,16 +231,16 @@ demo() { demo ## STDOUT: -declare -- __g=G -declare -- __x=X -declare -- __y=Y +declare -- ___g=G +declare -- ___x=X +declare -- ___y=Y status=0 - -declare -- __y=Y +declare -- ___y=Y status=0 --- -declare -- __g=G -declare -- __x=X +declare -- ___g=G +declare -- ___x=X status=0 - status=1 From 03128fd1e60b0b0a2061e9b5df59f78e8dcbe912 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 13:54:34 -0400 Subject: [PATCH 251/506] [mycpp] Fix and simplify NewDict() heuristic The issue is that we need the LHS types to generate the RHS in C++ There is still one hack I don't get, but it's cleaner, and the crash bug involving state.Mem.builtins is fixed. --- core/shell.py | 2 -- core/state.py | 1 - mycpp/cppgen_pass.py | 54 +++++++++++++++++++++--------------- mycpp/examples/containers.py | 34 +++++++++++++++++++++++ 4 files changed, 66 insertions(+), 25 deletions(-) diff --git a/core/shell.py b/core/shell.py index 1a90f65124..c5095c1550 100644 --- a/core/shell.py +++ b/core/shell.py @@ -8,7 +8,6 @@ from _devbuild.gen import arg_types from _devbuild.gen.option_asdl import option_i, builtin_i -from _devbuild.gen.runtime_asdl import scope_e from _devbuild.gen.syntax_asdl import (loc, source, source_t, IntParamBox, debug_frame, debug_frame_t) from _devbuild.gen.value_asdl import (value, value_e, value_t, Obj) @@ -33,7 +32,6 @@ unused1 = flag_def from frontend import flag_util -from frontend import location from frontend import reader from frontend import parse_lib diff --git a/core/state.py b/core/state.py index ce52332224..16043259ed 100644 --- a/core/state.py +++ b/core/state.py @@ -48,7 +48,6 @@ if TYPE_CHECKING: from _devbuild.gen.option_asdl import option_t from core import alloc - from core import vm from osh import sh_expr_eval _ = log diff --git a/mycpp/cppgen_pass.py b/mycpp/cppgen_pass.py index c91f75a1cc..0d358d6bcb 100644 --- a/mycpp/cppgen_pass.py +++ b/mycpp/cppgen_pass.py @@ -1412,25 +1412,30 @@ def _ListComprehensionImpl(self, o, lval, c_type): self.def_write_ind('}\n') - def _AssignNewDictImpl(self, lval): - """ + def _AssignNewDictImpl(self, lval, prefix=''): + """Translate NewDict() -> Alloc> + + This function is a specal case because the RHS need TYPES from the LHS. + + e.g. here is how we make ORDERED dictionaries, which can't be done with {}: + d = NewDict() # type: Dict[int, int] - -> auto* d = NewDict(); - - - NewDict exists in Python, it makes ordered dictionaries - - We translate it here because we need type inference - - I think we could get rid of NewDict in C++, and have it only in - Python. - - We used to have the "allocating in a constructor" rooting - problem, but I believe that's gone now. + + -> one of + + auto* d = Alloc>(); # declare + d = Alloc>(); # mutate + + We also have: + + self.d = NewDict() + -> + this->d = Alloc)(); """ lval_type = self.types[lval] + #self.log('lval type %s', lval_type) # Fix for Dict[str, value]? in ASDL - - #self.log('lval type %s', lval_type) if (isinstance(lval_type, UnionType) and len(lval_type.items) == 2 and isinstance(lval_type.items[1], NoneTyp)): lval_type = lval_type.items[0] @@ -1440,12 +1445,7 @@ def _AssignNewDictImpl(self, lval): self.local_var_list.append((lval.name, lval_type)) assert c_type.endswith('*') - - # Hack for declaration vs. definition. TODO: clean this up - prefix = '' if self.current_func_node else 'auto* ' - - self.def_write_ind('%s%s = Alloc<%s>();\n', prefix, lval.name, - c_type[:-1]) + self.def_write('Alloc<%s>()', c_type[:-1]) def _AssignCastImpl(self, o, lval): """ @@ -1585,10 +1585,20 @@ def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: callee = o.rvalue.callee if callee.name == 'NewDict': - self._AssignNewDictImpl(lval) + self.def_write_ind('') + + # Hack for non-members - why does this work? + # Tests cases in mycpp/examples/containers.py + if not isinstance(lval, MemberExpr) and self.current_func_node is None: + self.def_write('auto* ') + + self.accept(lval) + self.def_write(' = ') + self._AssignNewDictImpl(lval) # uses lval, not rval + self.def_write(';\n') - # Bug fix: self.front_frame = NewDict() needs to register member if isinstance(lval, MemberExpr): + # Bug fix: self.front_frame = NewDict() needs to register member self._MaybeAddMember(lval, self.current_member_vars) return diff --git a/mycpp/examples/containers.py b/mycpp/examples/containers.py index 9c11694a51..304e60c682 100755 --- a/mycpp/examples/containers.py +++ b/mycpp/examples/containers.py @@ -199,9 +199,43 @@ def ContainsDemo(): print('hi no') +class HasDictMember(object): + """ + based on state.Mem + """ + def __init__(self): + # type: () -> None + self.builtins = NewDict() # type: Dict[str, str] + + non_member = NewDict() # type: Dict[str, int] + + def Get(self, k): + # type: (str) -> Optional[str] + return self.builtins.get(k) + + +def NewDict_test(): + # type: () -> None + """ + regression test for a few bugs + """ + h = HasDictMember() + result = h.Get('foo') + if result is not None: + print('result %r' % result) + else: + print('OK: NewDict result is None') + + # mutation through non-self object + h.builtins = NewDict() + + def run_tests(): # type: () -> None + NewDict_test() + log('') + ListDemo() log('') TupleDemo() From aa9b683c24c1ca1941c84109581380d55030194e Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 14:46:04 -0400 Subject: [PATCH 252/506] [ysh] io->eval() now supports dollar0, pos_args, vars We want to prever call io->eval(myblock) over eval (myblock) One reason is that eval $mystr uses main_loop.Batch(), which behaves a bit differently. We are more clearly separating the "string-ish" OSH world, and the typed YSH world. io->eval() will not accept string - instead you use parseCommand() to create a value.Command. TODO: remove eval (myblock) altogether. --- builtin/method_io.py | 28 ++++++++++++++++---- doc/ref/chap-builtin-cmd.md | 6 ----- doc/ref/chap-type-method.md | 18 +++++++++++-- mycpp/cppgen_pass.py | 3 ++- mycpp/examples/containers.py | 1 + spec/ysh-builtin-eval.test.sh | 50 +++++++++++++++++------------------ spec/ysh-proc-meta.test.sh | 7 ++--- stdlib/testing.ysh | 2 +- stdlib/ysh/yblocks.ysh | 4 +-- 9 files changed, 72 insertions(+), 47 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index 4ed8eafead..ca1bda6f54 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -1,7 +1,7 @@ """Methods on IO type""" from __future__ import print_function -from _devbuild.gen.value_asdl import value, value_t +from _devbuild.gen.value_asdl import value, value_e, value_t from core import error from core import num @@ -10,7 +10,7 @@ from mycpp.mylib import log, NewDict from osh import prompt -from typing import Dict, TYPE_CHECKING +from typing import Dict, List, cast, TYPE_CHECKING if TYPE_CHECKING: from frontend import typed_args from osh import cmd_eval @@ -46,14 +46,32 @@ def Call(self, rd): # type: (typed_args.Reader) -> value_t unused = rd.PosValue() cmd = rd.PosCommand() - rd.Done() # no more args + + dollar0 = rd.NamedStr("dollar0", None) + pos_args_raw = rd.NamedList("pos_args", None) + vars_ = rd.NamedDict("vars", None) + rd.Done() + + pos_args = None # type: List[str] + if pos_args_raw is not None: + pos_args = [] + for arg in pos_args_raw: + if arg.tag() != value_e.Str: + raise error.TypeErr( + arg, "Expected pos_args to be a List of Strs", + rd.LeftParenToken()) + + pos_args.append(cast(value.Str, arg).s) if self.which == EVAL_NULL: - # errors can arise from false' and 'exit' - unused_status = self.cmd_ev.EvalCommand(cmd) + with state.ctx_Eval(self.cmd_ev.mem, dollar0, pos_args, vars_): + unused_status = self.cmd_ev.EvalCommand(cmd) return value.Null elif self.which == EVAL_DICT: + # TODO: dollar0, pos_args, vars_ not supposed + # Does ctx_FrontFrame has different scoping rules? For "vars"? + bindings = NewDict() # type: Dict[str, value_t] with state.ctx_FrontFrame(self.cmd_ev.mem, bindings): unused_status = self.cmd_ev.EvalCommand(cmd) diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index 2e5da08bf5..3e9a2bc3f4 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -789,12 +789,6 @@ issues][]. [security issues]: https://mywiki.wooledge.org/BashFAQ/048 -YSH eval: - - var myblock = ^(echo hi) - eval (myblock) # => hi - - ### trap trap FLAG* CMD SIGNAL* diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 5c9500d27e..bfa36acf0e 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -548,16 +548,30 @@ A module is a file with YSH code. Evaluate a command, and return `null`. - var c = ^(echo hi) - call io->eval(c) + var cmd = ^(echo hi) + call io->eval(cmd) It's like like the `eval` builtin, and meant to be used in pure functions. +You can also bind: + +- positional args `$1 $2 $3` +- dollar0 `$0` +- named variables + +Examples: + + var cmd = ^(echo "zero $0, one $1, named $x") + call io->eval(cmd, dollar0="z", pos_args=['one'], vars={x: "x"}) + # => zero z, one one, named x + ### evalToDict() diff --git a/mycpp/cppgen_pass.py b/mycpp/cppgen_pass.py index 0d358d6bcb..fdeec73c19 100644 --- a/mycpp/cppgen_pass.py +++ b/mycpp/cppgen_pass.py @@ -1589,7 +1589,8 @@ def visit_assignment_stmt(self, o: 'mypy.nodes.AssignmentStmt') -> T: # Hack for non-members - why does this work? # Tests cases in mycpp/examples/containers.py - if not isinstance(lval, MemberExpr) and self.current_func_node is None: + if not isinstance( + lval, MemberExpr) and self.current_func_node is None: self.def_write('auto* ') self.accept(lval) diff --git a/mycpp/examples/containers.py b/mycpp/examples/containers.py index 304e60c682..d5f621d267 100755 --- a/mycpp/examples/containers.py +++ b/mycpp/examples/containers.py @@ -203,6 +203,7 @@ class HasDictMember(object): """ based on state.Mem """ + def __init__(self): # type: () -> None self.builtins = NewDict() # type: Dict[str, str] diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index a299fe2e1a..12fc4d7d75 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -22,7 +22,7 @@ command literal #### Eval a block within a proc proc run (;;; block) { - eval (block) + call io->eval(block) } run { @@ -43,15 +43,15 @@ lazy-block (&my_block) { json write (myglobal) } -eval (my_block) +call io->eval(my_block) setvar myglobal = 1 -eval (my_block) +call io->eval(my_block) ## STDOUT: 0 1 ## END -#### eval (block) can read variables like eval '' +#### io->eval(block) can read variables like eval '' proc p2(code_str) { var mylocal = 42 @@ -62,7 +62,7 @@ p2 'echo mylocal=$mylocal' proc p (;;; block) { var mylocal = 99 - eval (block) + call io->eval(block) } p { @@ -85,7 +85,7 @@ proc p (;;; block) { # # I think we want to provide full control over the stack. push-frame { - eval (block) + call io->eval(block) } } @@ -98,9 +98,9 @@ p { TODO ## END -#### eval with argv bindings -eval (^(echo "$@"), pos_args=:| foo bar baz |) -eval (^(pp test_ (:| $1 $2 $3 |)), pos_args=:| foo bar baz |) +#### io->eval with argv bindings +call io->eval(^(echo "$@"), pos_args=:| foo bar baz |) +call io->eval(^(pp test_ (:| $1 $2 $3 |)), pos_args=:| foo bar baz |) ## STDOUT: foo bar baz (List) ["foo","bar","baz"] @@ -110,7 +110,7 @@ foo bar baz proc my-split (;;; block) { while read --raw-line { var cols = split(_reply) - eval (block, pos_args=cols) + call io->eval(block, pos_args=cols) } } @@ -149,7 +149,7 @@ d c local2 proc my-split (;;; block) { while read --raw-line { var cols = split(_reply) - eval (block, vars={_line: _reply, _first: cols[0]}) + call io->eval(block, vars={_line: _reply, _first: cols[0]}) } } @@ -179,18 +179,18 @@ c d | c local2 #### eval with custom dollar0 var b = ^(write $0) -eval (b, dollar0="my arg0") +call io->eval(b, dollar0="my arg0") ## STDOUT: my arg0 ## END #### eval with vars bindings var myVar = "abc" -eval (^(pp test_ (myVar))) -eval (^(pp test_ (myVar)), vars={ 'myVar': '123' }) +call io->eval(^(pp test_ (myVar))) +call io->eval(^(pp test_ (myVar)), vars={ 'myVar': '123' }) # eval doesn't modify it's environment -eval (^(pp test_ (myVar))) +call io->eval(^(pp test_ (myVar))) ## STDOUT: (Str) "abc" @@ -205,7 +205,7 @@ proc foreach (binding, in_; list ;; block) { } for item in (list) { - eval (block, vars={ [binding]: item }) + call io->eval(block, vars={ [binding]: item }) } } @@ -243,7 +243,7 @@ proc __arg (name) { } proc parser (; spec ;; block) { - eval (block, vars={ 'flag': __flag, 'arg': __arg }) + call io->eval(block, vars={ 'flag': __flag, 'arg': __arg }) } parser (&spec) { @@ -267,7 +267,7 @@ arg file #### vars initializes the variable frame, but does not remember it var vars = { 'foo': 123 } -eval (^(var bar = 321;), vars=vars) +call io->eval(^(var bar = 321;), vars=vars) pp test_ (vars) ## STDOUT: @@ -275,20 +275,20 @@ pp test_ (vars) ## END #### eval pos_args must be strings -eval (^(true), pos_args=[1, 2, 3]) +call io->eval(^(true), pos_args=[1, 2, 3]) ## status: 3 #### eval with vars follows same scoping as without proc local-scope { var myVar = "foo" - eval (^(echo $myVar), vars={ someOtherVar: "bar" }) - eval (^(echo $myVar)) + call io->eval(^(echo $myVar), vars={ someOtherVar: "bar" }) + call io->eval(^(echo $myVar)) } # In global scope var myVar = "baz" -eval (^(echo $myVar), vars={ someOtherVar: "bar" }) -eval (^(echo $myVar)) +call io->eval(^(echo $myVar), vars={ someOtherVar: "bar" }) +call io->eval (^(echo $myVar)) local-scope ## STDOUT: @@ -298,7 +298,7 @@ foo foo ## END -#### eval 'mystring' vs. eval (myblock) +#### eval 'mystring' vs. call io->eval(myblock) eval 'echo plain' echo plain=$? @@ -322,7 +322,7 @@ pp test_ (_error) var b = ^(echo one; false; echo two) try { - eval (b) + call io->eval(b) } pp test_ (_error) diff --git a/spec/ysh-proc-meta.test.sh b/spec/ysh-proc-meta.test.sh index ba25858554..996ec53d69 100644 --- a/spec/ysh-proc-meta.test.sh +++ b/spec/ysh-proc-meta.test.sh @@ -74,7 +74,7 @@ prefix a prefix b ## END -#### with parseCommand() then eval vars={out_dict: {}} +#### with parseCommand() then io->eval(cmd, vars={out_dict: {}}) # This could take the place of evalToDict()? But evalToDict() is useful in # Hay? @@ -89,10 +89,7 @@ func genProcs() { setvar out_dict.echo_$param = echo_$param """ var cmd = parseCommand(s) - - # TODO: io->eval() should support vars=vars - #call io->eval(cmd) - eval (cmd, vars=vars) + call io->eval(cmd, vars=vars) } return (vars.out_dict) } diff --git a/stdlib/testing.ysh b/stdlib/testing.ysh index 7abce14367..cf01a298d4 100644 --- a/stdlib/testing.ysh +++ b/stdlib/testing.ysh @@ -106,7 +106,7 @@ proc run-tests { for cmd in (_describe) { # TODO: print filename and 'describe' name? try { - eval (cmd) + call io->eval(cmd) } if (_status !== 0) { echo 'failed' diff --git a/stdlib/ysh/yblocks.ysh b/stdlib/ysh/yblocks.ysh index c2b9b2f689..0a170cf33f 100755 --- a/stdlib/ysh/yblocks.ysh +++ b/stdlib/ysh/yblocks.ysh @@ -17,7 +17,7 @@ proc yb-capture(; out; ; block) { var stdout = '' try { - eval (block) | read --all (&stdout) + call io->eval(block) | read --all (&stdout) } # TODO: if 'block' contains a pipeline, we lose this magic var var result = {status: _pipeline_status[0], stdout} @@ -33,7 +33,7 @@ proc yb-capture-2(; out; ; block) { var stderr = '' try { - eval (block) 2>&1 | read --all (&stderr) + call io->eval(block) 2>&1 | read --all (&stderr) } #pp test_ (_pipeline_status) From 936175bc121d43ad6f3f45ed2686dc8038dbca8a Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 16:21:25 -0400 Subject: [PATCH 253/506] [ysh breaking] Remove eval (cmd) in favor of io->eval(cmd) As mentioned in the last change, eval $mystr uses a slightly different algorithm to evaluate code -- main_loop.Batch. Error handling is different. This caused some parsing problems in yblocks.ysh - the "command ends with expression" problem. I worked aronud it with with { } and fopen { } wrappers. I made a note on Zulip about it. --- builtin/meta_osh.py | 36 ----------------------------------- spec/ysh-blocks.test.sh | 8 ++++---- spec/ysh-builtin-eval.test.sh | 10 +++++----- spec/ysh-control-flow.test.sh | 6 +++--- spec/ysh-proc.test.sh | 6 +++--- stdlib/ysh/yblocks.ysh | 12 ++++++++++-- 6 files changed, 25 insertions(+), 53 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index a8e38f0012..dfe986b807 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -62,44 +62,8 @@ def __init__( self.errfmt = errfmt self.mem = mem - def RunTyped(self, cmd_val): - # type: (cmd_value.Argv) -> int - """For eval (mycmd) - - Note: this doesn't have the exact same interface as main_loop.Batch(). - I wonder if it's better to have - - var cmd = parseCommand(s) - var expr = parseExpr(s) - - eval-command (cmd) or eval-block (b) - = evalExpr(expr) - """ - rd = typed_args.ReaderForProc(cmd_val) - cmd = rd.PosCommand() - dollar0 = rd.NamedStr("dollar0", None) - pos_args_raw = rd.NamedList("pos_args", None) - vars = rd.NamedDict("vars", None) - rd.Done() - - pos_args = None # type: List[str] - if pos_args_raw is not None: - pos_args = [] - for arg in pos_args_raw: - if arg.tag() != value_e.Str: - raise error.TypeErr( - arg, "Expected pos_args to be a list of Strs", - rd.LeftParenToken()) - - pos_args.append(cast(value.Str, arg).s) - - with state.ctx_Eval(self.mem, dollar0, pos_args, vars): - return self.cmd_ev.EvalCommand(cmd) - def Run(self, cmd_val): # type: (cmd_value.Argv) -> int - if cmd_val.proc_args: - return self.RunTyped(cmd_val) # There are no flags, but we need it to respect -- _, arg_r = flag_util.ParseCmdVal('eval', cmd_val) diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh index e88b420f5a..a7604a40f6 100644 --- a/spec/ysh-blocks.test.sh +++ b/spec/ysh-blocks.test.sh @@ -102,13 +102,13 @@ cat out ## END #### block literal in expression mode: ^(echo $PWD) -shopt -s oil:all +shopt -s ysh:all const myblock = ^(echo $PWD | wc -l) -eval (myblock) +call io->eval(myblock) const b2 = ^(echo one; echo two) -eval (b2) +call io->eval(b2) ## STDOUT: 1 @@ -259,7 +259,7 @@ shopt --set parse_brace parse_proc parse_paren proc task(name ; ; ; b = null) { echo "task name=$name" if (b) { - eval (b) + call io->eval(b) return 33 } else { echo 'no block' diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 12fc4d7d75..969a3a7de9 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -3,17 +3,17 @@ ## our_shell: ysh ## oils_failures_allowed: 3 -#### Eval does not take a literal block - can restore this later +#### eval builtin does not take a literal block - can restore this later var b = ^(echo obj) -eval (b) +call io->eval (b) -eval (^(echo command literal)) +call io->eval (^(echo command literal)) # Doesn't work because it's a positional arg eval { echo block } -## status: 3 +## status: 2 ## STDOUT: obj command literal @@ -303,7 +303,7 @@ foo eval 'echo plain' echo plain=$? var b = ^(echo plain) -eval (b) +call io->eval(b) echo plain=$? echo diff --git a/spec/ysh-control-flow.test.sh b/spec/ysh-control-flow.test.sh index 87ccebb6ad..02f579a812 100644 --- a/spec/ysh-control-flow.test.sh +++ b/spec/ysh-control-flow.test.sh @@ -55,7 +55,7 @@ one shopt -s ysh:all proc proc-that-runs-block (; ; ; b) { - eval (b) + call io->eval(b) } proc-that-runs-block { echo one @@ -71,7 +71,7 @@ one shopt -s ysh:all proc proc-that-runs-block (; ; ; b) { - eval (b) + call io->eval(b) } f() { @@ -93,7 +93,7 @@ end func shopt -s ysh:all proc proc-that-runs-block (; ; ; b) { - eval (b) + call io->eval(b) } f() { diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index a611aca66b..fb0382ec29 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -175,7 +175,7 @@ proc f(x, y ; ; ; block) { echo f word $x $y if (block) { - eval (block) + call io->eval(block) } } f a b { echo FFF } @@ -188,7 +188,7 @@ proc g(x, y, ...rest ; ; ; block) { echo g rest @rest if (block) { - eval (block) + call io->eval(block) } } g a b c d { @@ -313,7 +313,7 @@ brace shopt --set ysh:upgrade proc p ( ; ; ; block) { - eval (block) + call io->eval(block) } p { echo literal } diff --git a/stdlib/ysh/yblocks.ysh b/stdlib/ysh/yblocks.ysh index 0a170cf33f..323428284e 100755 --- a/stdlib/ysh/yblocks.ysh +++ b/stdlib/ysh/yblocks.ysh @@ -17,7 +17,11 @@ proc yb-capture(; out; ; block) { var stdout = '' try { - call io->eval(block) | read --all (&stdout) + { call io->eval(block) } | read --all (&stdout) + + # Note that this doesn't parse because of expression issue: + # call io->eval(block) | read --all (&stdout) + # used to be eval (block) } # TODO: if 'block' contains a pipeline, we lose this magic var var result = {status: _pipeline_status[0], stdout} @@ -33,7 +37,11 @@ proc yb-capture-2(; out; ; block) { var stderr = '' try { - call io->eval(block) 2>&1 | read --all (&stderr) + fopen 2>&1 { call io->eval(block); } | read --all (&stderr) + + # Note that this doesn't parse because of expression issue: + # call io->eval(block) 2>&1 | read --all (&stderr) + # used to be eval (block) 2>&1 } #pp test_ (_pipeline_status) From 818eb8add7bf002271e31fc5f5f36e853540bb87 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 18:11:43 -0400 Subject: [PATCH 254/506] [fix] Spec tests, docs, lint errors --- builtin/meta_osh.py | 4 +--- doc/ysh-tour.md | 3 +-- spec/builtin-eval-source.test.sh | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index dfe986b807..bcfe372f14 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -6,7 +6,6 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus -from _devbuild.gen.value_asdl import value, value_e from _devbuild.gen.syntax_asdl import source, loc from core import alloc from core import dev @@ -22,7 +21,6 @@ from frontend import flag_util from frontend import consts from frontend import reader -from frontend import typed_args from mycpp.mylib import log, print_stderr from pylib import os_path from osh import cmd_eval @@ -32,7 +30,7 @@ _ = log -from typing import Dict, List, Tuple, Optional, cast, TYPE_CHECKING +from typing import Dict, List, Tuple, Optional, TYPE_CHECKING if TYPE_CHECKING: from frontend import args from frontend.parse_lib import ParseContext diff --git a/doc/ysh-tour.md b/doc/ysh-tour.md index 09916649a0..3262360672 100644 --- a/doc/ysh-tour.md +++ b/doc/ysh-tour.md @@ -651,8 +651,7 @@ At the call site, they can look like any of these forms: cd /tmp { echo $PWD } # word arg, then block arg - var mycmd = ^(echo hi) # expression for a value.Command - eval (mycmd) # positional arg + pp value ([1, 2]) # positional, typed arg diff --git a/spec/builtin-eval-source.test.sh b/spec/builtin-eval-source.test.sh index 48e6df6695..cc2868f3b9 100644 --- a/spec/builtin-eval-source.test.sh +++ b/spec/builtin-eval-source.test.sh @@ -93,7 +93,7 @@ shopt -s ysh:all proc proc_that_evals(; ; ;b) { for i in 1 2; do echo $i - eval (b) + call io->eval(b) done echo 'end func' } From bdb169534ec10a43da28353a25d819ce80b80c63 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 18:23:14 -0400 Subject: [PATCH 255/506] [ysh] Rename fopen builtin -> redir 'fopen' is kept for backward compatibility. It's in spec/ysh-TODO-deprecate --- builtin/io_ysh.py | 6 +++--- core/shell.py | 4 +++- frontend/builtin_def.py | 2 +- frontend/flag_def.py | 2 +- spec/ysh-TODO-deprecate.test.sh | 15 +++++++++++++++ spec/ysh-builtin-meta.test.sh | 2 +- spec/ysh-builtins.test.sh | 8 ++++---- spec/ysh-funcs-external.test.sh | 2 +- spec/ysh-json.test.sh | 4 ++-- spec/ysh-regex-api.test.sh | 2 +- 10 files changed, 32 insertions(+), 15 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 95a71bdc4d..4090d0c073 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -286,8 +286,8 @@ def Run(self, cmd_val): return 0 -class Fopen(vm._Builtin): - """fopen does nothing but run a block. +class RunBlock(vm._Builtin): + """Used for 'redir' builtin It's used solely for its redirects. fopen >out.txt { echo hi } @@ -303,7 +303,7 @@ def __init__(self, mem, cmd_ev): def Run(self, cmd_val): # type: (cmd_value.Argv) -> int - _, arg_r = flag_util.ParseCmdVal('fopen', + _, arg_r = flag_util.ParseCmdVal('redir', cmd_val, accept_typed_args=True) diff --git a/core/shell.py b/core/shell.py index c5095c1550..7bc9881c71 100644 --- a/core/shell.py +++ b/core/shell.py @@ -677,7 +677,9 @@ def Main( b[builtin_i.printf] = printf_osh.Printf(mem, parse_ctx, unsafe_arith, errfmt) b[builtin_i.write] = io_ysh.Write(mem, errfmt) - b[builtin_i.fopen] = io_ysh.Fopen(mem, cmd_ev) + redir_builtin = io_ysh.RunBlock(mem, cmd_ev) # used only for redirects + b[builtin_i.redir] = redir_builtin + b[builtin_i.fopen] = redir_builtin # alias for backward compatibility # (pp output format isn't stable) b[builtin_i.pp] = io_ysh.Pp(expr_ev, mem, errfmt, procs, arena) diff --git a/frontend/builtin_def.py b/frontend/builtin_def.py index 80a0c43153..e1689a9552 100644 --- a/frontend/builtin_def.py +++ b/frontend/builtin_def.py @@ -58,7 +58,7 @@ # take a block # push-registers added below 'fork', 'forkwait', - 'fopen', + 'redir', 'fopen', # fopen is for backward compat 'shvar', 'ctx', diff --git a/frontend/flag_def.py b/frontend/flag_def.py index 9998fa418a..8df5019623 100644 --- a/frontend/flag_def.py +++ b/frontend/flag_def.py @@ -506,7 +506,7 @@ def _DefineCompletionActions(spec): PUSH_REGISTERS_SPEC = FlagSpec('push-registers') -FOPEN_SPEC = FlagSpec('fopen') +FOPEN_SPEC = FlagSpec('redir') # # JSON diff --git a/spec/ysh-TODO-deprecate.test.sh b/spec/ysh-TODO-deprecate.test.sh index 05811fc35a..b540a6029d 100644 --- a/spec/ysh-TODO-deprecate.test.sh +++ b/spec/ysh-TODO-deprecate.test.sh @@ -86,3 +86,18 @@ echo $['foo' => upper()] ## STDOUT: FOO ## END + +#### fopen can be spelled redir +shopt --set ysh:upgrade + +fopen >out { + echo 1 + echo 2 +} + +tac out + +## STDOUT: +2 +1 +## END diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index c2cd3fcc97..ba8c95b3dd 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -95,7 +95,7 @@ Block shopt -s ysh:upgrade -fopen >out.txt { +redir >out.txt { x=42 setvar y = {foo: x} diff --git a/spec/ysh-builtins.test.sh b/spec/ysh-builtins.test.sh index 875cc04d0d..0fdbabe9fb 100644 --- a/spec/ysh-builtins.test.sh +++ b/spec/ysh-builtins.test.sh @@ -489,14 +489,14 @@ hi status=0 ## END -#### fopen +#### redir shopt --set parse_brace parse_proc proc p { echo 'proc' } -fopen >out.txt { +redir >out.txt { p echo 'builtin' } @@ -505,12 +505,12 @@ cat out.txt echo --- -fopen left.txt {right}>right.txt { +redir {left}>left.txt {right}>right.txt { echo 1 >& $left echo 1 >& $right diff --git a/spec/ysh-funcs-external.test.sh b/spec/ysh-funcs-external.test.sh index e6635999cc..399b508b34 100644 --- a/spec/ysh-funcs-external.test.sh +++ b/spec/ysh-funcs-external.test.sh @@ -7,7 +7,7 @@ proc myadd { json read (&args) # convenient! - fopen >&2 { + redir >&2 { = args } diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index 2b7ba48699..a124645ec7 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -235,7 +235,7 @@ var L = [1, 2, 3] setvar L[0] = L shopt -s ysh:upgrade -fopen >tmp.txt { +redir >tmp.txt { pp test_ (L) } fgrep -n -o '[ -->' tmp.txt @@ -254,7 +254,7 @@ var d = {} setvar d.k = d shopt -s ysh:upgrade -fopen >tmp.txt { +redir >tmp.txt { pp test_ (d) } fgrep -n -o '{ -->' tmp.txt diff --git a/spec/ysh-regex-api.test.sh b/spec/ysh-regex-api.test.sh index f6874db877..0a0df973ec 100644 --- a/spec/ysh-regex-api.test.sh +++ b/spec/ysh-regex-api.test.sh @@ -46,7 +46,7 @@ got expected status 3 shopt -s ysh:upgrade # Hm it's hard to test this, we can't get stderr of YSH from within YSH? -#fopen 2>err.txt { +#redir 2>err.txt { # if ('abc' ~ '+') { # echo 'bad' # } From ab4678c3e53658f141f4f8c7eca77908c0e0ac6a Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 19:31:22 -0400 Subject: [PATCH 256/506] [osh] declare -F -f only print shell function names We distinguish between - all invokables - shell functions For complete -A function, we return ALL invokables. Solutions for introspecting on all invokables - We need list-vars - written with listVars() - or names(__frame__), and __frame__ is an object that points at itself - does this slow things down for now reason? - how about names(_frame) or names(thisFrame()) --- builtin/assign_osh.py | 2 +- builtin/completion_osh.py | 14 +++---------- builtin/io_ysh.py | 2 +- core/shell.py | 4 +++- core/state.py | 39 +++++++++++++++++++++++++++++++++++-- spec/ysh-object.test.sh | 22 ++++++++++++++++++++- spec/ysh-proc.test.sh | 41 ++++++++++++++++++++++++++++++++------- 7 files changed, 100 insertions(+), 24 deletions(-) diff --git a/builtin/assign_osh.py b/builtin/assign_osh.py index 1176417b56..3428546ead 100644 --- a/builtin/assign_osh.py +++ b/builtin/assign_osh.py @@ -405,7 +405,7 @@ def Run(self, cmd_val): status = self._PrintFuncs(names) else: # bash quirk: with no names, they're printed in a different format! - for func_name in self.procs.GetNames(): + for func_name in self.procs.ShellFuncNames(): print('declare -f %s' % (func_name)) return status diff --git a/builtin/completion_osh.py b/builtin/completion_osh.py index 1f911c71c0..ee56ed8b80 100644 --- a/builtin/completion_osh.py +++ b/builtin/completion_osh.py @@ -48,11 +48,7 @@ def Print(self, f): class _DynamicProcDictAction(completion.CompletionAction): - """For completing from proc and aliases dicts, which are mutable. - - Note: this is the same as _FixedWordsAction now, but won't be when the code - is statically typed! - """ + """For completing shell functions/procs/invokables.""" def __init__(self, d): # type: (state.Procs) -> None @@ -60,7 +56,7 @@ def __init__(self, d): def Matches(self, comp): # type: (Api) -> Iterator[str] - for name in self.d.GetNames(): + for name in self.d.InvokableNames(): if name.startswith(comp.to_complete): yield name @@ -70,11 +66,7 @@ def Print(self, f): class _DynamicStrDictAction(completion.CompletionAction): - """For completing from proc and aliases dicts, which are mutable. - - Note: this is the same as _FixedWordsAction now, but won't be when the code - is statically typed! - """ + """For completing from the alias dicts, which is mutable.""" def __init__(self, d): # type: (Dict[str, str]) -> None diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 4090d0c073..ee02f5cc1c 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -209,7 +209,7 @@ def Run(self, cmd_val): blame_loc=locs[i]) return 1 else: - names = self.procs.GetNames() + names = self.procs.InvokableNames() # TSV8 header print('proc_name\tdoc_comment') diff --git a/core/shell.py b/core/shell.py index 7bc9881c71..94d5b53da1 100644 --- a/core/shell.py +++ b/core/shell.py @@ -783,6 +783,9 @@ def Main( methods[value_e.Dict] = { 'M/erase': method_dict.Erase(), + # TODO: names(d) get(d, k) has(d, k) might be better + # values(d) is OK too + # Dict.get() # Dict.keys() # Dict.values() @@ -803,7 +806,6 @@ def Main( 'M/accum': None, } methods[value_e.List] = { - # TODO: __mut_{reverse,append,extend,pop,insert,remove} 'M/reverse': method_list.Reverse(), 'M/append': method_list.Append(), 'M/extend': method_list.Extend(), diff --git a/core/state.py b/core/state.py index 16043259ed..4f3e1784f7 100644 --- a/core/state.py +++ b/core/state.py @@ -2509,11 +2509,28 @@ def Get(self, name): First, we search for a proc, and then a sh-func. This means that procs can shadow the definition of sh-funcs. + + Callers + cmd_eval: check for redefining proc or sh-func + lookup for runproc - does this find sh-funcs too? + type -a - should print a separate entry + pp proc + complete -F myfunc + declare -p - should not print procs, only shell stuff """ maybe_proc = self.mem.GetValue(name) if maybe_proc.tag() == value_e.Proc: return cast(value.Proc, maybe_proc) + if maybe_proc.tag() == value_e.Obj: + obj = cast(Obj, maybe_proc) + # Now does it have + + # Error cases for proc lookup: + # 1. value.Int + # 2. value.Obj with __invoke__, but it's not a value.Proc + # 2. value.Obj without __invoke__ + if name in self.sh_funcs: return self.sh_funcs[name] @@ -2524,17 +2541,35 @@ def Del(self, to_del): """Undefine a sh-func with name `to_del`, if it exists.""" mylib.dict_erase(self.sh_funcs, to_del) - def GetNames(self): + def ShellFuncNames(self): # type: () -> List[str] - """Returns a *sorted* list of all proc names""" + """Returns a *sorted* list of all shell function names + + Callers: + declare -f -F + """ names = self.sh_funcs.keys() + names.sort() + return names + def InvokableNames(self): + # type: () -> List[str] + """Returns a *sorted* list of all invokable names + + Callers: + complete -A function + """ + names = self.sh_funcs.keys() + + # TODO: look up the call stack - local and global var_frame = self.mem.var_stack[0] for name in var_frame: cell = var_frame[name] if cell.val.tag() == value_e.Proc: names.append(name) + # TODO: value.Obj + names.sort() return names diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index cd7abb6c3b..fc0c5efed6 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -1,5 +1,5 @@ ## our_shell: ysh -## oils_failures_allowed: 2 +## oils_failures_allowed: 3 #### Object() creates prototype chain @@ -210,3 +210,23 @@ pp test_ (Dict.get(d, 'key', 'default')) ## STDOUT: ## END + +#### Bound Proc? + +proc p (word1, word2; self, int1, int2) { + echo "sum = $[self.x + self.y]" + pp test_ (self) + pp test_ ([word1, word2, int1, int2]) +} + +p a b ({x: 5, y: 6}, 42, 43) + +var methods = Object(null, {__invoke__: p}) + +var callable = Object(methods, {x: 98, y: 99}) + +# TODO: change this error message +callable a b (42, 43) + +## STDOUT: +## END diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index fb0382ec29..e08b99fc97 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 0 +## oils_failures_allowed: 1 #### Open proc (any number of args) shopt --set parse_proc @@ -238,7 +238,7 @@ p ## STDOUT: ## END -#### declare -F prints procs and shell-funcs +#### declare -F only prints shell functions shopt --set parse_proc myfunc() { @@ -254,7 +254,35 @@ declare -F ## status: 0 ## STDOUT: declare -f myfunc -declare -f myproc +## END + +#### sh-func vs. proc vs. Obj: type -a, pp proc, runproc, declare -p -F, etc. +shopt --set ysh:upgrade + +myfunc() { + echo hi +} + +proc myproc { + echo hi +} + +type myfunc +echo + +type myproc +echo + +pp proc +echo + +declare -p +echo + +declare -F +echo + +## STDOUT: ## END #### procs are in same namespace as variables @@ -476,10 +504,9 @@ proc foo() { try { foo } echo status=$[_error.code] -# TODO: should we abandon declare -F in favour of `pp proc`? -declare -F +pp test_ (foo) unset foo -declare -F +#pp test_ (foo) try { foo } echo status=$[_error.code] @@ -487,7 +514,7 @@ echo status=$[_error.code] ## STDOUT: bar status=0 -declare -f foo + status=127 ## END From 84dba9a979d1952460977aef5bce81740040b8b9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 21:10:32 -0400 Subject: [PATCH 257/506] [ysh] Complete invokables in the local frame as well as the global InvokableNames() is called by compgen -A function pp proc - rewrite this in YSH, and deprecate it This is a complement to the change that defined Procs in the local frame. We should also allow procs to be defined within procs. But there are no closures - we bind 'self' instead. --- builtin/assign_osh.py | 4 +- core/state.py | 95 +++++++++++++++++++++++++------------------ osh/cmd_eval.py | 11 ++++- spec/ysh-proc.test.sh | 33 ++++++++------- 4 files changed, 84 insertions(+), 59 deletions(-) diff --git a/builtin/assign_osh.py b/builtin/assign_osh.py index 3428546ead..bf5f96fa4b 100644 --- a/builtin/assign_osh.py +++ b/builtin/assign_osh.py @@ -524,7 +524,7 @@ def _UnsetVar(self, arg, location, proc_fallback): return False if proc_fallback and not found: - self.procs.Del(arg) + self.procs.EraseShellFunc(arg) return True @@ -538,7 +538,7 @@ def Run(self, cmd_val): location = arg_locs[i] if arg.f: - self.procs.Del(name) + self.procs.EraseShellFunc(name) elif arg.v: if not self._UnsetVar(name, location, False): diff --git a/core/state.py b/core/state.py index 4f3e1784f7..7de53a79dd 100644 --- a/core/state.py +++ b/core/state.py @@ -2488,6 +2488,14 @@ def PopContextStack(self): return self.ctx_stack.pop() +def _AddNames(unique, frame): + # type: (Dict[str, bool], Dict[str, Cell]) -> None + for name in frame: + cell = frame[name] + if cell.val.tag() == value_e.Proc: + unique[name] = True + + class Procs: def __init__(self, mem): @@ -2495,13 +2503,56 @@ def __init__(self, mem): self.mem = mem self.sh_funcs = {} # type: Dict[str, value.Proc] - def SetProc(self, name, proc): + def DefineShellFunc(self, name, proc): # type: (str, value.Proc) -> None - self.mem.var_stack[-1][name] = Cell(False, False, False, proc) + self.sh_funcs[name] = proc + + def EraseShellFunc(self, to_del): + # type: (str) -> None + """Undefine a sh-func with name `to_del`, if it exists.""" + mylib.dict_erase(self.sh_funcs, to_del) + + def ShellFuncNames(self): + # type: () -> List[str] + """Returns a *sorted* list of all shell function names + + Callers: + declare -f -F + """ + names = self.sh_funcs.keys() + names.sort() + return names - def SetShFunc(self, name, proc): + def DefineProc(self, name, proc): # type: (str, value.Proc) -> None - self.sh_funcs[name] = proc + self.mem.var_stack[-1][name] = Cell(False, False, False, proc) + + def InvokableNames(self): + # type: () -> List[str] + """Returns a *sorted* list of all invokable names + + Callers: + complete -A function + pp proc - should deprecate this + """ + unique = {} # type: Dict[str, bool] + for name in self.sh_funcs: + unique[name] = True + + top_frame = self.mem.var_stack[-1] + _AddNames(unique, top_frame) + + global_frame = self.mem.var_stack[0] + #log('%d %d', id(top_frame), id(global_frame)) + if global_frame is not top_frame: + _AddNames(unique, global_frame) + + #log('%s', unique) + + names = unique.keys() + names.sort() + + return names def Get(self, name): # type: (str) -> value.Proc @@ -2536,42 +2587,6 @@ def Get(self, name): return None - def Del(self, to_del): - # type: (str) -> None - """Undefine a sh-func with name `to_del`, if it exists.""" - mylib.dict_erase(self.sh_funcs, to_del) - - def ShellFuncNames(self): - # type: () -> List[str] - """Returns a *sorted* list of all shell function names - - Callers: - declare -f -F - """ - names = self.sh_funcs.keys() - names.sort() - return names - - def InvokableNames(self): - # type: () -> List[str] - """Returns a *sorted* list of all invokable names - - Callers: - complete -A function - """ - names = self.sh_funcs.keys() - - # TODO: look up the call stack - local and global - var_frame = self.mem.var_stack[0] - for name in var_frame: - cell = var_frame[name] - if cell.val.tag() == value_e.Proc: - names.append(name) - - # TODO: value.Obj - - names.sort() - return names # diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 02f8c39141..35ec879d54 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1302,11 +1302,18 @@ def _DoShFunction(self, node): node.name, node.name_tok) sh_func = value.Proc(node.name, node.name_tok, proc_sig.Open, node.body, None, True, None) - self.procs.SetShFunc(node.name, sh_func) + self.procs.DefineShellFunc(node.name, sh_func) def _DoProc(self, node): # type: (Proc) -> None proc_name = lexer.TokenVal(node.name) + + # Note: this is similar 'const x = 42' and redefine_const -- it's a + # dynamic check that it doesn't already exist + # Also modules make this less necessary, because there are fewer name + # conflicts + # We could also define procs as READ-ONLY, but that means we need + # Dict[str, Cell] and not Dict[str, value_t] if (self.procs.Get(proc_name) and not self.exec_opts.redefine_proc_func()): e_die( @@ -1322,7 +1329,7 @@ def _DoProc(self, node): # no dynamic scope proc = value.Proc(proc_name, node.name, node.sig, node.body, proc_defaults, False, None) - self.procs.SetProc(proc_name, proc) + self.procs.DefineProc(proc_name, proc) def _DoFunc(self, node): # type: (Func) -> None diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index e08b99fc97..b9496d94c4 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 #### Open proc (any number of args) shopt --set parse_proc @@ -256,10 +256,10 @@ declare -F declare -f myfunc ## END -#### sh-func vs. proc vs. Obj: type -a, pp proc, runproc, declare -p -F, etc. +#### compgen -A function completes all invokables - shell funcs, Proc, Obj shopt --set ysh:upgrade -myfunc() { +my-shell-func() { echo hi } @@ -267,22 +267,25 @@ proc myproc { echo hi } -type myfunc -echo - -type myproc -echo - -pp proc -echo +compgen -A function -declare -p -echo +echo --- -declare -F -echo +proc p { + eval 'proc inner { echo inner }' + #eval 'proc myproc { echo inner }' # shadowed name + compgen -A function +} +p ## STDOUT: +my-shell-func +myproc +--- +inner +my-shell-func +myproc +p ## END #### procs are in same namespace as variables From d0d4a66eff6bb411ae907de40818ecde027af8d5 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 27 Sep 2024 21:38:47 -0400 Subject: [PATCH 258/506] [builtin/type] Distinguishes shell function, proc, invokable Obj We are not yet calling invokable Obj! That's next. --- builtin/meta_osh.py | 17 +++++++---- core/state.py | 35 ++++++++++++++++++++-- spec/ysh-proc.test.sh | 68 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 8 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index bcfe372f14..fc938a2245 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -208,11 +208,12 @@ def _PrintFreeForm(row): elif kind == 'alias': what = ('an alias for %s' % j8_lite.EncodeString(resolved, unquoted_ok=True)) + elif kind in ('proc', 'invokable'): + # Note: haynode should be an invokable + what = 'a YSH %s' % kind else: # builtin, function, keyword what = 'a shell %s' % kind - # TODO: Should also print haynode - print('%s is %s' % (name, what)) # if kind == 'function': @@ -375,7 +376,7 @@ def Run(self, cmd_val): def _ResolveName( name, # type: str - funcs, # type: state.Procs + procs, # type: state.Procs aliases, # type: Dict[str, str] search_path, # type: state.SearchPath do_all, # type: bool @@ -387,8 +388,14 @@ def _ResolveName( results = [] # type: List[Tuple[str, str, Optional[str]]] - if funcs and funcs.Get(name): - results.append((name, 'function', no_str)) + if procs: + if procs.IsShellFunc(name): + results.append((name, 'function', no_str)) + + if procs.IsProc(name): + results.append((name, 'proc', no_str)) + elif procs.IsObj(name): # can't be both proc and obj + results.append((name, 'invokable', no_str)) if name in aliases: results.append((name, 'alias', aliases[name])) diff --git a/core/state.py b/core/state.py index 7de53a79dd..a031bd969f 100644 --- a/core/state.py +++ b/core/state.py @@ -2507,6 +2507,10 @@ def DefineShellFunc(self, name, proc): # type: (str, value.Proc) -> None self.sh_funcs[name] = proc + def IsShellFunc(self, name): + # type: (str) -> bool + return name in self.sh_funcs + def EraseShellFunc(self, to_del): # type: (str) -> None """Undefine a sh-func with name `to_del`, if it exists.""" @@ -2525,8 +2529,35 @@ def ShellFuncNames(self): def DefineProc(self, name, proc): # type: (str, value.Proc) -> None + """ + procs are defined in the local scope. + """ self.mem.var_stack[-1][name] = Cell(False, False, False, proc) + def IsProc(self, name): + # type: (str) -> bool + + maybe_proc = self.mem.GetValue(name) + # Could be Undef + return maybe_proc.tag() == value_e.Proc + + def IsObj(self, name): + # type: (str) -> bool + + UP_obj = self.mem.GetValue(name) + if UP_obj.tag() != value_e.Obj: + return False + + obj = cast(Obj, UP_obj) + if not obj.prototype: + return False + + invoke = obj.prototype.d.get('__invoke__') + if invoke is None: + return False + + return invoke.tag() == value_e.Proc + def InvokableNames(self): # type: () -> List[str] """Returns a *sorted* list of all invokable names @@ -2562,9 +2593,8 @@ def Get(self, name): can shadow the definition of sh-funcs. Callers - cmd_eval: check for redefining proc or sh-func + cmd_eval: check for redefining proc or sh-func (remove) lookup for runproc - does this find sh-funcs too? - type -a - should print a separate entry pp proc complete -F myfunc declare -p - should not print procs, only shell stuff @@ -2588,7 +2618,6 @@ def Get(self, name): return None - # # Wrappers to Set Variables # diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index b9496d94c4..bc53f7794a 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -288,6 +288,74 @@ myproc p ## END +#### type / type -a builtin on invokables - shell func, proc, invokable +shopt --set ysh:upgrade + +my-shell-func() { + echo hi +} + +proc myproc { + echo hi +} + +proc boundProc(; self) { + echo hi +} + +var methods = Object(null, {__invoke__: boundProc}) +var invokable = Object(methods, {}) + +type -t my-shell-func +type -t myproc +type -t invokable +try { + type -t methods # not invokable! +} +echo $[_error.code] + +echo --- + +type my-shell-func +type myproc +type invokable +try { + type methods # not invokable! +} +echo $[_error.code] + +echo --- + +type -a my-shell-func +type -a myproc +type -a invokable + +echo --- + +if false { # can't redefine right now + invokable() { + echo sh-func + } + type -a invokable +} + +## STDOUT: +function +proc +invokable +1 +--- +my-shell-func is a shell function +myproc is a YSH proc +invokable is a YSH invokable +1 +--- +my-shell-func is a shell function +myproc is a YSH proc +invokable is a YSH invokable +--- +## END + #### procs are in same namespace as variables shopt --set parse_proc From 6644ff904980520deb89cef3ccf5fcbf90196a46 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 28 Sep 2024 00:22:27 -0400 Subject: [PATCH 259/506] [osh] More distinction between shell functions and invokables - bash completion only takes shell functions - declare -F foo bar and declare -f foo bar only find shell functions - just like declare -F only prints them - everywhere else, we treat them the same --- builtin/assign_osh.py | 2 +- builtin/completion_osh.py | 6 ++++-- builtin/io_ysh.py | 4 ++-- builtin/meta_osh.py | 5 +++-- core/executor.py | 2 +- core/state.py | 6 +++++- osh/cmd_eval.py | 4 ++-- spec/ysh-proc.test.sh | 12 +++++++++++- 8 files changed, 29 insertions(+), 12 deletions(-) diff --git a/builtin/assign_osh.py b/builtin/assign_osh.py index bf5f96fa4b..22bb1e0962 100644 --- a/builtin/assign_osh.py +++ b/builtin/assign_osh.py @@ -371,7 +371,7 @@ def _PrintFuncs(self, names): # type: (List[str]) -> int status = 0 for name in names: - if self.procs.Get(name): + if self.procs.GetShellFunc(name): print(name) # TODO: Could print LST for -f, or render LST. Bash does this. 'trap' # could use that too. diff --git a/builtin/completion_osh.py b/builtin/completion_osh.py index ee56ed8b80..1701e79bbe 100644 --- a/builtin/completion_osh.py +++ b/builtin/completion_osh.py @@ -131,9 +131,11 @@ def Build(self, argv, attrs, base_opts): # obviously it's better to check here. if arg.F is not None: func_name = arg.F - func = cmd_ev.procs.Get(func_name) + func = cmd_ev.procs.GetShellFunc(func_name) if func is None: - raise error.Usage('function %r not found' % func_name, + # Note: we will have a different protocol for YSH procs and invokables + # The ideal thing would be some kind of generator ... + raise error.Usage('shell function %r not found' % func_name, loc.Missing) actions.append( completion.ShellFuncAction(cmd_ev, func, self.comp_lookup)) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index ee02f5cc1c..2d7b23d87d 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -203,7 +203,7 @@ def Run(self, cmd_val): names, locs = arg_r.Rest2() if len(names): for i, name in enumerate(names): - node = self.procs.Get(name) + node = self.procs.GetInvokable(name) if node is None: self.errfmt.Print_('Invalid proc %r' % name, blame_loc=locs[i]) @@ -214,7 +214,7 @@ def Run(self, cmd_val): # TSV8 header print('proc_name\tdoc_comment') for name in names: - proc = self.procs.Get(name) # must exist + proc = self.procs.GetInvokable(name) # must exist #log('Proc %s', proc) body = proc.body diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index fc938a2245..e6c3699418 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -362,8 +362,9 @@ def Run(self, cmd_val): raise error.Usage('requires arguments', loc.Missing) name = argv[0] - if not self.procs.Get(name): - self.errfmt.PrintMessage('runproc: no proc named %r' % name) + if not self.procs.GetInvokable(name): + # note: should runproc be invoke? + self.errfmt.PrintMessage('runproc: no invokable named %r' % name) return 1 cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd, diff --git a/core/executor.py b/core/executor.py index 3b3e65b206..b332dce21d 100644 --- a/core/executor.py +++ b/core/executor.py @@ -279,7 +279,7 @@ def RunSimpleCommand(self, cmd_val, cmd_st, run_flags): # Pitfall: What happens if there are two of the same name? I guess # that's why you have = and 'type' inspect them - proc_node = self.procs.Get(arg0) + proc_node = self.procs.GetInvokable(arg0) if proc_node is not None: if self.exec_opts.strict_errexit(): disabled_tok = self.mutable_opts.ErrExitDisabledToken() diff --git a/core/state.py b/core/state.py index a031bd969f..5ac26f7def 100644 --- a/core/state.py +++ b/core/state.py @@ -2511,6 +2511,10 @@ def IsShellFunc(self, name): # type: (str) -> bool return name in self.sh_funcs + def GetShellFunc(self, name): + # type: (str) -> Optional[value.Proc] + return self.sh_funcs.get(name) + def EraseShellFunc(self, to_del): # type: (str) -> None """Undefine a sh-func with name `to_del`, if it exists.""" @@ -2585,7 +2589,7 @@ def InvokableNames(self): return names - def Get(self, name): + def GetInvokable(self, name): # type: (str) -> value.Proc """Try to find a proc/sh-func by `name`, or return None if not found. diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 35ec879d54..d2b8d6bc48 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1295,7 +1295,7 @@ def _DoForExpr(self, node): def _DoShFunction(self, node): # type: (command.ShFunction) -> None - if (self.procs.Get(node.name) and + if (self.procs.GetInvokable(node.name) and not self.exec_opts.redefine_proc_func()): e_die( "Function %s was already defined (redefine_proc_func)" % @@ -1314,7 +1314,7 @@ def _DoProc(self, node): # conflicts # We could also define procs as READ-ONLY, but that means we need # Dict[str, Cell] and not Dict[str, value_t] - if (self.procs.Get(proc_name) and + if (self.procs.GetInvokable(proc_name) and not self.exec_opts.redefine_proc_func()): e_die( "Proc %s was already defined (redefine_proc_func)" % proc_name, diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index bc53f7794a..33afb18b31 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -238,7 +238,7 @@ p ## STDOUT: ## END -#### declare -F only prints shell functions +#### declare -f -F only prints shell functions shopt --set parse_proc myfunc() { @@ -250,10 +250,20 @@ proc myproc { } declare -F +echo --- + +declare -F myproc +echo status=$? + +declare -f myproc +echo status=$? ## status: 0 ## STDOUT: declare -f myfunc +--- +status=1 +status=1 ## END #### compgen -A function completes all invokables - shell funcs, Proc, Obj From e57e685caef394bce810f109e8f2488f90d78114 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 28 Sep 2024 02:31:08 -0400 Subject: [PATCH 260/506] [ysh] Add setVar(), move keys() values() get() The latter are now free functions, instead of non-mutating methods that must be looked up. var k = keys(d) instead of var k = d => keys() The => syntax is too unfamiliar, and will probably be reserved for function chaining only. I don't think we need method lookup. e.g. = mystr.upper() => split() => join() This makes another example in spec/ysh-proc-meta idiomatic. --- builtin/func_reflect.py | 23 +++++++++++++- core/shell.py | 28 ++++++++--------- doc/ref/chap-builtin-func.md | 55 +++++++++++++++++++++++++++++++++ doc/ref/chap-type-method.md | 41 ------------------------ doc/ref/toc-ysh.md | 6 ++-- spec/ysh-TODO-deprecate.test.sh | 13 ++++++++ spec/ysh-dict.test.sh | 20 ++++++++++++ spec/ysh-expr-compare.test.sh | 2 +- spec/ysh-methods.test.sh | 11 ------- spec/ysh-object.test.sh | 19 +----------- spec/ysh-proc-meta.test.sh | 42 +++++++++++++++++++++++++ 11 files changed, 171 insertions(+), 89 deletions(-) diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index db1b2e8d4e..4f40193937 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -13,6 +13,7 @@ from core import main_loop from core import state from core import vm +from frontend import location from frontend import reader from frontend import typed_args from mycpp.mylib import log @@ -42,7 +43,7 @@ def Call(self, rd): class GetVar(vm._Callable): - """Look up normal scoping rules.""" + """Look up a variable, with normal scoping rules.""" def __init__(self, mem): # type: (state.Mem) -> None @@ -56,6 +57,26 @@ def Call(self, rd): return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal) +class SetVar(vm._Callable): + """Set a variable in the local scope. + + We could have a separae setGlobal() too. + """ + + def __init__(self, mem): + # type: (state.Mem) -> None + vm._Callable.__init__(self) + self.mem = mem + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + var_name = rd.PosStr() + val = rd.PosValue() + rd.Done() + self.mem.SetNamed(location.LName(var_name), val, scope_e.LocalOnly) + return value.Null + + class ParseCommand(vm._Callable): def __init__(self, parse_ctx, errfmt): diff --git a/core/shell.py b/core/shell.py index 94d5b53da1..fe41f252d5 100644 --- a/core/shell.py +++ b/core/shell.py @@ -781,21 +781,10 @@ def Main( 'fullMatch': None, } methods[value_e.Dict] = { + # keys() values() get() are FREE functions, not methods + # I think items() isn't as necessary because dicts are ordered? YSH + # code shouldn't use the List of Lists representation. 'M/erase': method_dict.Erase(), - - # TODO: names(d) get(d, k) has(d, k) might be better - # values(d) is OK too - - # Dict.get() - # Dict.keys() - # Dict.values() - 'get': method_dict.Get(), - 'keys': method_dict.Keys(), - 'values': method_dict.Values(), - - # I think items() isn't as necessary because dicts are ordered? - # YSH code shouldn't use the List of Lists representation. - # could be d->tally() or d->increment(), but inc() is short # # call d->inc('mycounter') @@ -804,6 +793,11 @@ def Main( # call d->accum('mygroup', 'value') 'M/accum': None, + + # DEPRECATED - use free functions + 'get': method_dict.Get(), + 'keys': method_dict.Keys(), + 'values': method_dict.Values(), } methods[value_e.List] = { 'M/reverse': method_list.Reverse(), @@ -870,6 +864,7 @@ def Main( _AddBuiltinFunc(mem, 'shvarGet', func_reflect.Shvar_get(mem)) _AddBuiltinFunc(mem, 'getVar', func_reflect.GetVar(mem)) + _AddBuiltinFunc(mem, 'setVar', func_reflect.SetVar(mem)) _AddBuiltinFunc(mem, 'Object', func_misc.Object()) _AddBuiltinFunc(mem, 'prototype', func_misc.Prototype()) @@ -883,6 +878,11 @@ def Main( _AddBuiltinFunc(mem, 'list', func_misc.List_()) _AddBuiltinFunc(mem, 'dict', func_misc.DictFunc()) + # Dict functions + _AddBuiltinFunc(mem, 'get', method_dict.Get()) + _AddBuiltinFunc(mem, 'keys', method_dict.Keys()) + _AddBuiltinFunc(mem, 'values', method_dict.Values()) + _AddBuiltinFunc(mem, 'runes', func_misc.Runes()) _AddBuiltinFunc(mem, 'encodeRunes', func_misc.EncodeRunes()) _AddBuiltinFunc(mem, 'bytes', func_misc.Bytes()) diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index 623d2a2ca8..c4aea41587 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -210,6 +210,49 @@ It's also often called with the `=>` chaining operator: json write (items => join(' ')) # => "1 2 3" json write (items => join(', ')) # => "1, 2, 3" +## Dict + +### keys() + +Returns all existing keys from a dict as a list of strings. + + var en2fr = { + hello: "bonjour", + friend: "ami", + cat: "chat" + } + = keys(en2fr) + # => (List 0x4689) ["hello","friend","cat"] + +### values() + +Similar to `keys()`, but returns the values of the dictionary. + + var person = { + name: "Foo", + age: 25, + hobbies: :|walking reading| + } + = values(en2fr) + # => (List 0x4689) ["Foo",25,["walking","reading"]] + +### get() + +Return value for given key, falling back to the default value if the key +doesn't exist. Default is required. + + var book = { + title: "Hitchhiker's Guide", + published: 1979, + } + var published = get(book, "published", null) + = published + # => (Int 1979) + + var author = get(book, "author", "???") + = author + # => (Str "???") + ## Float ### floatsEqual() @@ -392,6 +435,18 @@ scope" rule.) If the variable isn't defined, `getVar()` returns `null`. So there's no way to distinguish an undefined variable from one that's `null`. +### `setVar()` + +Bind a name to a value, in the local scope. Returns nothing. + + call setVar('myname', 42) + +This is like + + setvar myname = 42 + +except the name can is a string, which can be constructed at runtime. + ### `parseCommand()` Given a code string, parse it as a command (with the current parse options). diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index bfa36acf0e..4d161b43d4 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -352,47 +352,6 @@ Reverses a list in place. A Dict contains an ordered sequence of key-value pairs. Given the key, the value can be retrieved efficiently. -### keys() - -Returns all existing keys from a dict as a list of strings. - - var en2fr = { - hello: "bonjour", - friend: "ami", - cat: "chat" - } - = en2fr => keys() - # => (List 0x4689) ["hello","friend","cat"] - -### values() - -Similar to `keys()`, but returns the values of the dictionary. - - var person = { - name: "Foo", - age: 25, - hobbies: :|walking reading| - } - = en2fr => values()] - # => (List 0x4689) ["Foo",25,["walking","reading"]] - -### get() - -Return value for given key, falling back to the default value if the key -doesn't exist. Default is required. - - var book = { - title: "Hitchhiker's Guide", - published: 1979, - } - var published = book => get("published", null) - = published - # => (Int 1979) - - var author = book => get("author", "???") - = author - # => (Str "???") - ### erase() Ensures that the given key does not exist in the dictionary. diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index c885c5c03c..dfc9434b3b 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -48,8 +48,7 @@ error handling, and more. search() leftMatch() [List] List/append() pop() extend() indexOf() X insert() X remove() reverse() - [Dict] keys() values() get() erase() - X inc() X accum() + [Dict] erase() X inc() X accum() [Range] [Eggex] [Match] group() start() end() @@ -76,6 +75,7 @@ X [Proc] name() location() toJson() X bytes() X encodeBytes() [Str] X strcmp() X split() shSplit() [List] join() + [Dict] keys() values() get() [Float] floatsEqual() X isinf() X isnan() [Obj] Object() prototype() propView() [Word] glob() maybe() @@ -83,7 +83,7 @@ X [Proc] name() location() toJson() toJson8() fromJson8() X [J8 Decode] J8.Bool() J8.Int() ... [Pattern] _group() _start() _end() - [Introspection] shvarGet() getVar() + [Introspection] shvarGet() getVar() setVar() parseCommand() X parseExpr() evalExpr() [Hay Config] parseHay() evalHay() X [Hashing] sha1dc() sha256() diff --git a/spec/ysh-TODO-deprecate.test.sh b/spec/ysh-TODO-deprecate.test.sh index b540a6029d..08ad723060 100644 --- a/spec/ysh-TODO-deprecate.test.sh +++ b/spec/ysh-TODO-deprecate.test.sh @@ -101,3 +101,16 @@ tac out 2 1 ## END + + +#### Dict => keys() +var en2fr = {} +setvar en2fr["hello"] = "bonjour" +setvar en2fr["friend"] = "ami" +setvar en2fr["cat"] = "chat" +pp test_ (en2fr => keys()) +## status: 0 +## STDOUT: +(List) ["hello","friend","cat"] +## END + diff --git a/spec/ysh-dict.test.sh b/spec/ysh-dict.test.sh index e721703832..7d8ae519ea 100644 --- a/spec/ysh-dict.test.sh +++ b/spec/ysh-dict.test.sh @@ -89,3 +89,23 @@ echo $v2 456 ## END + +#### keys(d), values(d), get(d, key) + +var d = {a: 42, b: 99} + +pp test_ (keys(d)) +pp test_ (values(d)) + +pp test_ (get(d, 'a', 'default')) +pp test_ (get(d, 'key', 'default')) + +## STDOUT: +(List) ["a","b"] +(List) [42,99] +(Int) 42 +(Str) "default" +## END + + + diff --git a/spec/ysh-expr-compare.test.sh b/spec/ysh-expr-compare.test.sh index 61e3df954c..d5a6c2b2b3 100644 --- a/spec/ysh-expr-compare.test.sh +++ b/spec/ysh-expr-compare.test.sh @@ -367,7 +367,7 @@ var unimpl = [ myexpr, # Expr ^(echo hello), # Block f, # Func - mydict=>keys, # BoundFunc + ''.upper, # BoundFunc # These cannot be constructed # - Proc # - Slice diff --git a/spec/ysh-methods.test.sh b/spec/ysh-methods.test.sh index 1744a1bb59..291f67121b 100644 --- a/spec/ysh-methods.test.sh +++ b/spec/ysh-methods.test.sh @@ -371,17 +371,6 @@ json write (b' \y00 ' => trimEnd()) " \u0000" ## END -#### Dict => keys() -var en2fr = {} -setvar en2fr["hello"] = "bonjour" -setvar en2fr["friend"] = "ami" -setvar en2fr["cat"] = "chat" -pp test_ (en2fr => keys()) -## status: 0 -## STDOUT: -(List) ["hello","friend","cat"] -## END - #### Str => split(sep), non-empty str sep pp test_ ('a,b,c'.split(',')) pp test_ ('aa'.split('a')) diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index fc0c5efed6..cd1ff242ee 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -1,5 +1,5 @@ ## our_shell: ysh -## oils_failures_allowed: 3 +## oils_failures_allowed: 2 #### Object() creates prototype chain @@ -194,23 +194,6 @@ pp test_ (y) (Str) "--foo" ## END - -#### Dict.keys(d), Dict.values(d), Dict.get(d, key) - -var d = {a: 42, b: 99} - -pp test_ (Dict.keys(d)) -pp test_ (Dict.values(d)) - -pp test_ (Dict.get(d, 'key', 'default')) - -# mutating methods are OK? -# call d->inc(x) - -## STDOUT: -## END - - #### Bound Proc? proc p (word1, word2; self, int1, int2) { diff --git a/spec/ysh-proc-meta.test.sh b/spec/ysh-proc-meta.test.sh index 996ec53d69..c4196ace65 100644 --- a/spec/ysh-proc-meta.test.sh +++ b/spec/ysh-proc-meta.test.sh @@ -46,6 +46,48 @@ prefix a prefix b ## END +#### with eval builtin command, making them global with names() and setVar() + +func genProcs() { + var result = {} + for param in a b { + eval """ + proc echo_$param(prefix) { + echo \$prefix $param + } + """ + setvar result["echo_$param"] = getVar("echo_$param") + } + + echo 'local' + echo_a prefix + echo_b prefix + echo + + return (result) +} + +var procs = genProcs() + +# bind to global scope +for name in (procs) { + call setVar("my_$name", procs[name]) +} + +echo 'global' +my_echo_a prefix +my_echo_b prefix + +## STDOUT: +local +prefix a +prefix b + +global +prefix a +prefix b +## END + #### with parseCommand() then io->eval(), in local scope proc p { From 47126c50752e8cc45ac0cd28056037eba9ecce35 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 28 Sep 2024 11:23:27 -0400 Subject: [PATCH 261/506] [test/spec] Refine and test definition of invokable Obj We should be able to hook it up to core/executor.py with this code. We get a (me, __invoke__) tuple, and then we can add me=Obj as a keyword argument. Do we need a BoundProc? It doesn't seem like it. The Obj itself is invokable. We don't need an (obj, method name) pair like we do in BoundFounc. --- builtin/meta_osh.py | 12 +++++- core/state.py | 89 ++++++++++++++++++++++++++++++++--------- spec/ysh-object.test.sh | 42 +++++++++++++++++++ spec/ysh-proc.test.sh | 20 +++++++-- 4 files changed, 138 insertions(+), 25 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index e6c3699418..e0f8fb9651 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -383,6 +383,16 @@ def _ResolveName( do_all, # type: bool ): # type: (...) -> List[Tuple[str, str, Optional[str]]] + """ + TODO: Can this be moved to pure YSH? + + All of these could be in YSH: + + type, type -t, type -a + pp proc + + We would have primitive isShellFunc() and isInvokableObj() functions + """ # MyPy tuple type no_str = None # type: Optional[str] @@ -395,7 +405,7 @@ def _ResolveName( if procs.IsProc(name): results.append((name, 'proc', no_str)) - elif procs.IsObj(name): # can't be both proc and obj + elif procs.IsInvokableObj(name): # can't be both proc and obj results.append((name, 'invokable', no_str)) if name in aliases: diff --git a/core/state.py b/core/state.py index 5ac26f7def..f81c433f02 100644 --- a/core/state.py +++ b/core/state.py @@ -2496,7 +2496,67 @@ def _AddNames(unique, frame): unique[name] = True -class Procs: +def _InvokableObj(val): + # type: (value_t) -> Optional[Tuple[Obj, value.Proc]] + """ + Returns: + None if the value is not invokable + (self Obj, __invoke__ Proc) if so + """ + if val.tag() != value_e.Obj: + return None + + obj = cast(Obj, val) + if not obj.prototype: + return None + + invoke_val = obj.prototype.d.get('__invoke__') + if invoke_val is None: + return None + + # TODO: __invoke__ of wrong type could be fatal error? + if invoke_val.tag() != value_e.Proc: + return None + + return obj, cast(value.Proc, invoke_val) + + +class Procs(object): + """ + Terminology: + + - invokable - these are INTERIOR + - value.Proc - which can be shell function in __sh_funcs__ namespace, or + YSH proc + - value.Obj with __invoke__ + - YSH runproc builtin, shell command/builtin, and type/type -a can be + generalized + - invoke --builtin + - do we need invoke --builtin-special ? This is POSIX + - invoke --proc myproc (42) + - invoke --sh-func + - invoke --obj + - invoke --external + - there is also 'keyword' and 'assign builtin' + - those are type- -a + - invoke --list-keywords + - invoke --list-assign + + - and you can combine the flags + - invoke --proc --sh-func --obj + - how about invoke --user-defined + - could be invoke -u + + - invoke --x-internal --no-builtin? + - x-internal can be a mask + - --no- can be a negation + + - with no args, print a table + - invoke --builtin + - invoke --proc + - and then you can parse that + - exterior - external commands + """ def __init__(self, mem): # type: (Mem) -> None @@ -2545,22 +2605,12 @@ def IsProc(self, name): # Could be Undef return maybe_proc.tag() == value_e.Proc - def IsObj(self, name): + def IsInvokableObj(self, name): # type: (str) -> bool - UP_obj = self.mem.GetValue(name) - if UP_obj.tag() != value_e.Obj: - return False - - obj = cast(Obj, UP_obj) - if not obj.prototype: - return False - - invoke = obj.prototype.d.get('__invoke__') - if invoke is None: - return False - - return invoke.tag() == value_e.Proc + val = self.mem.GetValue(name) + result = _InvokableObj(val) + return result is not None def InvokableNames(self): # type: () -> List[str] @@ -2570,7 +2620,7 @@ def InvokableNames(self): complete -A function pp proc - should deprecate this """ - unique = {} # type: Dict[str, bool] + unique = NewDict() # type: Dict[str, bool] for name in self.sh_funcs: unique[name] = True @@ -2597,11 +2647,10 @@ def GetInvokable(self, name): can shadow the definition of sh-funcs. Callers + executor.py: running + meta_osh.py runproc lookup - this is not 'invoke', because it is + INTERIOR shell functions, procs, invokable Obj cmd_eval: check for redefining proc or sh-func (remove) - lookup for runproc - does this find sh-funcs too? - pp proc - complete -F myfunc - declare -p - should not print procs, only shell stuff """ maybe_proc = self.mem.GetValue(name) if maybe_proc.tag() == value_e.Proc: diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index cd1ff242ee..0c6e0f7b43 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -213,3 +213,45 @@ callable a b (42, 43) ## STDOUT: ## END + +#### invokable object must be value.Obj with prototype containing __invoke__ of value.Proc + +proc p (w; self) { + pp test_ ([w, self]) +} +p a ({x: 5, y: 6}) +echo + +var methods = Object(null, {__invoke__: p}) + +var o1 = Object(methods, {}) +type -t o1 +echo + +# errors + +var o2 = Object(null, {}) +if ! type -t o2 { + echo 'no prototype' +} + +var o3 = Object(Object(null, {}), {}) +if ! type -t o3 { + echo 'no __invoke__ method in prototype' +} + +var bad_methods = Object(null, {__invoke__: 42}) +var o4 = Object(bad_methods, {}) +if ! type -t o4 { + echo '__invoke__ of wrong type' +} + +## STDOUT: +(List) ["a",{"x":5,"y":6}] + +invokable + +no prototype +no __invoke__ method in prototype +__invoke__ of wrong type +## END diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 33afb18b31..706ee02640 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 0 +## oils_failures_allowed: 1 #### Open proc (any number of args) shopt --set parse_proc @@ -281,21 +281,33 @@ compgen -A function echo --- -proc p { +proc define-inner { eval 'proc inner { echo inner }' #eval 'proc myproc { echo inner }' # shadowed name compgen -A function } -p +define-inner + +echo --- + +proc myinvoke (w; self) { + pp test_ ([w, self]) +} + +var methods = Object(null, {__invoke__: myinvoke}) +var myobj = Object(methods, {}) + +compgen -A function ## STDOUT: my-shell-func myproc --- +define-inner inner my-shell-func myproc -p +--- ## END #### type / type -a builtin on invokables - shell func, proc, invokable From 5a9eb69f90000730c744714ac0650aeadc25d427 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 28 Sep 2024 19:17:48 -0400 Subject: [PATCH 262/506] [osh, ysh] Invokable Obj definition is respected by compgen -A function Next, we can hook it up to the executor. Also add stubs for 2 builtins: - invoke - to generalize runproc - runproc could be an alias for invoke --proc --obj, or invoke --proc-like - extern - for passing a custom environment --- builtin/meta_osh.py | 70 +++++++++++++++++++++++++++++++++++++++++ core/shell.py | 2 ++ core/state.py | 57 +++++++-------------------------- frontend/builtin_def.py | 2 ++ frontend/flag_def.py | 3 ++ spec/ysh-object.test.sh | 42 ++++++++++++------------- spec/ysh-proc.test.sh | 9 ++++-- 7 files changed, 117 insertions(+), 68 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index e0f8fb9651..076ec961f1 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -375,6 +375,76 @@ def Run(self, cmd_val): return self.shell_ex.RunSimpleCommand(cmd_val2, cmd_st, run_flags) +class Invoke(vm._Builtin): + """ + invoke - YSH introspection on first word + type --all - introspection on variables too? + - different than = type(x) + + - invoke --builtin + - do we need invoke --builtin-special ? This is POSIX + - invoke --proc myproc (42) + - invoke --sh-func + - invoke --obj + - invoke --external + - there is also 'keyword' and 'assign builtin' + - those are type- -a + - invoke --list-keywords + - invoke --list-assign + + - and you can combine the flags + - invoke --proc --sh-func --obj + - how about invoke --user-defined + - could be invoke -u + + - invoke --x-internal --no-builtin? + - x-internal can be a mask + - --no- can be a negation + + - with no args, print a table + - invoke --builtin + - invoke --proc + - and then you can parse that + """ + + def __init__(self, shell_ex, procs, errfmt): + # type: (vm._Executor, state.Procs, ui.ErrorFormatter) -> None + self.shell_ex = shell_ex + self.procs = procs + self.errfmt = errfmt + + def Run(self, cmd_val): + # type: (cmd_value.Argv) -> int + _, arg_r = flag_util.ParseCmdVal('invoke', + cmd_val, + accept_typed_args=True) + #argv, locs = arg_r.Rest2() + + print('TODO: invoke') + # TODO + return 0 + + +class Extern(vm._Builtin): + + def __init__(self, shell_ex, procs, errfmt): + # type: (vm._Executor, state.Procs, ui.ErrorFormatter) -> None + self.shell_ex = shell_ex + self.procs = procs + self.errfmt = errfmt + + def Run(self, cmd_val): + # type: (cmd_value.Argv) -> int + _, arg_r = flag_util.ParseCmdVal('extern', + cmd_val, + accept_typed_args=True) + #argv, locs = arg_r.Rest2() + + print('TODO: extern') + + return 0 + + def _ResolveName( name, # type: str procs, # type: state.Procs diff --git a/core/shell.py b/core/shell.py index fe41f252d5..0aa9d81660 100644 --- a/core/shell.py +++ b/core/shell.py @@ -628,6 +628,8 @@ def Main( search_path) # Part of YSH, but similar to builtin/command b[builtin_i.runproc] = meta_osh.RunProc(shell_ex, procs, errfmt) + b[builtin_i.invoke] = meta_osh.Invoke(shell_ex, procs, errfmt) + b[builtin_i.extern_] = meta_osh.Extern(shell_ex, procs, errfmt) # Meta builtins source_builtin = meta_osh.Source(parse_ctx, search_path, cmd_ev, fd_state, diff --git a/core/state.py b/core/state.py index f81c433f02..4d3f400f33 100644 --- a/core/state.py +++ b/core/state.py @@ -2488,14 +2488,6 @@ def PopContextStack(self): return self.ctx_stack.pop() -def _AddNames(unique, frame): - # type: (Dict[str, bool], Dict[str, Cell]) -> None - for name in frame: - cell = frame[name] - if cell.val.tag() == value_e.Proc: - unique[name] = True - - def _InvokableObj(val): # type: (value_t) -> Optional[Tuple[Obj, value.Proc]] """ @@ -2521,6 +2513,14 @@ def _InvokableObj(val): return obj, cast(value.Proc, invoke_val) +def _AddNames(unique, frame): + # type: (Dict[str, bool], Dict[str, Cell]) -> None + for name in frame: + val = frame[name].val + if val.tag() == value_e.Proc or _InvokableObj(val) is not None: + unique[name] = True + + class Procs(object): """ Terminology: @@ -2529,35 +2529,11 @@ class Procs(object): - value.Proc - which can be shell function in __sh_funcs__ namespace, or YSH proc - value.Obj with __invoke__ - - YSH runproc builtin, shell command/builtin, and type/type -a can be - generalized - - invoke --builtin - - do we need invoke --builtin-special ? This is POSIX - - invoke --proc myproc (42) - - invoke --sh-func - - invoke --obj - - invoke --external - - there is also 'keyword' and 'assign builtin' - - those are type- -a - - invoke --list-keywords - - invoke --list-assign - - - and you can combine the flags - - invoke --proc --sh-func --obj - - how about invoke --user-defined - - could be invoke -u - - - invoke --x-internal --no-builtin? - - x-internal can be a mask - - --no- can be a negation - - - with no args, print a table - - invoke --builtin - - invoke --proc - - and then you can parse that - - exterior - external commands - """ + - exterior - external commands, extern builtin + Note: the YSH 'invoke' builtin can generalize YSH 'runproc' builtin, shell command/builtin, + and also type / type -a + """ def __init__(self, mem): # type: (Mem) -> None self.mem = mem @@ -2656,15 +2632,6 @@ def GetInvokable(self, name): if maybe_proc.tag() == value_e.Proc: return cast(value.Proc, maybe_proc) - if maybe_proc.tag() == value_e.Obj: - obj = cast(Obj, maybe_proc) - # Now does it have - - # Error cases for proc lookup: - # 1. value.Int - # 2. value.Obj with __invoke__, but it's not a value.Proc - # 2. value.Obj without __invoke__ - if name in self.sh_funcs: return self.sh_funcs[name] diff --git a/frontend/builtin_def.py b/frontend/builtin_def.py index e1689a9552..8ee6e38551 100644 --- a/frontend/builtin_def.py +++ b/frontend/builtin_def.py @@ -62,6 +62,7 @@ 'shvar', 'ctx', + 'invoke', 'runproc', 'boolstatus', ] @@ -124,6 +125,7 @@ def _Init(b): b.Add(name, kind='assign') b.Add('export', enum_name='export_', kind='assign') # C++ keyword conflict + b.Add('extern', enum_name='extern_') b.Add('true', enum_name='true_') # C++ Keywords b.Add('false', enum_name='false_') b.Add('try', enum_name='try_') diff --git a/frontend/flag_def.py b/frontend/flag_def.py index 8df5019623..69a5adf81c 100644 --- a/frontend/flag_def.py +++ b/frontend/flag_def.py @@ -467,6 +467,9 @@ def _DefineCompletionActions(spec): RUNPROC_SPEC = FlagSpec('runproc') RUNPROC_SPEC.ShortFlag('-h', args.Bool, help='Show all procs') +INVOKE_SPEC = FlagSpec('invoke') +EXTERN_SPEC = FlagSpec('extern') + WRITE_SPEC = FlagSpec('write') WRITE_SPEC.LongFlag('--sep', args.String, diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index 0c6e0f7b43..02f5ae2e07 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -194,27 +194,7 @@ pp test_ (y) (Str) "--foo" ## END -#### Bound Proc? - -proc p (word1, word2; self, int1, int2) { - echo "sum = $[self.x + self.y]" - pp test_ (self) - pp test_ ([word1, word2, int1, int2]) -} - -p a b ({x: 5, y: 6}, 42, 43) - -var methods = Object(null, {__invoke__: p}) - -var callable = Object(methods, {x: 98, y: 99}) - -# TODO: change this error message -callable a b (42, 43) - -## STDOUT: -## END - -#### invokable object must be value.Obj with prototype containing __invoke__ of value.Proc +#### invokable Obj must be have prototype containing __invoke__ of value.Proc - type -t proc p (w; self) { pp test_ ([w, self]) @@ -255,3 +235,23 @@ no prototype no __invoke__ method in prototype __invoke__ of wrong type ## END + +#### Use Invokable Obj + +proc p (word1, word2; self, int1, int2) { + echo "sum = $[self.x + self.y]" + pp test_ (self) + pp test_ ([word1, word2, int1, int2]) +} + +p a b ({x: 5, y: 6}, 42, 43) + +var methods = Object(null, {__invoke__: p}) + +var callable = Object(methods, {x: 98, y: 99}) + +# TODO: change this error message +callable a b (42, 43) + +## STDOUT: +## END diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 706ee02640..8451e60b8d 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 #### Open proc (any number of args) shopt --set parse_proc @@ -266,7 +266,7 @@ status=1 status=1 ## END -#### compgen -A function completes all invokables - shell funcs, Proc, Obj +#### compgen -A function shows user-defined invokables - shell funcs, Proc, Obj shopt --set ysh:upgrade my-shell-func() { @@ -308,6 +308,11 @@ inner my-shell-func myproc --- +define-inner +my-shell-func +myinvoke +myobj +myproc ## END #### type / type -a builtin on invokables - shell func, proc, invokable From b872fc7863dbbd458769e83bc508173b50c21a70 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 28 Sep 2024 19:36:32 -0400 Subject: [PATCH 263/506] [test/spec] Test cases for invokable obj Also write comments on the design of 'invoke'. I think it is still needed. --- builtin/meta_osh.py | 53 +++--- core/state.py | 1 + frontend/flag_def.py | 6 + spec/ysh-proc-meta.test.sh | 7 + spec/ysh-proc.test.sh | 327 ++++++++++++++++++++----------------- 5 files changed, 223 insertions(+), 171 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index 076ec961f1..a646715e68 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -377,36 +377,39 @@ def Run(self, cmd_val): class Invoke(vm._Builtin): """ + Introspection: + invoke - YSH introspection on first word type --all - introspection on variables too? - different than = type(x) - - invoke --builtin - - do we need invoke --builtin-special ? This is POSIX - - invoke --proc myproc (42) - - invoke --sh-func - - invoke --obj - - invoke --external - - there is also 'keyword' and 'assign builtin' - - those are type- -a - - invoke --list-keywords - - invoke --list-assign - - - and you can combine the flags - - invoke --proc --sh-func --obj - - how about invoke --user-defined - - could be invoke -u - - - invoke --x-internal --no-builtin? - - x-internal can be a mask - - --no- can be a negation - - - with no args, print a table - - invoke --builtin - - invoke --proc - - and then you can parse that - """ + 3 Coarsed-grained categories + - invoke --builtin aka builtin + - including special builtins + - invoke --proc-like aka runproc + - myproc (42) + - sh-func + - invokable-obj + - invoke --extern aka extern + + Note: If you don't distinguish between proc, sh-func, and invokable-obj, + then 'runproc' suffices. + + invoke --proc-like reads more nicely though, and it also combines. + + invoke --builtin --extern # this is like 'command' + You can also negate: + + invoke --no-proc-like --no-builtin --no-extern + + - type -t also has 'keyword' and 'assign builtin' + + With no args, print a table of what's available + + invoke --builtin + invoke --builtin true + """ def __init__(self, shell_ex, procs, errfmt): # type: (vm._Executor, state.Procs, ui.ErrorFormatter) -> None self.shell_ex = shell_ex diff --git a/core/state.py b/core/state.py index 4d3f400f33..4b2b6f224c 100644 --- a/core/state.py +++ b/core/state.py @@ -2534,6 +2534,7 @@ class Procs(object): Note: the YSH 'invoke' builtin can generalize YSH 'runproc' builtin, shell command/builtin, and also type / type -a """ + def __init__(self, mem): # type: (Mem) -> None self.mem = mem diff --git a/frontend/flag_def.py b/frontend/flag_def.py index 69a5adf81c..759aa13937 100644 --- a/frontend/flag_def.py +++ b/frontend/flag_def.py @@ -468,6 +468,12 @@ def _DefineCompletionActions(spec): RUNPROC_SPEC.ShortFlag('-h', args.Bool, help='Show all procs') INVOKE_SPEC = FlagSpec('invoke') + +# 3 coarse-grained categories. +INVOKE_SPEC.LongFlag('--builtin') # like 'builtin', which includs special builtins +INVOKE_SPEC.LongFlag('--proc-like') # like 'runproc' - proc, sh func, or invokable obj +INVOKE_SPEC.LongFlag('--extern') # like 'extern' builtin + EXTERN_SPEC = FlagSpec('extern') WRITE_SPEC = FlagSpec('write') diff --git a/spec/ysh-proc-meta.test.sh b/spec/ysh-proc-meta.test.sh index c4196ace65..eeb6fd6497 100644 --- a/spec/ysh-proc-meta.test.sh +++ b/spec/ysh-proc-meta.test.sh @@ -207,6 +207,13 @@ for param in a b { var my_echo_a = procs.echo_a var my_echo_b = procs.echo_b +if false { + = my_echo_a + = my_echo_b + type -t my_echo_a + type -t my_echo_b +} + # Maybe show an error if this is not value.Obj? my_echo_a prefix my_echo_b prefix diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 8451e60b8d..010abbe041 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 0 +## oils_failures_allowed: 2 #### Open proc (any number of args) shopt --set parse_proc @@ -238,151 +238,6 @@ p ## STDOUT: ## END -#### declare -f -F only prints shell functions -shopt --set parse_proc - -myfunc() { - echo hi -} - -proc myproc { - echo hi -} - -declare -F -echo --- - -declare -F myproc -echo status=$? - -declare -f myproc -echo status=$? - -## status: 0 -## STDOUT: -declare -f myfunc ---- -status=1 -status=1 -## END - -#### compgen -A function shows user-defined invokables - shell funcs, Proc, Obj -shopt --set ysh:upgrade - -my-shell-func() { - echo hi -} - -proc myproc { - echo hi -} - -compgen -A function - -echo --- - -proc define-inner { - eval 'proc inner { echo inner }' - #eval 'proc myproc { echo inner }' # shadowed name - compgen -A function -} -define-inner - -echo --- - -proc myinvoke (w; self) { - pp test_ ([w, self]) -} - -var methods = Object(null, {__invoke__: myinvoke}) -var myobj = Object(methods, {}) - -compgen -A function - -## STDOUT: -my-shell-func -myproc ---- -define-inner -inner -my-shell-func -myproc ---- -define-inner -my-shell-func -myinvoke -myobj -myproc -## END - -#### type / type -a builtin on invokables - shell func, proc, invokable -shopt --set ysh:upgrade - -my-shell-func() { - echo hi -} - -proc myproc { - echo hi -} - -proc boundProc(; self) { - echo hi -} - -var methods = Object(null, {__invoke__: boundProc}) -var invokable = Object(methods, {}) - -type -t my-shell-func -type -t myproc -type -t invokable -try { - type -t methods # not invokable! -} -echo $[_error.code] - -echo --- - -type my-shell-func -type myproc -type invokable -try { - type methods # not invokable! -} -echo $[_error.code] - -echo --- - -type -a my-shell-func -type -a myproc -type -a invokable - -echo --- - -if false { # can't redefine right now - invokable() { - echo sh-func - } - type -a invokable -} - -## STDOUT: -function -proc -invokable -1 ---- -my-shell-func is a shell function -myproc is a YSH proc -invokable is a YSH invokable -1 ---- -my-shell-func is a shell function -myproc is a YSH proc -invokable is a YSH invokable ---- -## END - #### procs are in same namespace as variables shopt --set parse_proc @@ -706,3 +561,183 @@ if false { ## STDOUT: [frame_vars_] ARGV localproc ## END + + +#### declare -f -F only prints shell functions +shopt --set parse_proc + +myfunc() { + echo hi +} + +proc myproc { + echo hi +} + +declare -F +echo --- + +declare -F myproc +echo status=$? + +declare -f myproc +echo status=$? + +## status: 0 +## STDOUT: +declare -f myfunc +--- +status=1 +status=1 +## END + +#### compgen -A function shows user-defined invokables - shell funcs, Proc, Obj +shopt --set ysh:upgrade + +my-shell-func() { + echo hi +} + +proc myproc { + echo hi +} + +compgen -A function + +echo --- + +proc define-inner { + eval 'proc inner { echo inner }' + #eval 'proc myproc { echo inner }' # shadowed name + compgen -A function +} +define-inner + +echo --- + +proc myinvoke (w; self) { + pp test_ ([w, self]) +} + +var methods = Object(null, {__invoke__: myinvoke}) +var myobj = Object(methods, {}) + +compgen -A function + +## STDOUT: +my-shell-func +myproc +--- +define-inner +inner +my-shell-func +myproc +--- +define-inner +my-shell-func +myinvoke +myobj +myproc +## END + +#### type / type -a builtin on invokables - shell func, proc, invokable +shopt --set ysh:upgrade + +my-shell-func() { + echo hi +} + +proc myproc { + echo hi +} + +proc boundProc(; self) { + echo hi +} + +var methods = Object(null, {__invoke__: boundProc}) +var invokable = Object(methods, {}) + +type -t my-shell-func +type -t myproc +type -t invokable +try { + type -t methods # not invokable! +} +echo $[_error.code] + +echo --- + +type my-shell-func +type myproc +type invokable +try { + type methods # not invokable! +} +echo $[_error.code] + +echo --- + +type -a my-shell-func +type -a myproc +type -a invokable + +echo --- + +if false { # can't redefine right now + invokable() { + echo sh-func + } + type -a invokable +} + +## STDOUT: +function +proc +invokable +1 +--- +my-shell-func is a shell function +myproc is a YSH proc +invokable is a YSH invokable +1 +--- +my-shell-func is a shell function +myproc is a YSH proc +invokable is a YSH invokable +--- +## END + +#### call invokable Obj with self +shopt --set ysh:upgrade + +proc boundProc(; self) { + echo "sum = $[self.x + self.y]" +} + +var methods = Object(null, {__invoke__: boundProc}) +var invokable = Object(methods, {x: 3, y: 5}) + +invokable + +## STDOUT: +## END + +#### two different objects can share the same __invoke__ +shopt --set ysh:upgrade + +proc boundProc(; self) { + echo "sum = $[self.x + self.y]" +} + +var methods = Object(null, {__invoke__: boundProc}) + +var i1 = Object(methods, {x: 3, y: 5}) +var i2 = Object(methods, {x: 10, y: 42}) + +i1 +i2 + +## STDOUT: + +## END From 43ad0f298c226d91451394f61a33846f686c043e Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 28 Sep 2024 19:57:43 -0400 Subject: [PATCH 264/506] [translation] Fix conflict with --extern flag and C++ extern keyword Rename me -> self_val --- builtin/meta_osh.py | 1 + frontend/flag_def.py | 8 +++----- frontend/flag_gen.py | 14 +++++++++++--- osh/cmd_eval.py | 6 +++--- ysh/expr_eval.py | 2 +- ysh/func_proc.py | 13 +++++++------ 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index a646715e68..cd43538d2e 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -410,6 +410,7 @@ class Invoke(vm._Builtin): invoke --builtin invoke --builtin true """ + def __init__(self, shell_ex, procs, errfmt): # type: (vm._Executor, state.Procs, ui.ErrorFormatter) -> None self.shell_ex = shell_ex diff --git a/frontend/flag_def.py b/frontend/flag_def.py index 759aa13937..b031e3c798 100644 --- a/frontend/flag_def.py +++ b/frontend/flag_def.py @@ -468,11 +468,9 @@ def _DefineCompletionActions(spec): RUNPROC_SPEC.ShortFlag('-h', args.Bool, help='Show all procs') INVOKE_SPEC = FlagSpec('invoke') - -# 3 coarse-grained categories. -INVOKE_SPEC.LongFlag('--builtin') # like 'builtin', which includs special builtins -INVOKE_SPEC.LongFlag('--proc-like') # like 'runproc' - proc, sh func, or invokable obj -INVOKE_SPEC.LongFlag('--extern') # like 'extern' builtin +INVOKE_SPEC.LongFlag('--builtin') # like 'builtin' +INVOKE_SPEC.LongFlag('--proc-like') # like 'runproc' +INVOKE_SPEC.LongFlag('--extern') # like 'extern' EXTERN_SPEC = FlagSpec('extern') diff --git a/frontend/flag_gen.py b/frontend/flag_gen.py index b41b02f65b..589ad39429 100755 --- a/frontend/flag_gen.py +++ b/frontend/flag_gen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -"""Flag_gen.py.""" +""" flag_gen.py - generate Python and C++ from flag specs """ from __future__ import print_function import itertools @@ -32,6 +32,14 @@ def CString(s): return '"%s"' % s +def _CleanFieldName(name): + # Avoid C++ keyword for invoke --extern + if name == 'extern': + return 'extern_' + + return name.replace('-', '_') + + def _WriteStrArray(f, var_name, a): c_strs = ', '.join(CString(s) for s in sorted(a)) f.write('const char* %s[] = {%s, nullptr};\n' % (var_name, c_strs)) @@ -206,7 +214,7 @@ def Cpp(specs, header_f, cc_f): bits = [] for field_name in sorted(spec.fields): typ = spec.fields[field_name] - field_name = field_name.replace('-', '_') + field_name = _CleanFieldName(field_name) field_names.append(field_name) with switch(typ) as case: @@ -485,7 +493,7 @@ def __init__(self, attrs): i = 0 for field_name in sorted(spec.fields): typ = spec.fields[field_name] - field_name = field_name.replace('-', '_') + field_name = _CleanFieldName(field_name) with switch(typ) as case: if case(flag_type_e.Bool): diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index d2b8d6bc48..5e533a8ee6 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -2207,8 +2207,8 @@ def _MaybeRunErrTrap(self): with state.ctx_ErrTrap(self.mem): self._Execute(node) - def RunProc(self, proc, cmd_val): - # type: (value.Proc, cmd_value.Argv) -> int + def RunProc(self, proc, cmd_val, self_val=None): + # type: (value.Proc, cmd_value.Argv, value_t) -> int """Run procs aka "shell functions". For SimpleCommand and registered completion hooks. @@ -2222,7 +2222,7 @@ def RunProc(self, proc, cmd_val): # Hm this sets "$@". TODO: Set ARGV only with state.ctx_ProcCall(self.mem, self.mutable_opts, proc, proc_argv): - func_proc.BindProcArgs(proc, cmd_val, self.mem) + func_proc.BindProcArgs(proc, cmd_val, self.mem, self_val=self_val) # Redirects still valid for functions. # Here doc causes a pipe and Process(SubProgramThunk). diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 79b30e4877..a4335f17ee 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -863,7 +863,7 @@ def _EvalFuncCall(self, node): to_call = func.func pos_args, named_args = func_proc._EvalArgList(self, node.args, - me=func.me) + self_val=func.me) rd = typed_args.Reader(pos_args, named_args, None, diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 86c061df48..556ea1c359 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -180,7 +180,7 @@ def _EvalNamedArgs(expr_ev, named_exprs): def _EvalArgList( expr_ev, # type: expr_eval.ExprEvaluator args, # type: ArgList - me=None # type: Optional[value_t] + self_val=None # type: Optional[value_t] ): # type: (...) -> Tuple[List[value_t], Optional[Dict[str, value_t]]] """Evaluate arg list for funcs. @@ -195,8 +195,8 @@ def _EvalArgList( """ pos_args = [] # type: List[value_t] - if me: # self/this argument - pos_args.append(me) + if self_val: # self/this argument + pos_args.append(self_val) _EvalPosArgs(expr_ev, args.pos_args, pos_args) @@ -451,8 +451,8 @@ def _BindFuncArgs(func, rd, mem): (func.name, num_named), blame_loc) -def BindProcArgs(proc, cmd_val, mem): - # type: (value.Proc, cmd_value.Argv, state.Mem) -> None +def BindProcArgs(proc, cmd_val, mem, self_val=None): + # type: (value.Proc, cmd_value.Argv, state.Mem, value_t) -> None proc_args = cmd_val.proc_args @@ -489,7 +489,8 @@ def BindProcArgs(proc, cmd_val, mem): blame_loc = proc_args.typed_args.left pos_args = proc_args.pos_args if proc_args else None - if sig.positional: # or sig.block_param: + if sig.positional: + # TODO: Add self_val _BindTyped(proc.name, sig.positional, proc.defaults.for_typed, pos_args, mem, blame_loc) else: From c0e01406539ca554747e2b969761d5e1eb2a3022 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 28 Sep 2024 20:19:14 -0400 Subject: [PATCH 265/506] [ysh] Implement invokable Obj with __invoke__ magic method Documented in - doc/ref/toc-ysh - doc/proc-func See #language-design Zulip for the motivation. Summary: It started as a case in spec/ysh-proc-meta, motivated by generating procs dynamically. We had solutions based on: 1. eval $mystr 2. parseCommand() io->eval() Now we have a solution based on: 3. Invokable objects. The __invoke__ method makes an Obj instance "proc-like". I think this will be big! For "maximalist YSH". We can use it for the ctx builtin, which is used by the flag parser. And Hay, Markaby-style HTML generation, ... --- builtin/io_ysh.py | 5 ++--- builtin/meta_osh.py | 3 ++- core/executor.py | 6 ++++-- core/state.py | 40 ++++++++++++++++++++++--------------- doc/proc-func.md | 18 +++++++++++++++++ doc/ref/chap-type-method.md | 28 ++++++++++++++++++++++++++ doc/ref/toc-ysh.md | 1 + osh/cmd_eval.py | 8 ++++---- spec/ysh-proc-meta.test.sh | 2 +- spec/ysh-proc.test.sh | 30 ++++++++++++++++++++++------ ysh/func_proc.py | 16 ++++++++++++--- 11 files changed, 121 insertions(+), 36 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 2d7b23d87d..22255e4044 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -203,7 +203,7 @@ def Run(self, cmd_val): names, locs = arg_r.Rest2() if len(names): for i, name in enumerate(names): - node = self.procs.GetInvokable(name) + node, _ = self.procs.GetInvokable(name) if node is None: self.errfmt.Print_('Invalid proc %r' % name, blame_loc=locs[i]) @@ -214,8 +214,7 @@ def Run(self, cmd_val): # TSV8 header print('proc_name\tdoc_comment') for name in names: - proc = self.procs.GetInvokable(name) # must exist - #log('Proc %s', proc) + proc, _ = self.procs.GetInvokable(name) # must exist body = proc.body # TODO: not just command.ShFunction, but command.Proc! diff --git a/builtin/meta_osh.py b/builtin/meta_osh.py index cd43538d2e..2f5eb3900f 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_osh.py @@ -362,7 +362,8 @@ def Run(self, cmd_val): raise error.Usage('requires arguments', loc.Missing) name = argv[0] - if not self.procs.GetInvokable(name): + proc, _ = self.procs.GetInvokable(name) + if not proc: # note: should runproc be invoke? self.errfmt.PrintMessage('runproc: no invokable named %r' % name) return 1 diff --git a/core/executor.py b/core/executor.py index b332dce21d..8396178754 100644 --- a/core/executor.py +++ b/core/executor.py @@ -279,7 +279,7 @@ def RunSimpleCommand(self, cmd_val, cmd_st, run_flags): # Pitfall: What happens if there are two of the same name? I guess # that's why you have = and 'type' inspect them - proc_node = self.procs.GetInvokable(arg0) + proc_node, self_val = self.procs.GetInvokable(arg0) if proc_node is not None: if self.exec_opts.strict_errexit(): disabled_tok = self.mutable_opts.ErrExitDisabledToken() @@ -295,7 +295,9 @@ def RunSimpleCommand(self, cmd_val, cmd_st, run_flags): with dev.ctx_Tracer(self.tracer, 'proc', argv): # NOTE: Functions could call 'exit 42' directly, etc. - status = self.cmd_ev.RunProc(proc_node, cmd_val) + status = self.cmd_ev.RunProc(proc_node, + cmd_val, + self_val=self_val) return status # Notes: diff --git a/core/state.py b/core/state.py index 4b2b6f224c..b71d64129f 100644 --- a/core/state.py +++ b/core/state.py @@ -2489,35 +2489,38 @@ def PopContextStack(self): def _InvokableObj(val): - # type: (value_t) -> Optional[Tuple[Obj, value.Proc]] + # type: (value_t) -> Tuple[Optional[value.Proc], Optional[Obj]] """ Returns: None if the value is not invokable (self Obj, __invoke__ Proc) if so """ if val.tag() != value_e.Obj: - return None + return None, None obj = cast(Obj, val) if not obj.prototype: - return None + return None, None invoke_val = obj.prototype.d.get('__invoke__') if invoke_val is None: - return None + return None, None # TODO: __invoke__ of wrong type could be fatal error? if invoke_val.tag() != value_e.Proc: - return None + return None, None - return obj, cast(value.Proc, invoke_val) + return cast(value.Proc, invoke_val), obj def _AddNames(unique, frame): # type: (Dict[str, bool], Dict[str, Cell]) -> None for name in frame: val = frame[name].val - if val.tag() == value_e.Proc or _InvokableObj(val) is not None: + if val.tag() == value_e.Proc: + unique[name] = True + proc, _ = _InvokableObj(val) + if proc is not None: unique[name] = True @@ -2586,8 +2589,8 @@ def IsInvokableObj(self, name): # type: (str) -> bool val = self.mem.GetValue(name) - result = _InvokableObj(val) - return result is not None + proc, self_val = _InvokableObj(val) + return proc is not None def InvokableNames(self): # type: () -> List[str] @@ -2617,26 +2620,31 @@ def InvokableNames(self): return names def GetInvokable(self, name): - # type: (str) -> value.Proc + # type: (str) -> Tuple[Optional[value.Proc], Optional[Obj]] """Try to find a proc/sh-func by `name`, or return None if not found. First, we search for a proc, and then a sh-func. This means that procs can shadow the definition of sh-funcs. - Callers + Callers: executor.py: running meta_osh.py runproc lookup - this is not 'invoke', because it is INTERIOR shell functions, procs, invokable Obj cmd_eval: check for redefining proc or sh-func (remove) """ - maybe_proc = self.mem.GetValue(name) - if maybe_proc.tag() == value_e.Proc: - return cast(value.Proc, maybe_proc) + val = self.mem.GetValue(name) + + if val.tag() == value_e.Proc: + return cast(value.Proc, val), None + + proc, self_val = _InvokableObj(val) + if proc: + return proc, self_val if name in self.sh_funcs: - return self.sh_funcs[name] + return self.sh_funcs[name], None - return None + return None, None # diff --git a/doc/proc-func.md b/doc/proc-func.md index 7b9166d09a..92a7ed4c33 100644 --- a/doc/proc-func.md +++ b/doc/proc-func.md @@ -770,6 +770,24 @@ operators: - Thin arrow (`->`) looks for mutating methods, which have an `M/` prefix. - Reference: [thin-arrow](ref/chap-expr-lang.html#thin-arrow) +## The `__invoke__` method makes an Object "Proc-like" + +First, define a proc, with the first typed arg named `self`: + + proc myInvoke (word_param; self, int_param) { + echo "sum = $[self.x + self.y + int_param]" + } + +Make it the `__invoke__` method of an `Obj`: + + var methods = Object(null, {__invoke__: myInvoke}) + var invokable_obj = Object(methods, {x: 1, y: 2}) + +Then invoke it like a proc: + + invokable_obj myword (3) + # sum => 6 + ## Usage Notes ### 3 Ways to Return a Value diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 4d161b43d4..3b089913bc 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -589,3 +589,31 @@ database), and then C strftime(). TODO: The free function glob() actually does I/O. Although maybe it doesn't fail? +## Obj + +### `__invoke__` + + + +The `__invoke__` method makes an Object "proc-like". + +First, define a proc, with the first typed arg named `self`: + + proc myInvoke (word_param; self, int_param) { + echo "sum = $[self.x + self.y + int_param]" + } + +Make it the `__invoke__` method of an `Obj`: + + var methods = Object(null, {__invoke__: myInvoke}) + var invokable_obj = Object(methods, {x: 1, y: 2}) + +Then invoke it like a proc: + + invokable_obj myword (3) + # sum => 6 + +### `__call__` + +TODO + diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index dfc9434b3b..9011e4f8bf 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -61,6 +61,7 @@ X [Proc] name() location() toJson() [IO] eval() evalToDict() captureStdout() promptVal() X time() X strftime() X glob() + [Obj] __invoke__ X __call__ ```

diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 5e533a8ee6..0799b80ac4 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1295,8 +1295,8 @@ def _DoForExpr(self, node): def _DoShFunction(self, node): # type: (command.ShFunction) -> None - if (self.procs.GetInvokable(node.name) and - not self.exec_opts.redefine_proc_func()): + existing, _ = self.procs.GetInvokable(node.name) + if existing and not self.exec_opts.redefine_proc_func(): e_die( "Function %s was already defined (redefine_proc_func)" % node.name, node.name_tok) @@ -1314,8 +1314,8 @@ def _DoProc(self, node): # conflicts # We could also define procs as READ-ONLY, but that means we need # Dict[str, Cell] and not Dict[str, value_t] - if (self.procs.GetInvokable(proc_name) and - not self.exec_opts.redefine_proc_func()): + existing, _ = self.procs.GetInvokable(proc_name) + if existing and not self.exec_opts.redefine_proc_func(): e_die( "Proc %s was already defined (redefine_proc_func)" % proc_name, node.name) diff --git a/spec/ysh-proc-meta.test.sh b/spec/ysh-proc-meta.test.sh index eeb6fd6497..1af0c8b663 100644 --- a/spec/ysh-proc-meta.test.sh +++ b/spec/ysh-proc-meta.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 ## our_shell: ysh # dynamically generate procs diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 010abbe041..5de6b9b6c7 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 2 +## oils_failures_allowed: 0 #### Open proc (any number of args) shopt --set parse_proc @@ -708,6 +708,22 @@ invokable is a YSH invokable --- ## END +#### invokable Obj that doesn't declare self +shopt --set ysh:upgrade + +proc boundProc(no_self; ) { + echo 'bad' +} + +var methods = Object(null, {__invoke__: boundProc}) +var invokable = Object(methods, {x: 3, y: 5}) + +invokable no_self + +## status: 3 +## STDOUT: +## END + #### call invokable Obj with self shopt --set ysh:upgrade @@ -721,13 +737,14 @@ var invokable = Object(methods, {x: 3, y: 5}) invokable ## STDOUT: +sum = 8 ## END #### two different objects can share the same __invoke__ shopt --set ysh:upgrade -proc boundProc(; self) { - echo "sum = $[self.x + self.y]" +proc boundProc(; self, more) { + echo "sum = $[self.x + self.y + more]" } var methods = Object(null, {__invoke__: boundProc}) @@ -735,9 +752,10 @@ var methods = Object(null, {__invoke__: boundProc}) var i1 = Object(methods, {x: 3, y: 5}) var i2 = Object(methods, {x: 10, y: 42}) -i1 -i2 +i1 (1) +i2 (1) ## STDOUT: - +sum = 9 +sum = 53 ## END diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 556ea1c359..53eaf841ba 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -483,17 +483,27 @@ def BindProcArgs(proc, cmd_val, mem, self_val=None): "Proc %r takes no word args, but got %d" % (proc.name, num_word - 1), blame_loc) - ### Handle typed positional args. This includes a block arg, if any. + ### Handle typed positional args. if proc_args and proc_args.typed_args: # blame ( of call site blame_loc = proc_args.typed_args.left - pos_args = proc_args.pos_args if proc_args else None + if proc_args: + pos_args = proc_args.pos_args + else: + pos_args = [] + + if self_val: # Prepend to beginning + pos_args.insert(0, self_val) + if sig.positional: - # TODO: Add self_val _BindTyped(proc.name, sig.positional, proc.defaults.for_typed, pos_args, mem, blame_loc) else: + if self_val is not None: + raise error.Expr( + "Using proc %r as __invoke__ requires a 'self' param" % + proc.name, blame_loc) if pos_args is not None: num_pos = len(pos_args) if num_pos != 0: From 2ab7ee22531a0a353531f3d05ff33cc0a4c0f3ff Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 28 Sep 2024 21:04:26 -0400 Subject: [PATCH 266/506] [translation] Fix build We don't have the List::insert(0, item) method in mycpp --- ysh/func_proc.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 53eaf841ba..3357d00e67 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -488,13 +488,15 @@ def BindProcArgs(proc, cmd_val, mem, self_val=None): if proc_args and proc_args.typed_args: # blame ( of call site blame_loc = proc_args.typed_args.left - if proc_args: - pos_args = proc_args.pos_args + if self_val: + pos_args = [self_val] + if proc_args: + pos_args.extend(proc_args.pos_args) else: - pos_args = [] - - if self_val: # Prepend to beginning - pos_args.insert(0, self_val) + if proc_args: # save an allocation in this common case + pos_args = proc_args.pos_args + else: + pos_args = [] if sig.positional: _BindTyped(proc.name, sig.positional, proc.defaults.for_typed, From a7b6dde67503518077513089e9fc7afed486c555 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 09:15:23 -0400 Subject: [PATCH 267/506] [spec/ysh-proc] Example: stateful proc with counter We can do the same thing with __call__. [doc] fopen -> redir --- doc/idioms.md | 4 ++-- spec/ysh-proc.test.sh | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/idioms.md b/doc/idioms.md index 57c235648f..848ac459c0 100644 --- a/doc/idioms.md +++ b/doc/idioms.md @@ -350,12 +350,12 @@ No: Yes: - fopen > out.txt { + redir > out.txt { echo 1 echo 2 } -The `fopen` builtin is syntactic sugar -- it lets you see redirects before the +The `redir` builtin is syntactic sugar -- it lets you see redirects before the code that uses them. ### Temporarily Set Shell Options diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 5de6b9b6c7..a3d95321ed 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -759,3 +759,24 @@ i2 (1) sum = 9 sum = 53 ## END + + +#### Stateful proc with counter +shopt --set ysh:upgrade +proc invokeCounter(; self, inc) { + setvar self.i += inc + echo "counter = $[self.i]" +} + +var methods = Object(null, {__invoke__: invokeCounter}) +var counter = Object(methods, {i: 0}) + +counter (1) +counter (2) +counter (3) + +## STDOUT: +counter = 1 +counter = 3 +counter = 6 +## END From 0e09d50918db248cb421662185ef83deb6a6b6c3 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 10:16:23 -0400 Subject: [PATCH 268/506] [builtin/use] Spec tests, and planning Python-like modules --- builtin/module_ysh.py | 112 ++++++++++++++++++-------------- frontend/flag_def.py | 6 +- spec/hay.test.sh | 44 ------------- spec/testdata/config/ci.oil | 2 +- spec/ysh-builtin-module.test.sh | 46 ++++++++++++- 5 files changed, 113 insertions(+), 97 deletions(-) diff --git a/builtin/module_ysh.py b/builtin/module_ysh.py index bf5a82fc11..1c25fe9606 100644 --- a/builtin/module_ysh.py +++ b/builtin/module_ysh.py @@ -1,18 +1,12 @@ from __future__ import print_function -from _devbuild.gen.runtime_asdl import scope_e -from _devbuild.gen.syntax_asdl import loc -from _devbuild.gen.value_asdl import (value, value_e) - -from core import error from core import state from display import ui from core import vm -from frontend import args from frontend import flag_util from mycpp.mylib import log -from typing import Dict, cast, TYPE_CHECKING +from typing import Dict, TYPE_CHECKING if TYPE_CHECKING: from _devbuild.gen.runtime_asdl import cmd_value from core import optview @@ -63,13 +57,55 @@ def Run(self, cmd_val): class Use(vm._Builtin): - """use bin, use dialect to control the 'first word'. + """ + Module system with all the power of Python, but still a proc + + use util.ysh # util is a value.Obj + + # Importing a bunch of words + use dialect-ninja.ysh { all } # requires 'provide' in dialect-ninja + use dialect-github.ysh { all } + + # This declares some names + use --extern grep sed + + # Renaming + use util.ysh (&myutil) + + # Ignore + use util.ysh (&_) + + # Picking specifics + use util.ysh { + pick log die + pick foo (&myfoo) + } + + # A long way to write this is: + + use util.ysh + const log = util.log + const die = util.die + const myfoo = util.foo - Examples: - use bin grep sed + Another way is: + for name in log die { + call setVar(name, util[name]) - use dialect ninja # I think it must be in a 'dialect' scope - use dialect travis + # value.Obj may not support [] though + # get(propView(util), name, null) is a long way of writing it + } + + Other considerations: + + - Statically parseable subset? For fine-grained static tree-shaking + - We're doing coarse dynamic tree-shaking first though + + - if TYPE_CHECKING is an issue + - that can create circular dependencies, especially with gradual typing, + when you go dynamic to static (like Oils did) + - I guess you can have + - use --static parse_lib.ysh { pick ParseContext } """ def __init__(self, mem, errfmt): @@ -79,42 +115,20 @@ def __init__(self, mem, errfmt): def Run(self, cmd_val): # type: (cmd_value.Argv) -> int - arg_r = args.Reader(cmd_val.argv, locs=cmd_val.arg_locs) - arg_r.Next() # skip 'use' - - arg, arg_loc = arg_r.Peek2() - if arg is None: - raise error.Usage("expected 'bin' or 'dialect'", loc.Missing) - arg_r.Next() - - if arg == 'dialect': - expected, e_loc = arg_r.Peek2() - if expected is None: - raise error.Usage('expected dialect name', loc.Missing) - - UP_actual = self.mem.GetValue('_DIALECT', scope_e.Dynamic) - if UP_actual.tag() == value_e.Str: - actual = cast(value.Str, UP_actual).s - if actual == expected: - return 0 # OK - else: - self.errfmt.Print_('Expected dialect %r, got %r' % - (expected, actual), - blame_loc=e_loc) - - return 1 - else: - # Not printing expected value - self.errfmt.Print_('Expected dialect %r' % expected, - blame_loc=e_loc) - return 1 + _, arg_r = flag_util.ParseCmdVal('use', cmd_val) - # 'use bin' can be used for static analysis. Although could it also - # simplify the SearchPath logic? Maybe ensure that it is memoized? - if arg == 'bin': - rest = arg_r.Rest() - for name in rest: - log('bin %s', name) - return 0 + mod_path, _ = arg_r.ReadRequired2('requires a module path') - raise error.Usage("expected 'bin' or 'dialect'", arg_loc) + log('m %s', mod_path) + + arg_r.Done() + + # TODO on usage: + # - typed arg is value.Place + # - block arg binds 'pick' and 'all' + + # TODO: + # with ctx_Module + # and then do something very similar to 'source' + + return 0 diff --git a/frontend/flag_def.py b/frontend/flag_def.py index b031e3c798..aefc0f5696 100644 --- a/frontend/flag_def.py +++ b/frontend/flag_def.py @@ -462,7 +462,9 @@ def _DefineCompletionActions(spec): FORKWAIT_SPEC = FlagSpec('forkwait') # Might want --list at some point -MODULE_SPEC = FlagSpec('source-guard') +SOURCE_GUARD_SPEC = FlagSpec('source-guard') +USE_SPEC = FlagSpec('use') +USE_SPEC.LongFlag('--extern') RUNPROC_SPEC = FlagSpec('runproc') RUNPROC_SPEC.ShortFlag('-h', args.Bool, help='Show all procs') @@ -470,7 +472,7 @@ def _DefineCompletionActions(spec): INVOKE_SPEC = FlagSpec('invoke') INVOKE_SPEC.LongFlag('--builtin') # like 'builtin' INVOKE_SPEC.LongFlag('--proc-like') # like 'runproc' -INVOKE_SPEC.LongFlag('--extern') # like 'extern' +INVOKE_SPEC.LongFlag('--extern') # like 'extern' EXTERN_SPEC = FlagSpec('extern') diff --git a/spec/hay.test.sh b/spec/hay.test.sh index bb5a15a82c..51e0d624fa 100644 --- a/spec/hay.test.sh +++ b/spec/hay.test.sh @@ -2,50 +2,6 @@ ## oils_failures_allowed: 2 -#### use bin -use -echo status=$? -use z -echo status=$? - -use bin -echo bin status=$? -use bin sed grep -echo bin status=$? - -## STDOUT: -status=2 -status=2 -bin status=0 -bin status=0 -## END - -#### use dialect -shopt --set parse_brace - -use dialect -echo status=$? - -use dialect ninja -echo status=$? - -shvar _DIALECT=oops { - use dialect ninja - echo status=$? -} - -shvar _DIALECT=ninja { - use dialect ninja - echo status=$? -} - -## STDOUT: -status=2 -status=1 -status=1 -status=0 -## END - #### hay builtin usage hay define diff --git a/spec/testdata/config/ci.oil b/spec/testdata/config/ci.oil index 4a2b8a03a0..240332223b 100644 --- a/spec/testdata/config/ci.oil +++ b/spec/testdata/config/ci.oil @@ -1,6 +1,6 @@ # Similar to .builds/cpp.yaml for sourcehut -use dialect sourcehut +#use dialect sourcehut const image = 'debian/buster' diff --git a/spec/ysh-builtin-module.test.sh b/spec/ysh-builtin-module.test.sh index a47786949d..c6089fb445 100644 --- a/spec/ysh-builtin-module.test.sh +++ b/spec/ysh-builtin-module.test.sh @@ -1,5 +1,6 @@ +## oils_failures_allowed: 1 -#### source-guard +#### source-guard is an old way of preventing redefinition - could remove it shopt --set ysh:upgrade source-guard 'main' || return 0 @@ -36,3 +37,46 @@ status=0 stdin status=0 ## END + +#### use foo.ysh creates a value.Obj + +use $REPO_ROOT/spec/testdata/module2/util.ysh + +var methods = Object(null, {}) +var obj = Object(methods, {x: 1}) +pp test_ (obj) +pp test_ (methods) + + +# This is a value.Obj +pp test_ (util) + +util log 'hello' + +## STDOUT: +## END + +#### use builtin usage + +use +echo no-arg=$? + +use foo +echo one-arg=$? + +use --extern foo +echo extern=$? + +use --bad-flag +echo bad-flag=$? + +use too many +echo too-many=$? + +## STDOUT: +no-arg=2 +one-arg=0 +extern=0 +bad-flag=2 +too-many=2 +## END From 5f76709c298da2d05c967923907aef10060a28c4 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 10:41:41 -0400 Subject: [PATCH 269/506] [rename] builtin/meta_osh -> meta_oils Because - 'invoke' will generalize sh 'builtin command type' - 'use' will generalize sh 'source' Fix test/runtime-errors. --- builtin/{meta_osh.py => meta_oils.py} | 10 ++- builtin/meta_ysh.py | 101 -------------------------- core/shell.py | 24 +++--- core/state.py | 2 +- frontend/flag_def.py | 2 - metrics/source-code.sh | 8 +- osh/word_eval.py | 2 +- test/runtime-errors.sh | 4 +- 8 files changed, 32 insertions(+), 121 deletions(-) rename builtin/{meta_osh.py => meta_oils.py} (99%) delete mode 100644 builtin/meta_ysh.py diff --git a/builtin/meta_osh.py b/builtin/meta_oils.py similarity index 99% rename from builtin/meta_osh.py rename to builtin/meta_oils.py index 2f5eb3900f..944e0d2254 100644 --- a/builtin/meta_osh.py +++ b/builtin/meta_oils.py @@ -1,6 +1,14 @@ #!/usr/bin/env python2 """ -meta_osh.py - Builtins that call back into the interpreter. +meta_oils.py - Builtins that call back into the interpreter, or reflect on it. + +OSH builtins: + builtin command type + source eval + +YSH builtins: + invoke extern + use """ from __future__ import print_function diff --git a/builtin/meta_ysh.py b/builtin/meta_ysh.py deleted file mode 100644 index 8e603a62ee..0000000000 --- a/builtin/meta_ysh.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python2 -""" -meta_ysh.py - Builtins for introspection -""" -from __future__ import print_function - -from _devbuild.gen.runtime_asdl import cmd_value -from core import error -from core.error import e_usage -from core import vm -from frontend import flag_spec -from frontend import match -from frontend import typed_args -from mycpp import mylib -from mycpp.mylib import log - -_ = log - -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from core import state - from display import ui - - -class Shvm(vm._Builtin): - """ - shvm cell x - move pp cell x here - shvm gc-stats - like OILS_GC_STATS - shvm guts (x+y) - ASDL pretty printing - - similar to = x+y, but not stable - - Related: - _vm->heapId(obj) - a heap ID that can be used to detect cycles for - serialization - """ - - def __init__( - self, - mem, # type: state.Mem - errfmt, # type: ui.ErrorFormatter - ): - # type: (...) -> None - self.mem = mem - self.errfmt = errfmt - self.stdout_ = mylib.Stdout() - - def Run(self, cmd_val): - # type: (cmd_value.Argv) -> int - - arg, arg_r = flag_spec.ParseCmdVal('shvm', cmd_val) - - action, action_loc = arg_r.ReadRequired2( - 'expected an action (cell, gc-stats, guts)') - - if action == 'cell': - argv, locs = arg_r.Rest2() - - status = 0 - for i, name in enumerate(argv): - if name.startswith(':'): - name = name[1:] - - if not match.IsValidVarName(name): - raise error.Usage('got invalid variable name %r' % name, - locs[i]) - - cell = self.mem.GetCell(name) - if cell is None: - self.errfmt.Print_("Couldn't find a variable named %r" % - name, - blame_loc=locs[i]) - status = 1 - else: - self.stdout_.write('%s = ' % name) - if mylib.PYTHON: - cell.PrettyPrint() # may be color - - self.stdout_.write('\n') - - elif action == 'gc-stats': - # mylib.PrintGcStats() - print('TODO') - status = 0 - - elif action == 'guts': - # Print the value - print('TODO') - - if cmd_val.typed_args: # eval (myblock) - rd = typed_args.ReaderForProc(cmd_val) - val = rd.PosValue() - rd.Done() - if mylib.PYTHON: - print(val) - - status = 0 - - else: - e_usage('got invalid action %r' % action, action_loc) - - return status diff --git a/core/shell.py b/core/shell.py index 0aa9d81660..f4907321df 100644 --- a/core/shell.py +++ b/core/shell.py @@ -45,7 +45,7 @@ from builtin import io_osh from builtin import io_ysh from builtin import json_ysh -from builtin import meta_osh +from builtin import meta_oils from builtin import misc_osh from builtin import module_ysh from builtin import printf_osh @@ -622,22 +622,22 @@ def Main( b[builtin_i.haynode] = hay_ysh.HayNode_(hay_state, mem, cmd_ev) # Interpreter introspection - b[builtin_i.type] = meta_osh.Type(procs, aliases, search_path, errfmt) - b[builtin_i.builtin] = meta_osh.Builtin(shell_ex, errfmt) - b[builtin_i.command] = meta_osh.Command(shell_ex, procs, aliases, - search_path) + b[builtin_i.type] = meta_oils.Type(procs, aliases, search_path, errfmt) + b[builtin_i.builtin] = meta_oils.Builtin(shell_ex, errfmt) + b[builtin_i.command] = meta_oils.Command(shell_ex, procs, aliases, + search_path) # Part of YSH, but similar to builtin/command - b[builtin_i.runproc] = meta_osh.RunProc(shell_ex, procs, errfmt) - b[builtin_i.invoke] = meta_osh.Invoke(shell_ex, procs, errfmt) - b[builtin_i.extern_] = meta_osh.Extern(shell_ex, procs, errfmt) + b[builtin_i.runproc] = meta_oils.RunProc(shell_ex, procs, errfmt) + b[builtin_i.invoke] = meta_oils.Invoke(shell_ex, procs, errfmt) + b[builtin_i.extern_] = meta_oils.Extern(shell_ex, procs, errfmt) # Meta builtins - source_builtin = meta_osh.Source(parse_ctx, search_path, cmd_ev, fd_state, - tracer, errfmt, loader) + source_builtin = meta_oils.Source(parse_ctx, search_path, cmd_ev, fd_state, + tracer, errfmt, loader) b[builtin_i.source] = source_builtin b[builtin_i.dot] = source_builtin - b[builtin_i.eval] = meta_osh.Eval(parse_ctx, exec_opts, cmd_ev, tracer, - errfmt, mem) + b[builtin_i.eval] = meta_oils.Eval(parse_ctx, exec_opts, cmd_ev, tracer, + errfmt, mem) # Module builtins guards = {} # type: Dict[str, bool] diff --git a/core/state.py b/core/state.py index b71d64129f..538d07cd0c 100644 --- a/core/state.py +++ b/core/state.py @@ -2628,7 +2628,7 @@ def GetInvokable(self, name): Callers: executor.py: running - meta_osh.py runproc lookup - this is not 'invoke', because it is + meta_oils.py runproc lookup - this is not 'invoke', because it is INTERIOR shell functions, procs, invokable Obj cmd_eval: check for redefining proc or sh-func (remove) """ diff --git a/frontend/flag_def.py b/frontend/flag_def.py index aefc0f5696..cdbf7d375a 100644 --- a/frontend/flag_def.py +++ b/frontend/flag_def.py @@ -455,8 +455,6 @@ def _DefineCompletionActions(spec): PP_SPEC = FlagSpec('pp') -SHVM_SPEC = FlagSpec('shvm') - # --verbose? FORK_SPEC = FlagSpec('fork') FORKWAIT_SPEC = FlagSpec('forkwait') diff --git a/metrics/source-code.sh b/metrics/source-code.sh index 69d6c5f0f7..5e1800f561 100755 --- a/metrics/source-code.sh +++ b/metrics/source-code.sh @@ -182,7 +182,13 @@ osh-counts() { } ysh-files() { - ls ysh/*.{py,pgen2} builtin/{func,method}*.py builtin/*_ysh.py | filter-py + # Count meta_oils.py as YSH, not OSH, even though it contains the shell + # 'builtin command type' builtins. We will generalize that a bit + ls ysh/*.{py,pgen2} \ + builtin/{func,method}*.py \ + builtin/*_ysh.py \ + builtin/*_oils.py \ + | filter-py } ysh-counts() { diff --git a/osh/word_eval.py b/osh/word_eval.py index 47f085315c..d1878ce9c3 100644 --- a/osh/word_eval.py +++ b/osh/word_eval.py @@ -151,7 +151,7 @@ def _DetectMetaBuiltinStr(s): Fundamentally, assignment builtins have different WORD EVALUATION RULES for a=$x (no word splitting), so it seems hard to do this in - meta_osh.Builtin() or meta_osh.Command() + meta_oils.Builtin() or meta_oils.Command() """ return (consts.LookupNormalBuiltin(s) in (builtin_i.builtin, builtin_i.command)) diff --git a/test/runtime-errors.sh b/test/runtime-errors.sh index d8c01d1434..859edd5c52 100755 --- a/test/runtime-errors.sh +++ b/test/runtime-errors.sh @@ -1085,7 +1085,7 @@ test-control_flow_subshell() { ' } -test-fallback_locations() { +test-fallback-locations() { # Redirect _osh-error-1 'echo hi > /' @@ -1105,7 +1105,7 @@ test-fallback_locations() { _osh-error-1 '[[ $x =~ $(( 3 ** -2 )) ]]' _osh-error-2 'type -x' # correctly points to -x - _osh-error-2 'use x' + _osh-error-2 'use' # Assign builtin _osh-error-2 'export -f' From 20feae8277aad6e165eb040c0a50a45222f475de Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 11:10:11 -0400 Subject: [PATCH 270/506] [pea] Fix build Make note about conflict around PYTHONPATH. --- build/dev-shell.sh | 2 ++ pea/TEST.sh | 8 ++++++++ pea/oils-typecheck.txt | 3 ++- pea/pea_main.py | 4 ++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/build/dev-shell.sh b/build/dev-shell.sh index 16a1d919bd..3dcd6a4ad0 100644 --- a/build/dev-shell.sh +++ b/build/dev-shell.sh @@ -136,8 +136,10 @@ MYPY_VERSION=0.780 # Containers copy it here readonly MYPY_WEDGE=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION +#echo "MYPY_WEDGE $MYPY_WEDGE" if test -d "$MYPY_WEDGE"; then export PYTHONPATH="$MYPY_WEDGE:$PYTHONPATH" + #echo "PYTHONPATH $PYTHONPATH" fi # Hack for misconfigured RC cluster! Some machines have the empty string in diff --git a/pea/TEST.sh b/pea/TEST.sh index 5c2f80510f..e0c01b28b2 100755 --- a/pea/TEST.sh +++ b/pea/TEST.sh @@ -16,6 +16,13 @@ source build/dev-shell.sh # find python3 in /wedge PATH component # This is just like the yapf problem in devtools/format.sh ! # Pea needs a newer version of MyPy -- one that supports 'math' + +# 2024-09 - there is a conflict between: +# parse-all - 'import mypy' for mycpp/pass_state.py +# check-types - uses a newer version of MyPy +# +# The problem is importing MyPy as a LIBRARY vs. using it as a TOOL + unset PYTHONPATH export PYTHONPATH=. @@ -73,6 +80,7 @@ all-files() { } parse-all() { + #source $MYPY_VENV/bin/activate time all-files | xargs --verbose -- $0 pea-main parse } diff --git a/pea/oils-typecheck.txt b/pea/oils-typecheck.txt index f5f7ec9f8c..fd5e620b92 100644 --- a/pea/oils-typecheck.txt +++ b/pea/oils-typecheck.txt @@ -22,11 +22,12 @@ builtin/error_ysh.py builtin/func_eggex.py builtin/func_hay.py builtin/func_misc.py +builtin/func_reflect.py builtin/hay_ysh.py builtin/io_osh.py builtin/io_ysh.py builtin/json_ysh.py -builtin/meta_osh.py +builtin/meta_oils.py builtin/method_dict.py builtin/method_io.py builtin/method_list.py diff --git a/pea/pea_main.py b/pea/pea_main.py index 906e1329cf..225f857ff2 100755 --- a/pea/pea_main.py +++ b/pea/pea_main.py @@ -16,6 +16,10 @@ import sys import time +if 0: + for p in sys.path: + print('*** syspath: %s' % p) + import typing from typing import Optional, Any From 8613ec3f49cc9951d0992a412ffb30fb1c94a586 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 12:28:17 -0400 Subject: [PATCH 271/506] [ysh refactor] Move 'use' builtin under 'source' They will share much of the same logic, like tracing, and ///osh and ///ysh as embedded data. --- builtin/meta_oils.py | 90 ++++++++++++++++++++++++++++++++++++++++++- builtin/module_ysh.py | 78 ------------------------------------- core/shell.py | 13 +++++-- 3 files changed, 99 insertions(+), 82 deletions(-) diff --git a/builtin/meta_oils.py b/builtin/meta_oils.py index 944e0d2254..fe93399d2c 100644 --- a/builtin/meta_oils.py +++ b/builtin/meta_oils.py @@ -95,7 +95,12 @@ def Run(self, cmd_val): cmd_flags=cmd_eval.RaiseControlFlow) -class Source(vm._Builtin): +class ShellFile(vm._Builtin): + """ + These share code: + - 'source' builtin for OSH + - 'use' builtin for YSH + """ def __init__( self, @@ -106,6 +111,7 @@ def __init__( tracer, # type: dev.Tracer errfmt, # type: ui.ErrorFormatter loader, # type: pyutil._ResourceLoader + ysh_use=False, # type: bool ): # type: (...) -> None self.parse_ctx = parse_ctx @@ -116,10 +122,92 @@ def __init__( self.tracer = tracer self.errfmt = errfmt self.loader = loader + self.ysh_use = ysh_use self.mem = cmd_ev.mem def Run(self, cmd_val): + # type: (cmd_value.Argv) -> int + """ + Use is like Source + """ + if self.ysh_use: + return self._Use(cmd_val) + else: + return self._Source(cmd_val) + + def _Use(self, cmd_val): + # type: (cmd_value.Argv) -> int + """ + Module system with all the power of Python, but still a proc + + use util.ysh # util is a value.Obj + + # Importing a bunch of words + use dialect-ninja.ysh { all } # requires 'provide' in dialect-ninja + use dialect-github.ysh { all } + + # This declares some names + use --extern grep sed + + # Renaming + use util.ysh (&myutil) + + # Ignore + use util.ysh (&_) + + # Picking specifics + use util.ysh { + pick log die + pick foo (&myfoo) + } + + # A long way to write this is: + + use util.ysh + const log = util.log + const die = util.die + const myfoo = util.foo + + Another way is: + for name in log die { + call setVar(name, util[name]) + + # value.Obj may not support [] though + # get(propView(util), name, null) is a long way of writing it + } + + Other considerations: + + - Statically parseable subset? For fine-grained static tree-shaking + - We're doing coarse dynamic tree-shaking first though + + - if TYPE_CHECKING is an issue + - that can create circular dependencies, especially with gradual typing, + when you go dynamic to static (like Oils did) + - I guess you can have + - use --static parse_lib.ysh { pick ParseContext } + """ + _, arg_r = flag_util.ParseCmdVal('use', cmd_val) + + mod_path, _ = arg_r.ReadRequired2('requires a module path') + + log('m %s', mod_path) + + arg_r.Done() + + # TODO on usage: + # - typed arg is value.Place + # - block arg binds 'pick' and 'all' + + # TODO: + # with ctx_Module + # and then do something very similar to 'source' + + return 0 + return 0 + + def _Source(self, cmd_val): # type: (cmd_value.Argv) -> int attrs, arg_r = flag_util.ParseCmdVal('source', cmd_val) arg = arg_types.source(attrs.attrs) diff --git a/builtin/module_ysh.py b/builtin/module_ysh.py index 1c25fe9606..c5d601fa27 100644 --- a/builtin/module_ysh.py +++ b/builtin/module_ysh.py @@ -54,81 +54,3 @@ def Run(self, cmd_val): return 1 self.guards[name] = True return 0 - - -class Use(vm._Builtin): - """ - Module system with all the power of Python, but still a proc - - use util.ysh # util is a value.Obj - - # Importing a bunch of words - use dialect-ninja.ysh { all } # requires 'provide' in dialect-ninja - use dialect-github.ysh { all } - - # This declares some names - use --extern grep sed - - # Renaming - use util.ysh (&myutil) - - # Ignore - use util.ysh (&_) - - # Picking specifics - use util.ysh { - pick log die - pick foo (&myfoo) - } - - # A long way to write this is: - - use util.ysh - const log = util.log - const die = util.die - const myfoo = util.foo - - Another way is: - for name in log die { - call setVar(name, util[name]) - - # value.Obj may not support [] though - # get(propView(util), name, null) is a long way of writing it - } - - Other considerations: - - - Statically parseable subset? For fine-grained static tree-shaking - - We're doing coarse dynamic tree-shaking first though - - - if TYPE_CHECKING is an issue - - that can create circular dependencies, especially with gradual typing, - when you go dynamic to static (like Oils did) - - I guess you can have - - use --static parse_lib.ysh { pick ParseContext } - """ - - def __init__(self, mem, errfmt): - # type: (state.Mem, ui.ErrorFormatter) -> None - self.mem = mem - self.errfmt = errfmt - - def Run(self, cmd_val): - # type: (cmd_value.Argv) -> int - _, arg_r = flag_util.ParseCmdVal('use', cmd_val) - - mod_path, _ = arg_r.ReadRequired2('requires a module path') - - log('m %s', mod_path) - - arg_r.Done() - - # TODO on usage: - # - typed arg is value.Place - # - block arg binds 'pick' and 'all' - - # TODO: - # with ctx_Module - # and then do something very similar to 'source' - - return 0 diff --git a/core/shell.py b/core/shell.py index f4907321df..abd4577fde 100644 --- a/core/shell.py +++ b/core/shell.py @@ -632,8 +632,16 @@ def Main( b[builtin_i.extern_] = meta_oils.Extern(shell_ex, procs, errfmt) # Meta builtins - source_builtin = meta_oils.Source(parse_ctx, search_path, cmd_ev, fd_state, - tracer, errfmt, loader) + b[builtin_i.use] = meta_oils.ShellFile(parse_ctx, + search_path, + cmd_ev, + fd_state, + tracer, + errfmt, + loader, + ysh_use=True) + source_builtin = meta_oils.ShellFile(parse_ctx, search_path, cmd_ev, + fd_state, tracer, errfmt, loader) b[builtin_i.source] = source_builtin b[builtin_i.dot] = source_builtin b[builtin_i.eval] = meta_oils.Eval(parse_ctx, exec_opts, cmd_ev, tracer, @@ -644,7 +652,6 @@ def Main( b[builtin_i.source_guard] = module_ysh.SourceGuard(guards, exec_opts, errfmt) b[builtin_i.is_main] = module_ysh.IsMain(mem) - b[builtin_i.use] = module_ysh.Use(mem, errfmt) # Errors b[builtin_i.error] = error_ysh.Error() From 5bfca4065f0cc4e034b2d1dbd2655363eeffb553 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 15:41:59 -0400 Subject: [PATCH 272/506] [builtin/source] Fix error locations, and extract methods Preparing for 'use' to behave in a similar way. --- builtin/meta_oils.py | 116 ++++++++++++++++++++------------ spec/ysh-builtin-module.test.sh | 10 ++- spec/ysh-builtins.test.sh | 4 +- spec/ysh-source.test.sh | 4 +- 4 files changed, 85 insertions(+), 49 deletions(-) diff --git a/builtin/meta_oils.py b/builtin/meta_oils.py index fe93399d2c..9793b3e63c 100644 --- a/builtin/meta_oils.py +++ b/builtin/meta_oils.py @@ -14,7 +14,7 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus -from _devbuild.gen.syntax_asdl import source, loc +from _devbuild.gen.syntax_asdl import source, loc, loc_t from core import alloc from core import dev from core import error @@ -44,8 +44,9 @@ from frontend.parse_lib import ParseContext from core import optview from display import ui + from mycpp import mylib from osh.cmd_eval import CommandEvaluator - from osh.cmd_parse import CommandParser + from osh import cmd_parse class Eval(vm._Builtin): @@ -124,18 +125,47 @@ def __init__( self.loader = loader self.ysh_use = ysh_use + self.builtin_name = 'use' if ysh_use else 'source' self.mem = cmd_ev.mem def Run(self, cmd_val): # type: (cmd_value.Argv) -> int - """ - Use is like Source - """ if self.ysh_use: return self._Use(cmd_val) else: return self._Source(cmd_val) + def _LoadBuiltinFile(self, builtin_path, blame_loc): + # type: (str, loc_t) -> Tuple[str, cmd_parse.CommandParser] + try: + load_path = os_path.join("stdlib", builtin_path) + contents = self.loader.Get(load_path) + except (IOError, OSError): + self.errfmt.Print_('%r failed: No builtin file %r' % + (self.builtin_name, load_path), + blame_loc=blame_loc) + return None, None # error + + line_reader = reader.StringLineReader(contents, self.arena) + c_parser = self.parse_ctx.MakeOshParser(line_reader) + return load_path, c_parser + + def _LoadDiskFile(self, fs_path, blame_loc): + # type: (str, loc_t) -> Tuple[mylib.LineReader, cmd_parse.CommandParser] + try: + # Shell can't use descriptors 3-9 + f = self.fd_state.Open(fs_path) + except (IOError, OSError) as e: + self.errfmt.Print_( + '%s %r failed: %s' % + (self.builtin_name, fs_path, pyutil.strerror(e)), + blame_loc=blame_loc) + return None, None + + line_reader = reader.FileLineReader(f, self.arena) + c_parser = self.parse_ctx.MakeOshParser(line_reader) + return f, c_parser + def _Use(self, cmd_val): # type: (cmd_value.Argv) -> int """ @@ -189,33 +219,46 @@ def _Use(self, cmd_val): - use --static parse_lib.ysh { pick ParseContext } """ _, arg_r = flag_util.ParseCmdVal('use', cmd_val) + path_arg, path_loc = arg_r.ReadRequired2('requires a module path') + # TODO on usage: + # - typed arg is value.Place + # - block arg binds 'pick' and 'all' + # Although ALL these 3 mechanisms can be done with 'const' assignments. + # Hm. + arg_r.Done() - mod_path, _ = arg_r.ReadRequired2('requires a module path') + # I wonder if modules should be FROZEN value.Obj, not mutable? - log('m %s', mod_path) + # Duplicating logic below + if path_arg.startswith('///'): + builtin_path = path_arg[3:] + else: + builtin_path = None - arg_r.Done() + if builtin_path is not None: + load_path, c_parser = self._LoadBuiltinFile(builtin_path, path_loc) + if c_parser is None: + return 1 # error was already shown - # TODO on usage: - # - typed arg is value.Place - # - block arg binds 'pick' and 'all' + # TODO: ctx_Module + return self._Exec(cmd_val, arg_r, load_path, c_parser) + else: + f, c_parser = self._LoadDiskFile(path_arg, path_loc) + if c_parser is None: + return 1 # error was already shown - # TODO: - # with ctx_Module - # and then do something very similar to 'source' + # TODO: ctx_Module + with process.ctx_FileCloser(f): + return self._Exec(cmd_val, arg_r, path_arg, c_parser) - return 0 - return 0 + raise AssertionError() def _Source(self, cmd_val): # type: (cmd_value.Argv) -> int attrs, arg_r = flag_util.ParseCmdVal('source', cmd_val) arg = arg_types.source(attrs.attrs) - path_arg = arg_r.Peek() - if path_arg is None: - e_usage('missing required argument', loc.Missing) - arg_r.Next() + path_arg, path_loc = arg_r.ReadRequired2('requires a file path') # Old: # source --builtin two.sh # looks up stdlib/two.sh @@ -229,17 +272,10 @@ def _Source(self, cmd_val): builtin_path = path_arg[3:] if builtin_path is not None: - try: - load_path = os_path.join("stdlib", builtin_path) - contents = self.loader.Get(load_path) - except (IOError, OSError): - self.errfmt.Print_('source failed: No builtin file %r' % - load_path, - blame_loc=cmd_val.arg_locs[2]) - return 2 - - line_reader = reader.StringLineReader(contents, self.arena) - c_parser = self.parse_ctx.MakeOshParser(line_reader) + load_path, c_parser = self._LoadBuiltinFile(builtin_path, path_loc) + if c_parser is None: + return 1 # error was already shown + return self._Exec(cmd_val, arg_r, load_path, c_parser) else: @@ -249,23 +285,17 @@ def _Source(self, cmd_val): if resolved is None: resolved = path_arg - try: - # Shell can't use descriptors 3-9 - f = self.fd_state.Open(resolved) - except (IOError, OSError) as e: - self.errfmt.Print_('source %r failed: %s' % - (path_arg, pyutil.strerror(e)), - blame_loc=cmd_val.arg_locs[1]) - return 1 - - line_reader = reader.FileLineReader(f, self.arena) - c_parser = self.parse_ctx.MakeOshParser(line_reader) + f, c_parser = self._LoadDiskFile(resolved, path_loc) + if c_parser is None: + return 1 # error was already shown with process.ctx_FileCloser(f): return self._Exec(cmd_val, arg_r, path_arg, c_parser) + raise AssertionError() + def _Exec(self, cmd_val, arg_r, path, c_parser): - # type: (cmd_value.Argv, args.Reader, str, CommandParser) -> int + # type: (cmd_value.Argv, args.Reader, str, cmd_parse.CommandParser) -> int call_loc = cmd_val.arg_locs[0] # A sourced module CAN have a new arguments array, but it always shares diff --git a/spec/ysh-builtin-module.test.sh b/spec/ysh-builtin-module.test.sh index c6089fb445..8bbf4b521c 100644 --- a/spec/ysh-builtin-module.test.sh +++ b/spec/ysh-builtin-module.test.sh @@ -73,10 +73,16 @@ echo bad-flag=$? use too many echo too-many=$? +use ///no-builtin +echo no-builtin=$? + + ## STDOUT: no-arg=2 -one-arg=0 -extern=0 +one-arg=1 +extern=1 bad-flag=2 too-many=2 +no-builtin=1 ## END + diff --git a/spec/ysh-builtins.test.sh b/spec/ysh-builtins.test.sh index 0fdbabe9fb..7511cc5b2d 100644 --- a/spec/ysh-builtins.test.sh +++ b/spec/ysh-builtins.test.sh @@ -585,6 +585,6 @@ echo status=$? ## STDOUT: status=0 status=0 -status=2 -status=2 +status=1 +status=1 ## END diff --git a/spec/ysh-source.test.sh b/spec/ysh-source.test.sh index bee0570649..dd73c4d062 100644 --- a/spec/ysh-source.test.sh +++ b/spec/ysh-source.test.sh @@ -20,10 +20,10 @@ source --builtin ## STDOUT: ## END -#### non-existant path passed to --builtin flag +#### non-existent path passed to --builtin flag shopt --set ysh:upgrade source --builtin test/this-file-will-never-exist.ysh -## status: 2 +## status: 1 ## STDOUT: ## END From a362b879a563fb2178f2de98443e373b15734898 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 16:18:01 -0400 Subject: [PATCH 273/506] [reformat] Migrate some long signatures to "tall" format --- builtin/printf_osh.py | 10 ++++++++-- core/process.py | 13 ++++++++++--- core/state.py | 10 ++++++++-- frontend/typed_args.py | 17 ++++++++++------- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/builtin/printf_osh.py b/builtin/printf_osh.py index 2d7e15c18c..0b67c294d6 100644 --- a/builtin/printf_osh.py +++ b/builtin/printf_osh.py @@ -185,8 +185,14 @@ def __init__( # this object initialized in main() self.shell_start_time = time_.time() - def _Percent(self, pr, part, varargs, locs): - # type: (_PrintfState, printf_part.Percent, List[str], List[CompoundWord]) -> Optional[str] + def _Percent( + self, + pr, # type: _PrintfState + part, # type: printf_part.Percent + varargs, # type: List[str] + locs, # type: List[CompoundWord] + ): + # type: (...) -> Optional[str] num_args = len(varargs) diff --git a/core/process.py b/core/process.py index a62884fc40..eb36dd58fa 100644 --- a/core/process.py +++ b/core/process.py @@ -808,9 +808,16 @@ def Run(self): class SubProgramThunk(Thunk): """A subprogram that can be executed in another process.""" - def __init__(self, cmd_ev, node, trap_state, multi_trace, inherit_errexit, - inherit_errtrace): - # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool, bool) -> None + def __init__( + self, + cmd_ev, # type: CommandEvaluator + node, # type: command_t + trap_state, # type: trap_osh.TrapState + multi_trace, # type: dev.MultiTracer + inherit_errexit, # type: bool + inherit_errtrace, # type: bool + ): + # type: (...) -> None self.cmd_ev = cmd_ev self.node = node self.trap_state = trap_state diff --git a/core/state.py b/core/state.py index 538d07cd0c..0bc6d30ffc 100644 --- a/core/state.py +++ b/core/state.py @@ -1186,8 +1186,14 @@ def __exit__(self, type, value, traceback): class ctx_Eval(object): """Push temporary set of variables, $0, $1, $2, etc.""" - def __init__(self, mem, dollar0, pos_args, vars): - # type: (Mem, Optional[str], Optional[List[str]], Optional[Dict[str, value_t]]) -> None + def __init__( + self, + mem, # type: Mem + dollar0, # type: Optional[str] + pos_args, # type: Optional[List[str]] + vars, # type: Optional[Dict[str, value_t]] + ): + # type: (...) -> None self.mem = mem self.dollar0 = dollar0 self.pos_args = pos_args diff --git a/frontend/typed_args.py b/frontend/typed_args.py index e7553153c6..387350fcb4 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -113,13 +113,16 @@ class Reader(object): ReaderForProc() """ - def __init__(self, - pos_args, - named_args, - block_arg, - arg_list, - is_bound=False): - # type: (List[value_t], Dict[str, value_t], Optional[value_t], ArgList, bool) -> None + def __init__( + self, + pos_args, # type: List[value_t] + named_args, # type: Dict[str, value_t] + block_arg, # type: Optional[value_t] + arg_list, # type: ArgList + is_bound=False, # type: bool + ): + # type: (...) -> None + self.pos_args = pos_args self.pos_consumed = 0 # TODO: Add LHS of attribute expression to value.BoundFunc and pass From 0ba6f44e14f2a74b3d502f5b364ab42da68964e9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 21:24:40 -0400 Subject: [PATCH 274/506] [builtin/use] Improve sturcture, share code with 'source' Still need to make the value.Obj --- builtin/meta_oils.py | 223 +++++++++++++++++++++++++++---------------- 1 file changed, 143 insertions(+), 80 deletions(-) diff --git a/builtin/meta_oils.py b/builtin/meta_oils.py index 9793b3e63c..35bc4db49b 100644 --- a/builtin/meta_oils.py +++ b/builtin/meta_oils.py @@ -15,6 +15,7 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus from _devbuild.gen.syntax_asdl import source, loc, loc_t +from _devbuild.gen.value_asdl import Obj from core import alloc from core import dev from core import error @@ -36,6 +37,8 @@ import posix_ as posix from posix_ import X_OK # translated directly to C macro +import libc + _ = log from typing import Dict, List, Tuple, Optional, TYPE_CHECKING @@ -128,6 +131,13 @@ def __init__( self.builtin_name = 'use' if ysh_use else 'source' self.mem = cmd_ev.mem + # Don't load modules more than once + # keyed by libc.realpath(arg) + self._disk_cache = {} # type: Dict[str, Obj] + + # keyed by /// + self._embed_cache = {} # type: Dict[str, Obj] + def Run(self, cmd_val): # type: (cmd_value.Argv) -> int if self.ysh_use: @@ -135,10 +145,10 @@ def Run(self, cmd_val): else: return self._Source(cmd_val) - def _LoadBuiltinFile(self, builtin_path, blame_loc): + def LoadEmbeddedFile(self, embed_path, blame_loc): # type: (str, loc_t) -> Tuple[str, cmd_parse.CommandParser] try: - load_path = os_path.join("stdlib", builtin_path) + load_path = os_path.join("stdlib", embed_path) contents = self.loader.Get(load_path) except (IOError, OSError): self.errfmt.Print_('%r failed: No builtin file %r' % @@ -166,6 +176,102 @@ def _LoadDiskFile(self, fs_path, blame_loc): c_parser = self.parse_ctx.MakeOshParser(line_reader) return f, c_parser + def _SourceExec(self, cmd_val, arg_r, path, c_parser): + # type: (cmd_value.Argv, args.Reader, str, cmd_parse.CommandParser) -> int + call_loc = cmd_val.arg_locs[0] + + # A sourced module CAN have a new arguments array, but it always shares + # the same variable scope as the caller. The caller could be at either a + # global or a local scope. + + # TODO: I wonder if we compose the enter/exit methods more easily. + + with dev.ctx_Tracer(self.tracer, 'source', cmd_val.argv): + source_argv = arg_r.Rest() + with state.ctx_Source(self.mem, path, source_argv): + with state.ctx_ThisDir(self.mem, path): + src = source.SourcedFile(path, call_loc) + with alloc.ctx_SourceCode(self.arena, src): + try: + status = main_loop.Batch( + self.cmd_ev, + c_parser, + self.errfmt, + cmd_flags=cmd_eval.RaiseControlFlow) + except vm.IntControlFlow as e: + if e.IsReturn(): + status = e.StatusCode() + else: + raise + + return status + + def _UseExec(self, cmd_val, arg_r, path, c_parser): + # type: (cmd_value.Argv, args.Reader, str, cmd_parse.CommandParser) -> int + call_loc = cmd_val.arg_locs[0] + + with dev.ctx_Tracer(self.tracer, 'use', None): + with state.ctx_ThisDir(self.mem, path): + + # TODO: change the src to source.ShellFile + + src = source.SourcedFile(path, call_loc) + with alloc.ctx_SourceCode(self.arena, src): + try: + status = main_loop.Batch( + self.cmd_ev, + c_parser, + self.errfmt, + cmd_flags=cmd_eval.RaiseControlFlow) + except vm.IntControlFlow as e: + if e.IsReturn(): + status = e.StatusCode() + else: + raise + + return status + + def _Source(self, cmd_val): + # type: (cmd_value.Argv) -> int + attrs, arg_r = flag_util.ParseCmdVal('source', cmd_val) + arg = arg_types.source(attrs.attrs) + + path_arg, path_loc = arg_r.ReadRequired2('requires a file path') + + # Old: + # source --builtin two.sh # looks up stdlib/two.sh + # New: + # source $LIB_OSH/two.sh # looks up stdlib/osh/two.sh + # source ///osh/two.sh # looks up stdlib/osh/two.sh + embed_path = None # type: Optional[str] + if arg.builtin: + embed_path = path_arg + elif path_arg.startswith('///'): + embed_path = path_arg[3:] + + if embed_path is not None: + load_path, c_parser = self.LoadEmbeddedFile(embed_path, path_loc) + if c_parser is None: + return 1 # error was already shown + + return self._SourceExec(cmd_val, arg_r, load_path, c_parser) + + else: + # 'source' respects $PATH + resolved = self.search_path.LookupOne(path_arg, + exec_required=False) + if resolved is None: + resolved = path_arg + + f, c_parser = self._LoadDiskFile(resolved, path_loc) + if c_parser is None: + return 1 # error was already shown + + with process.ctx_FileCloser(f): + return self._SourceExec(cmd_val, arg_r, path_arg, c_parser) + + raise AssertionError() + def _Use(self, cmd_val): # type: (cmd_value.Argv) -> int """ @@ -217,6 +323,20 @@ def _Use(self, cmd_val): when you go dynamic to static (like Oils did) - I guess you can have - use --static parse_lib.ysh { pick ParseContext } + + # Crazy idea - pure ysh + + use $LIB_YSH/pick.ysh + pick $LIB_YSH/table.ysh { + names foo bar + name x (&alias) + + all + names * # perhaps, if you turn off globbing + } + + import $LIB_YSH/stdlib + """ _, arg_r = flag_util.ParseCmdVal('use', cmd_val) path_arg, path_loc = arg_r.ReadRequired2('requires a module path') @@ -229,101 +349,44 @@ def _Use(self, cmd_val): # I wonder if modules should be FROZEN value.Obj, not mutable? - # Duplicating logic below + # Similar logic as 'source' if path_arg.startswith('///'): - builtin_path = path_arg[3:] - else: - builtin_path = None - - if builtin_path is not None: - load_path, c_parser = self._LoadBuiltinFile(builtin_path, path_loc) - if c_parser is None: - return 1 # error was already shown - - # TODO: ctx_Module - return self._Exec(cmd_val, arg_r, load_path, c_parser) + embed_path = path_arg[3:] else: - f, c_parser = self._LoadDiskFile(path_arg, path_loc) - if c_parser is None: - return 1 # error was already shown - - # TODO: ctx_Module - with process.ctx_FileCloser(f): - return self._Exec(cmd_val, arg_r, path_arg, c_parser) + embed_path = None - raise AssertionError() - - def _Source(self, cmd_val): - # type: (cmd_value.Argv) -> int - attrs, arg_r = flag_util.ParseCmdVal('source', cmd_val) - arg = arg_types.source(attrs.attrs) - - path_arg, path_loc = arg_r.ReadRequired2('requires a file path') - - # Old: - # source --builtin two.sh # looks up stdlib/two.sh - # New: - # source $LIB_OSH/two.sh # looks up stdlib/osh/two.sh - # source ///osh/two.sh # looks up stdlib/osh/two.sh - builtin_path = None # type: Optional[str] - if arg.builtin: - builtin_path = path_arg - elif path_arg.startswith('///'): - builtin_path = path_arg[3:] + if embed_path is not None: + # TODO: consult _embed_cache = {} - if builtin_path is not None: - load_path, c_parser = self._LoadBuiltinFile(builtin_path, path_loc) + load_path, c_parser = self.LoadEmbeddedFile(embed_path, path_loc) if c_parser is None: return 1 # error was already shown - return self._Exec(cmd_val, arg_r, load_path, c_parser) + # TODO: + # - ctx_Module is like ctx_FrontFrame, but it fiddles the global + # frame, mem.var_stack[0] + # - it returns value.Obj, and you bind that + return self._UseExec(cmd_val, arg_r, load_path, c_parser) else: - # 'source' respects $PATH - resolved = self.search_path.LookupOne(path_arg, - exec_required=False) - if resolved is None: - resolved = path_arg + normalized = libc.realpath(path_arg) + if normalized is None: + self.errfmt.Print_("use: couldn't find %r" % path_arg, + blame_loc=path_loc) + return 1 - f, c_parser = self._LoadDiskFile(resolved, path_loc) + # TODO: consult _disk_cache = {} + + f, c_parser = self._LoadDiskFile(normalized, path_loc) if c_parser is None: return 1 # error was already shown + # TODO: ctx_Module with process.ctx_FileCloser(f): - return self._Exec(cmd_val, arg_r, path_arg, c_parser) + return self._UseExec(cmd_val, arg_r, path_arg, c_parser) raise AssertionError() - def _Exec(self, cmd_val, arg_r, path, c_parser): - # type: (cmd_value.Argv, args.Reader, str, cmd_parse.CommandParser) -> int - call_loc = cmd_val.arg_locs[0] - - # A sourced module CAN have a new arguments array, but it always shares - # the same variable scope as the caller. The caller could be at either a - # global or a local scope. - - # TODO: I wonder if we compose the enter/exit methods more easily. - - with dev.ctx_Tracer(self.tracer, 'source', cmd_val.argv): - source_argv = arg_r.Rest() - with state.ctx_Source(self.mem, path, source_argv): - with state.ctx_ThisDir(self.mem, path): - src = source.SourcedFile(path, call_loc) - with alloc.ctx_SourceCode(self.arena, src): - try: - status = main_loop.Batch( - self.cmd_ev, - c_parser, - self.errfmt, - cmd_flags=cmd_eval.RaiseControlFlow) - except vm.IntControlFlow as e: - if e.IsReturn(): - status = e.StatusCode() - else: - raise - - return status - def _PrintFreeForm(row): # type: (Tuple[str, str, Optional[str]]) -> None From fe5dd8e8aed389c3d3d20c3a6f1443e2e5e69b52 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 23:12:40 -0400 Subject: [PATCH 275/506] [builtin/use] Basic version of ctx_ModuleEval and caching It kinda works, but I need to improve the tests. And ctx_ModuleEval should not use the "front frame". Because then it could read from the calling module! It should just replace the global frame, which is mem.var_stack[0]. I also need to add the __invoke__ method. That should be a value.Proc, although the body will be written in typed Python, not YSH. --- builtin/meta_oils.py | 114 ++++++++++++++++++++++---------- core/state.py | 45 +++++++++++++ spec/testdata/module2/util.ysh | 10 +++ spec/ysh-builtin-module.test.sh | 39 +++++++++-- 4 files changed, 168 insertions(+), 40 deletions(-) create mode 100644 spec/testdata/module2/util.ysh diff --git a/builtin/meta_oils.py b/builtin/meta_oils.py index 35bc4db49b..1c30f412b7 100644 --- a/builtin/meta_oils.py +++ b/builtin/meta_oils.py @@ -15,22 +15,22 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus from _devbuild.gen.syntax_asdl import source, loc, loc_t -from _devbuild.gen.value_asdl import Obj +from _devbuild.gen.value_asdl import Obj, value_t from core import alloc from core import dev from core import error +from core.error import e_usage from core import executor from core import main_loop from core import process -from core.error import e_usage from core import pyutil # strerror from core import state from core import vm from data_lang import j8_lite -from frontend import flag_util from frontend import consts +from frontend import flag_util from frontend import reader -from mycpp.mylib import log, print_stderr +from mycpp.mylib import log, print_stderr, NewDict from pylib import os_path from osh import cmd_eval @@ -99,6 +99,33 @@ def Run(self, cmd_val): cmd_flags=cmd_eval.RaiseControlFlow) +def _VarName(module_path): + # type: (str) -> str + """Convert ///path/foo-bar.ysh -> foo_bar + + Design issue: proc vs. func naming conventinos imply treating hyphens + differently. + + foo-bar myproc + var x = `foo-bar`.myproc + + I guess use this for now: + + foo_bar myproc + var x = foo_bar.myproc + + The user can also choose this: + + fooBar myproc + var x = fooBar.myproc + """ + basename = os_path.basename(module_path) + i = basename.rfind('.') + if i != -1: + basename = basename[:i] + return basename.replace('-', '_') + + class ShellFile(vm._Builtin): """ These share code: @@ -207,29 +234,32 @@ def _SourceExec(self, cmd_val, arg_r, path, c_parser): return status def _UseExec(self, cmd_val, arg_r, path, c_parser): - # type: (cmd_value.Argv, args.Reader, str, cmd_parse.CommandParser) -> int + # type: (cmd_value.Argv, args.Reader, str, cmd_parse.CommandParser) -> Obj call_loc = cmd_val.arg_locs[0] - with dev.ctx_Tracer(self.tracer, 'use', None): - with state.ctx_ThisDir(self.mem, path): - - # TODO: change the src to source.ShellFile - - src = source.SourcedFile(path, call_loc) - with alloc.ctx_SourceCode(self.arena, src): - try: - status = main_loop.Batch( - self.cmd_ev, - c_parser, - self.errfmt, - cmd_flags=cmd_eval.RaiseControlFlow) - except vm.IntControlFlow as e: - if e.IsReturn(): - status = e.StatusCode() - else: - raise + d = NewDict() # type: Dict[str, value_t] + with state.ctx_ModuleEval(self.mem, d): + with dev.ctx_Tracer(self.tracer, 'use', None): + with state.ctx_ThisDir(self.mem, path): - return status + # TODO: change the src to source.ShellFile + + src = source.SourcedFile(path, call_loc) + with alloc.ctx_SourceCode(self.arena, src): + try: + unused_status = main_loop.Batch( + self.cmd_ev, + c_parser, + self.errfmt, + cmd_flags=cmd_eval.RaiseControlFlow) + except vm.IntControlFlow as e: + if e.IsReturn(): + status = e.StatusCode() + else: + raise + + module_obj = Obj(None, d) + return module_obj def _Source(self, cmd_val): # type: (cmd_value.Argv) -> int @@ -355,19 +385,29 @@ def _Use(self, cmd_val): else: embed_path = None + # Important, consider: + # use symlink.ysh # where symlink.ysh -> realfile.ysh + # + # Then the cache key would be '/some/path/realfile.ysh' + # But the variable name bound is 'symlink' + var_name = _VarName(path_arg) + #log('var %s', var_name) + if embed_path is not None: - # TODO: consult _embed_cache = {} + # Embedded modules are cached using /// path as cache key + cached_obj = self._embed_cache.get(embed_path) + if cached_obj: + state.SetLocalValue(self.mem, var_name, cached_obj) + return 0 load_path, c_parser = self.LoadEmbeddedFile(embed_path, path_loc) if c_parser is None: return 1 # error was already shown - # TODO: - # - ctx_Module is like ctx_FrontFrame, but it fiddles the global - # frame, mem.var_stack[0] - # - it returns value.Obj, and you bind that + obj = self._UseExec(cmd_val, arg_r, load_path, c_parser) + state.SetLocalValue(self.mem, var_name, obj) + self._embed_cache[embed_path] = obj - return self._UseExec(cmd_val, arg_r, load_path, c_parser) else: normalized = libc.realpath(path_arg) if normalized is None: @@ -375,17 +415,23 @@ def _Use(self, cmd_val): blame_loc=path_loc) return 1 - # TODO: consult _disk_cache = {} + # Disk modules are cached using normalized path as cache key + cached_obj = self._disk_cache.get(normalized) + if cached_obj: + var_name = _VarName(path_arg) + state.SetLocalValue(self.mem, var_name, cached_obj) + return 0 f, c_parser = self._LoadDiskFile(normalized, path_loc) if c_parser is None: return 1 # error was already shown - # TODO: ctx_Module with process.ctx_FileCloser(f): - return self._UseExec(cmd_val, arg_r, path_arg, c_parser) + obj = self._UseExec(cmd_val, arg_r, path_arg, c_parser) + state.SetLocalValue(self.mem, var_name, obj) + self._disk_cache[normalized] = obj - raise AssertionError() + return 0 def _PrintFreeForm(row): diff --git a/core/state.py b/core/state.py index 0bc6d30ffc..848948597e 100644 --- a/core/state.py +++ b/core/state.py @@ -1183,6 +1183,45 @@ def __exit__(self, type, value, traceback): self.mem.var_stack[-1] = self.rear_frame +class ctx_ModuleEval(object): + + def __init__(self, mem, out_dict): + # type: (Mem, Dict[str, value_t]) -> None + pass + self.rear_frame = mem.var_stack[-1] + + # __rear__ gets a lookup rule + self.front_frame = NewDict() # type: Dict[str, Cell] + self.front_frame['__rear__'] = Cell(False, False, False, + value.Frame(self.rear_frame)) + + mem.var_stack[-1] = self.front_frame + + self.mem = mem + self.out_dict = out_dict + + def __enter__(self): + # type: () -> None + pass + + def __exit__(self, type, value, traceback): + # type: (Any, Any, Any) -> None + + for name, cell in iteritems(self.front_frame): + #log('name %r', name) + #log('cell %r', cell) + + # User can hide variables with _ suffix + # e.g. for i_ in foo bar { echo $i_ } + if name.endswith('_'): + continue + + self.out_dict[name] = cell.val + + # Restore + self.mem.var_stack[-1] = self.rear_frame + + class ctx_Eval(object): """Push temporary set of variables, $0, $1, $2, etc.""" @@ -2721,6 +2760,12 @@ def _SetGlobalValue(mem, name, val): mem.SetNamed(location.LName(name), val, scope_e.GlobalOnly) +def SetLocalValue(mem, name, val): + # type: (Mem, str, value_t) -> None + """For 'use' builtin.""" + mem.SetNamed(location.LName(name), val, scope_e.LocalOnly) + + def ExportGlobalString(mem, name, s): # type: (Mem, str, str) -> None """Helper for completion, $PWD, $OLDPWD, etc.""" diff --git a/spec/testdata/module2/util.ysh b/spec/testdata/module2/util.ysh new file mode 100644 index 0000000000..0e5948d18b --- /dev/null +++ b/spec/testdata/module2/util.ysh @@ -0,0 +1,10 @@ + +const MY_INTEGER = 42 + +proc log { + echo log @ARGV +} + +proc die { + echo die @ARGV +} diff --git a/spec/ysh-builtin-module.test.sh b/spec/ysh-builtin-module.test.sh index 8bbf4b521c..665892fa72 100644 --- a/spec/ysh-builtin-module.test.sh +++ b/spec/ysh-builtin-module.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 2 #### source-guard is an old way of preventing redefinition - could remove it shopt --set ysh:upgrade @@ -38,20 +38,47 @@ stdin status=0 ## END -#### use foo.ysh creates a value.Obj +#### use foo.ysh creates a value.Obj, and it's cached on later invocations + +shopt --set ysh:upgrade + +use $REPO_ROOT/spec/testdata/module2/util.ysh + +# This is a value.Obj +pp test_ (util) + +var saved_util = util use $REPO_ROOT/spec/testdata/module2/util.ysh -var methods = Object(null, {}) -var obj = Object(methods, {x: 1}) -pp test_ (obj) -pp test_ (methods) +# These should have the same ID += saved_util += util + +# TODO: also create a symlink + +ln -s $REPO_ROOT/spec/testdata/module2/util.ysh symlink.ysh +use symlink.ysh +echo 'symlink' += symlink + + +#util log 'hello' + +## STDOUT: +## END + +#### use foo.ysh creates a value.Obj with __invoke__ +shopt --set ysh:upgrade + +use $REPO_ROOT/spec/testdata/module2/util.ysh # This is a value.Obj pp test_ (util) util log 'hello' +util die 'hello' ## STDOUT: ## END From b62840266c4450f5fbab55f1b8fa9da5c93ffc78 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 29 Sep 2024 23:51:24 -0400 Subject: [PATCH 276/506] [builtin] Add id() function to test 'use' module caching It's defined on mutable values. Implement ctx_ModuleEval() properly, so variables don't leak. Next: __invoke__. --- builtin/func_reflect.py | 37 +++++++++++++++++++++++++++++++-- core/shell.py | 1 + core/state.py | 32 ++++++++++++++-------------- doc/ref/chap-builtin-func.md | 8 +++++++ doc/ref/toc-ysh.md | 3 ++- spec/testdata/module2/util.ysh | 6 ++++++ spec/ysh-builtin-module.test.sh | 34 +++++++++++++++++++++--------- 7 files changed, 92 insertions(+), 29 deletions(-) diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index 4f40193937..b7339fdf30 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -6,17 +6,19 @@ from _devbuild.gen.runtime_asdl import (scope_e) from _devbuild.gen.syntax_asdl import source -from _devbuild.gen.value_asdl import (value, value_t) +from _devbuild.gen.value_asdl import (value, value_e, value_t) from core import alloc from core import error from core import main_loop from core import state from core import vm +from data_lang import j8 from frontend import location from frontend import reader from frontend import typed_args -from mycpp.mylib import log +from mycpp import mops +from mycpp.mylib import log, tagswitch from ysh import expr_eval from typing import TYPE_CHECKING @@ -27,6 +29,37 @@ _ = log +class Id(vm._Callable): + """Return an integer object ID, like Python's id(). + + Long shot: pointer tagging, boxless value_t, and small string optimization + could mean that value.Str is no longer heap-allocated, and thus doesn't + have a GC ID? + + What about value.{Bool,Int,Float}? + + I guess only mutable objects can have IDs then + """ + def __init__(self): + # type: () -> None + vm._Callable.__init__(self) + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + val = rd.PosValue() + rd.Done() + + # Select mutable values for now + with tagswitch(val) as case: + if case(value_e.List, value_e.Dict, value_e.Obj): + id_ = j8.HeapValueId(val) + return value.Int(mops.IntWiden(id_)) + else: + raise error.TypeErr(val, 'id() expected List, Dict, or Obj', + rd.BlamePos()) + raise AssertionError() + + class Shvar_get(vm._Callable): """Look up with dynamic scope.""" diff --git a/core/shell.py b/core/shell.py index abd4577fde..5b51fe3fb2 100644 --- a/core/shell.py +++ b/core/shell.py @@ -865,6 +865,7 @@ def Main( func_eggex.MatchFunc(func_eggex.S, None, mem)) _AddBuiltinFunc(mem, '_end', func_eggex.MatchFunc(func_eggex.E, None, mem)) + _AddBuiltinFunc(mem, 'id', func_reflect.Id()) _AddBuiltinFunc(mem, 'parseCommand', func_reflect.ParseCommand(parse_ctx, errfmt)) _AddBuiltinFunc(mem, 'parseExpr', diff --git a/core/state.py b/core/state.py index 848948597e..e13a6b3410 100644 --- a/core/state.py +++ b/core/state.py @@ -1149,6 +1149,9 @@ class ctx_FrontFrame(object): def __init__(self, mem, out_dict): # type: (Mem, Dict[str, value_t]) -> None + self.mem = mem + self.out_dict = out_dict + self.rear_frame = mem.var_stack[-1] # __rear__ gets a lookup rule @@ -1158,9 +1161,6 @@ def __init__(self, mem, out_dict): mem.var_stack[-1] = self.front_frame - self.mem = mem - self.out_dict = out_dict - def __enter__(self): # type: () -> None pass @@ -1184,22 +1184,23 @@ def __exit__(self, type, value, traceback): class ctx_ModuleEval(object): + """Evaluate a module with a new global stack frame. - def __init__(self, mem, out_dict): - # type: (Mem, Dict[str, value_t]) -> None - pass - self.rear_frame = mem.var_stack[-1] + e.g. setglobal in the new module doesn't leak - # __rear__ gets a lookup rule - self.front_frame = NewDict() # type: Dict[str, Cell] - self.front_frame['__rear__'] = Cell(False, False, False, - value.Frame(self.rear_frame)) - - mem.var_stack[-1] = self.front_frame + Different from ctx_FrontFrame because the new code can't see variables in + the old frame. + """ + def __init__(self, mem, out_dict): + # type: (Mem, Dict[str, value_t]) -> None self.mem = mem self.out_dict = out_dict + self.new_frame = NewDict() # type: Dict[str, Cell] + self.saved_frame = mem.var_stack[0] + mem.var_stack[0] = self.new_frame + def __enter__(self): # type: () -> None pass @@ -1207,7 +1208,7 @@ def __enter__(self): def __exit__(self, type, value, traceback): # type: (Any, Any, Any) -> None - for name, cell in iteritems(self.front_frame): + for name, cell in iteritems(self.new_frame): #log('name %r', name) #log('cell %r', cell) @@ -1218,8 +1219,7 @@ def __exit__(self, type, value, traceback): self.out_dict[name] = cell.val - # Restore - self.mem.var_stack[-1] = self.rear_frame + self.mem.var_stack[0] = self.saved_frame class ctx_Eval(object): diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index c4aea41587..cdd9c12166 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -389,6 +389,14 @@ Like `Match => end()`, but accesses the global match created by `~`: ## Introspection +### `id()` + +Returns an integer ID for mutable values like List, Dict, and Obj. + +You can use it to test if two names refer to the same instance. + +`id()` is undefined on immutable values like Bool, Int, Float, Str, etc. + ### `shvarGet()` Given a variable name, return its value. It uses the "dynamic scope" rule, diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 9011e4f8bf..0402f992a2 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -84,7 +84,8 @@ X [Proc] name() location() toJson() toJson8() fromJson8() X [J8 Decode] J8.Bool() J8.Int() ... [Pattern] _group() _start() _end() - [Introspection] shvarGet() getVar() setVar() + [Introspection] id() + shvarGet() getVar() setVar() parseCommand() X parseExpr() evalExpr() [Hay Config] parseHay() evalHay() X [Hashing] sha1dc() sha256() diff --git a/spec/testdata/module2/util.ysh b/spec/testdata/module2/util.ysh index 0e5948d18b..7ad72cfd91 100644 --- a/spec/testdata/module2/util.ysh +++ b/spec/testdata/module2/util.ysh @@ -1,4 +1,7 @@ +# should be null +echo "caller_no_leak = $[getVar('caller_no_leak')]" + const MY_INTEGER = 42 proc log { @@ -8,3 +11,6 @@ proc log { proc die { echo die @ARGV } + +setvar setvar_noleak = 'util.ysh' +setglobal setglobal_noleak = 'util.ysh' diff --git a/spec/ysh-builtin-module.test.sh b/spec/ysh-builtin-module.test.sh index 665892fa72..aead441180 100644 --- a/spec/ysh-builtin-module.test.sh +++ b/spec/ysh-builtin-module.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 2 +## oils_failures_allowed: 1 #### source-guard is an old way of preventing redefinition - could remove it shopt --set ysh:upgrade @@ -42,31 +42,45 @@ status=0 shopt --set ysh:upgrade +var caller_no_leak = 42 + use $REPO_ROOT/spec/testdata/module2/util.ysh # This is a value.Obj -pp test_ (util) +pp test_ (['util', util]) +var id1 = id(util) var saved_util = util use $REPO_ROOT/spec/testdata/module2/util.ysh +pp test_ (['repeated', util]) +var id2 = id(util) -# These should have the same ID -= saved_util -= util - -# TODO: also create a symlink +# Create a symlink to test normalization ln -s $REPO_ROOT/spec/testdata/module2/util.ysh symlink.ysh use symlink.ysh -echo 'symlink' -= symlink +pp test_ (['symlink', symlink]) +var id3 = id(symlink) + +#pp test_ ([id1, id2, id3]) +# Make sure they are all the same object +assert [id1 === id2] +assert [id2 === id3] -#util log 'hello' +# Doesn't leak from util.ysh +echo "setvar_noleak $[getVar('setvar_noleak')]" +echo "setglobal_noleak $[getVar('setglobal_noleak')]" ## STDOUT: +caller_no_leak = null +(List) ["util",{"MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh"}] +(List) ["repeated",{"MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh"}] +(List) ["symlink",{"MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh"}] +setvar_noleak null +setglobal_noleak null ## END #### use foo.ysh creates a value.Obj with __invoke__ From b0dbb88641583b755dfcb74b53bfcd764769b5f6 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 30 Sep 2024 00:24:38 -0400 Subject: [PATCH 277/506] [builtin/use] Implement use --extern Add spec tests Start documenting the 'use' command --- builtin/func_reflect.py | 1 + builtin/meta_oils.py | 8 +++- builtin/pure_ysh.py | 6 +-- doc/ref/chap-builtin-cmd.md | 23 +++++++--- doc/ref/toc-ysh.md | 7 +++- frontend/args.py | 5 +++ spec/ysh-builtin-module.test.sh | 74 ++++++++++++++++++++------------- 7 files changed, 82 insertions(+), 42 deletions(-) diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index b7339fdf30..af8fddc823 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -40,6 +40,7 @@ class Id(vm._Callable): I guess only mutable objects can have IDs then """ + def __init__(self): # type: () -> None vm._Callable.__init__(self) diff --git a/builtin/meta_oils.py b/builtin/meta_oils.py index 1c30f412b7..c27ccdedc7 100644 --- a/builtin/meta_oils.py +++ b/builtin/meta_oils.py @@ -368,7 +368,13 @@ def _Use(self, cmd_val): import $LIB_YSH/stdlib """ - _, arg_r = flag_util.ParseCmdVal('use', cmd_val) + attrs, arg_r = flag_util.ParseCmdVal('use', cmd_val) + arg = arg_types.use(attrs.attrs) + + # Accepts any args + if arg.extern_: # use --extern grep # no-op for static analysis + return 0 + path_arg, path_loc = arg_r.ReadRequired2('requires a module path') # TODO on usage: # - typed arg is value.Place diff --git a/builtin/pure_ysh.py b/builtin/pure_ysh.py index 810ed19c96..4e2bcccf7b 100644 --- a/builtin/pure_ysh.py +++ b/builtin/pure_ysh.py @@ -206,9 +206,9 @@ def Run(self, cmd_val): # type: (cmd_value.Argv) -> int # This means we ignore -- , which is consistent - arg, arg_r = flag_util.ParseCmdVal('append', - cmd_val, - accept_typed_args=True) + _, arg_r = flag_util.ParseCmdVal('append', + cmd_val, + accept_typed_args=True) rd = typed_args.ReaderForProc(cmd_val) val = rd.PosValue() diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index 3e9a2bc3f4..fa0915c1aa 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -353,13 +353,22 @@ Use it like this: ### use -TODO +Import code from other files, creating an `Obj` that acts like a namespace. -Reuse code from other files, respecting namespaces. + use my-dir/my-module.ysh - use lib/foo.ysh # foo myproc, $[foo.attr] - # implicit $_this_dir aka relative import + echo $[my_module.my_integer] # the module Obj has attributes + my_module myproc # the module Obj is invokable +The evaluation of such files is cached, so it won't be re-evaluated if `use` is called again. + + + + -Also a declaration +The `--extern` flag make the invocation do nothing. It can be used be tools to +analyze what names are in the file. - use --extern grep sed + use --extern grep sed awk ## I/O diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 0402f992a2..cdb800305e 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -116,10 +116,13 @@ X [Wok] _field() shvar Temporary modify global settings ctx Share and update a temporary "context" push-registers Save registers like $?, PIPESTATUS - [Modules] runproc Run a proc; use as main entry point + [Introspection] runproc Run a proc; use as main entry point + X extern Run an external command, with an ENV + X invoke Control which "invokables" are run + [Modules] source-guard guard against duplicate 'source' is-main false when sourcing a file - X use use names, + use create a module Obj from a source file [I/O] ysh-read flags --all, -0 ysh-echo no -e -n with simple_echo write Like echo, with --, --sep, --end diff --git a/frontend/args.py b/frontend/args.py index 35d277420a..8cad83240e 100644 --- a/frontend/args.py +++ b/frontend/args.py @@ -105,6 +105,11 @@ def Set(self, name, val): # debug-completion -> debug_completion name = name.replace('-', '_') + + # similar hack to avoid C++ keyword in frontend/flag_gen.py + if name == 'extern': + name = 'extern_' + self.attrs[name] = val if 0: diff --git a/spec/ysh-builtin-module.test.sh b/spec/ysh-builtin-module.test.sh index aead441180..5a41b61091 100644 --- a/spec/ysh-builtin-module.test.sh +++ b/spec/ysh-builtin-module.test.sh @@ -38,6 +38,50 @@ stdin status=0 ## END +#### use builtin usage + +use +echo no-arg=$? + +use foo +echo one-arg=$? + +use --extern foo +echo extern=$? + +use --bad-flag +echo bad-flag=$? + +use too many +echo too-many=$? + +use ///no-builtin +echo no-builtin=$? + + +## STDOUT: +no-arg=2 +one-arg=1 +extern=0 +bad-flag=2 +too-many=2 +no-builtin=1 +## END + + +#### use --extern is a no-op, for static analysis + +use --extern grep sed awk +echo status=$? + +use --extern zzz +echo status=$? + +## STDOUT: +status=0 +status=0 +## END + #### use foo.ysh creates a value.Obj, and it's cached on later invocations shopt --set ysh:upgrade @@ -97,33 +141,3 @@ util die 'hello' ## STDOUT: ## END -#### use builtin usage - -use -echo no-arg=$? - -use foo -echo one-arg=$? - -use --extern foo -echo extern=$? - -use --bad-flag -echo bad-flag=$? - -use too many -echo too-many=$? - -use ///no-builtin -echo no-builtin=$? - - -## STDOUT: -no-arg=2 -one-arg=1 -extern=1 -bad-flag=2 -too-many=2 -no-builtin=1 -## END - From c982b901e609ad79d27865d2c476519c4f1fa205 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 30 Sep 2024 13:37:04 -0400 Subject: [PATCH 278/506] [builtin] get() accepts an Obj. This is equivalent to = get(propView(obj), 'name', 'default') But this is nicer: = get(obj, 'name', 'default') It is essential reflection. TODO: the third default param could be optional. [ysh modules] Save the module namespace on procs and funcs And also on sh funcs for now, since they are value.Proc --- builtin/method_dict.py | 23 +++++++++++++---- core/completion_test.py | 7 ++++-- core/state.py | 27 +++++++++++++++++--- core/value.asdl | 4 +-- doc/ref/chap-special-var.md | 13 ++++++++++ doc/ref/toc-ysh.md | 2 ++ osh/cmd_eval.py | 11 +++++--- spec/testdata/module2/globals.ysh | 36 ++++++++++++++++++++++++++ spec/ysh-builtin-module.test.sh | 42 ++++++++++++++++++++++++++++++- 9 files changed, 148 insertions(+), 17 deletions(-) create mode 100644 spec/testdata/module2/globals.ysh diff --git a/builtin/method_dict.py b/builtin/method_dict.py index 648c5c18a4..747caf3b6d 100644 --- a/builtin/method_dict.py +++ b/builtin/method_dict.py @@ -2,14 +2,15 @@ from __future__ import print_function -from _devbuild.gen.value_asdl import (value, value_t) +from _devbuild.gen.value_asdl import (value, value_e, value_t, Obj) +from core import error from core import vm from frontend import typed_args from mycpp import mylib -from mycpp.mylib import log +from mycpp.mylib import log, tagswitch -from typing import List +from typing import cast, List _ = log @@ -72,9 +73,21 @@ def __init__(self): def Call(self, rd): # type: (typed_args.Reader) -> value_t - dictionary = rd.PosDict() + obj = rd.PosValue() key = rd.PosStr() default_value = rd.PosValue() rd.Done() - return dictionary.get(key, default_value) + UP_obj = obj + with tagswitch(obj) as case: + if case(value_e.Dict): + obj = cast(value.Dict, UP_obj) + d = obj.d + elif case(value_e.Obj): + obj = cast(Obj, UP_obj) + d = obj.d + else: + raise error.TypeErr(obj, 'get() expected Dict or Obj', + rd.BlamePos()) + + return d.get(key, default_value) diff --git a/core/completion_test.py b/core/completion_test.py index 067474fa63..95a6cf28ae 100755 --- a/core/completion_test.py +++ b/core/completion_test.py @@ -203,11 +203,14 @@ def testShellFuncExecution(self): """, arena=arena) node = c_parser.ParseLogicalLine() - proc = value.Proc(node.name, node.name_tok, proc_sig.Open, node.body, - [], True, None) cmd_ev = test_lib.InitCommandEvaluator(arena=arena) + frame = cmd_ev.mem.var_stack[0] + assert frame is not None + proc = value.Proc(node.name, node.name_tok, proc_sig.Open, node.body, + [], True, frame) + comp_lookup = completion.Lookup() a = completion.ShellFuncAction(cmd_ev, proc, comp_lookup) comp = self._CompApi(['f'], 0, 'f') diff --git a/core/state.py b/core/state.py index e13a6b3410..206ca6c364 100644 --- a/core/state.py +++ b/core/state.py @@ -970,10 +970,16 @@ class ctx_FuncCall(object): def __init__(self, mem, func): # type: (Mem, value.Func) -> None + self.saved_globals = mem.var_stack[0] + + assert func.module_frame is not None + mem.var_stack[0] = func.module_frame + frame = NewDict() # type: Dict[str, Cell] mem.var_stack.append(frame) mem.PushCall(func.name, func.parsed.name) + self.mem = mem def __enter__(self): @@ -985,6 +991,8 @@ def __exit__(self, type, value, traceback): self.mem.PopCall() self.mem.var_stack.pop() + self.mem.var_stack[0] = self.saved_globals + class ctx_ProcCall(object): """For proc calls, including shell functions.""" @@ -993,14 +1001,15 @@ def __init__(self, mem, mutable_opts, proc, argv): # type: (Mem, MutableOpts, value.Proc, List[str]) -> None # TODO: - # - argv stack shouldn't be used for procs - # - we can bind a real variable @A if we want - # - procs should be in the var namespace - # # should we separate procs and shell functions? # - dynamic scope is one difference # - '$@" shift etc. are another difference + self.saved_globals = mem.var_stack[0] + + assert proc.module_frame is not None + mem.var_stack[0] = proc.module_frame + frame = NewDict() # type: Dict[str, Cell] assert argv is not None @@ -1039,6 +1048,8 @@ def __exit__(self, type, value, traceback): if self.sh_compat: self.mem.argv_stack.pop() + self.mem.var_stack[0] = self.saved_globals + class ctx_Temp(object): """For FOO=bar myfunc, etc.""" @@ -1613,6 +1624,14 @@ def InsideFunction(self): # Don't run it inside functions return len(self.var_stack) > 1 + def GlobalFrame(self): + # type: () -> Dict[str, Cell] + """For defining the global scope of modules. + + It's affected by ctx_ModuleEval() + """ + return self.var_stack[0] + def PushSource(self, source_name, argv): # type: (str, List[str]) -> None """ For 'source foo.sh 1 2 3' """ diff --git a/core/value.asdl b/core/value.asdl index b1d38dc3e4..2755488c45 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -155,12 +155,12 @@ module value | Proc(str name, Token name_tok, proc_sig sig, command body, ProcDefaults? defaults, bool sh_compat, # module is where "global" lookups happen - Dict[str, Cell]? module_) + Dict[str, Cell] module_frame) | Func(str name, Func parsed, List[value] pos_defaults, Dict[str, value] named_defaults, # module is where "global" lookups happen - Dict[str, Cell]? module_) + Dict[str, Cell] module_frame) # for i in (1:n) { echo $i } # both ends are required | Range(int lower, int upper) diff --git a/doc/ref/chap-special-var.md b/doc/ref/chap-special-var.md index 3a2bb21248..55b9cfdf73 100644 --- a/doc/ref/chap-special-var.md +++ b/doc/ref/chap-special-var.md @@ -155,6 +155,19 @@ The float value for "infinity". You can negate it to get "negative infinity". (The name is consistent with the C language.) +## Module + +### `__export__` + +A module is evaluated upon `import`. After evaluation, the names in the +`__export__` `List` are put in the resulting module `Obj` instance. + + + ## Shell Vars ### IFS diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index cdb800305e..436eb9b3f9 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -79,6 +79,7 @@ X [Proc] name() location() toJson() [Dict] keys() values() get() [Float] floatsEqual() X isinf() X isnan() [Obj] Object() prototype() propView() + get() [Word] glob() maybe() [Serialize] toJson() fromJson() toJson8() fromJson8() @@ -343,6 +344,7 @@ X [External Lang] BEGIN END when (awk) OILS_GC_STATS OILS_GC_STATS_FD LIB_YSH [Float] NAN INFINITY + [Module] __export__ ``` diff --git a/spec/ysh-builtin-help.test.sh b/spec/ysh-builtin-help.test.sh index b842e12cfb..5703585c14 100644 --- a/spec/ysh-builtin-help.test.sh +++ b/spec/ysh-builtin-help.test.sh @@ -97,3 +97,28 @@ status=0 status=0 ## END + +#### help oils-err-12 (case insensitive) + +# note that the topics are lower-casedo + +help oils-err-12 | grep -o 'catalog.html#oils-err-12' +echo status=$? + +help OILS-ERR-12 | grep -o 'catalog.html#oils-err-12' +echo status=$? + +# these are bad + +# help oils-err-zz +# echo status=$? + +# help OILS-ERR-zz +# echo status=$? + +## STDOUT: +catalog.html#oils-err-12 +status=0 +catalog.html#oils-err-12 +status=0 +## END From 17b0cbc8d806f6bafbfd8f93f2d6058c3865661b Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 1 Oct 2024 12:07:25 -0400 Subject: [PATCH 282/506] [ysh] Rename __export__ -> __provide__ export is used for env vars! Add test cases for use foo.ysh --pick use foo.ysh --all-provided use foo.ysh --all-for-testing --- core/state.py | 18 +++++----- doc/ref/chap-special-var.md | 8 ++--- doc/ref/toc-ysh.md | 2 +- spec/testdata/module2/bad-export.ysh | 6 ---- ...d-export-type.ysh => bad-provide-type.ysh} | 2 +- spec/testdata/module2/bad-provide.ysh | 6 ++++ spec/testdata/module2/globals.ysh | 4 +-- .../module2/{no-export.ysh => no-provide.ysh} | 2 +- spec/testdata/module2/util.ysh | 1 + spec/ysh-builtin-module.test.sh | 35 +++++++++++++++---- 10 files changed, 52 insertions(+), 32 deletions(-) delete mode 100644 spec/testdata/module2/bad-export.ysh rename spec/testdata/module2/{bad-export-type.ysh => bad-provide-type.ysh} (67%) create mode 100644 spec/testdata/module2/bad-provide.ysh rename spec/testdata/module2/{no-export.ysh => no-provide.ysh} (60%) diff --git a/core/state.py b/core/state.py index 913b99f61f..f352c8293a 100644 --- a/core/state.py +++ b/core/state.py @@ -1224,33 +1224,33 @@ def __exit__(self, type, value_, traceback): # Now look in __export__ for the list of names to expose - cell = self.new_frame.get('__export__') + cell = self.new_frame.get('__provide__') if cell is None: - self.out_errors.append("Module is missing 'export' List") + self.out_errors.append("Module is missing 'provide' List") return - export_val = cell.val - with tagswitch(export_val) as case: + provide_val = cell.val + with tagswitch(provide_val) as case: if case(value_e.List): - for val in cast(value.List, export_val).items: + for val in cast(value.List, provide_val).items: if val.tag() == value_e.Str: name = cast(value.Str, val).s cell = self.new_frame.get(name) if cell is None: self.out_errors.append( - "Name %r was exported, but not defined" % name) + "Name %r was provided, but not defined" % name) continue self.out_dict[name] = cell.val else: self.out_errors.append( - "Expected Str in __export__ List, got %s" % + "Expected Str in __provide__ List, got %s" % ui.ValType(val)) else: - self.out_errors.append("__export__ should be a List, got %s" % - ui.ValType(export_val)) + self.out_errors.append("__provide__ should be a List, got %s" % + ui.ValType(provide_val)) class ctx_Eval(object): diff --git a/doc/ref/chap-special-var.md b/doc/ref/chap-special-var.md index 55b9cfdf73..a8c9a22d23 100644 --- a/doc/ref/chap-special-var.md +++ b/doc/ref/chap-special-var.md @@ -157,13 +157,13 @@ The float value for "infinity". You can negate it to get "negative infinity". ## Module -### `__export__` +### `__provide__` -A module is evaluated upon `import`. After evaluation, the names in the -`__export__` `List` are put in the resulting module `Obj` instance. +A module is evaluated upon `use`. After evaluation, the names in the +`__provide__` `List` are put in the resulting module `Obj` instance. diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 0a3b50c474..3425a1e21d 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -344,7 +344,7 @@ X [External Lang] BEGIN END when (awk) OILS_GC_STATS OILS_GC_STATS_FD LIB_YSH [Float] NAN INFINITY - [Module] __export__ + [Module] __provide__ ``` %s ]' % ValueIdString(val)) + # Showing the ID would be nice for pretty printing, but + # the problem is we'd have to show it TWICE to make it + # meaningful + # + #self.buf.write('[ -->%s ]' % ValueIdString(val)) + self.buf.write('[...]') return else: # node.js prints which index closes the cycle raise error.Encode( "Can't encode List%s in object cycle" % ValueIdString(val)) - - self.visited[heap_id] = EXPLORING - self._PrintList(val, level) - self.visited[heap_id] = FINISHED + else: + self.visiting[heap_id] = True + self._PrintList(val, level) + self.visiting[heap_id] = False elif case(value_e.Dict): val = cast(value.Dict, UP_val) @@ -567,26 +557,19 @@ def Print(self, val, level=0): # Cycle detection, only for containers that can be in cycles heap_id = HeapValueId(val) - node_state = self.visited.get(heap_id, UNSEEN) - if node_state == FINISHED: - # Print it AGAIN. We print a JSON tree, which means we can - # visit and print nodes MANY TIMES, as long as they're not - # in a cycle. - self._PrintDict(val, level) - return - if node_state == EXPLORING: + if self.visiting.get(heap_id, False): if self.options & SHOW_CYCLES: - self.buf.write('{ -->%s }' % ValueIdString(val)) + self.buf.write('{...}') return else: # node.js prints which key closes the cycle raise error.Encode( "Can't encode Dict%s in object cycle" % ValueIdString(val)) - - self.visited[heap_id] = EXPLORING - self._PrintDict(val, level) - self.visited[heap_id] = FINISHED + else: + self.visiting[heap_id] = True + self._PrintDict(val, level) + self.visiting[heap_id] = False elif case(value_e.Obj): val = cast(Obj, UP_val) @@ -597,31 +580,19 @@ def Print(self, val, level=0): # Cycle detection, only for containers that can be in cycles heap_id = HeapValueId(val) - node_state = self.visited.get(heap_id, UNSEEN) - if node_state == FINISHED: - # Print it AGAIN. We print a JSON tree, which means we can - # visit and print nodes MANY TIMES, as long as they're not - # in a cycle. - self._PrintObj(val, level) - return - if node_state == EXPLORING: + if self.visiting.get(heap_id, False): if self.options & SHOW_CYCLES: - self.buf.write('{ -->%s }' % ValueIdString(val)) + self.buf.write('{...}') return else: # node.js prints which key closes the cycle raise error.Encode( "Can't encode Obj%s in object cycle" % ValueIdString(val)) - - # TODO: cycle detection is a bit wrong, I think because the - # properties are a Dict[str, value_t], not something with an - # identity - # - # This is only used for pp test_, because SHOW_NON_DATA. - self.visited[heap_id] = EXPLORING - self._PrintObj(val, level) - self.visited[heap_id] = FINISHED + else: + self.visiting[heap_id] = True + self._PrintObj(val, level) + self.visiting[heap_id] = False elif case(value_e.SparseArray): val = cast(value.SparseArray, UP_val) diff --git a/spec/ysh-json.test.sh b/spec/ysh-json.test.sh index a124645ec7..08b60c13fb 100644 --- a/spec/ysh-json.test.sh +++ b/spec/ysh-json.test.sh @@ -233,19 +233,15 @@ echo 'should have failed' var L = [1, 2, 3] setvar L[0] = L - -shopt -s ysh:upgrade -redir >tmp.txt { - pp test_ (L) -} -fgrep -n -o '[ -->' tmp.txt +pp test_ (L) json write (L) -echo 'should have failed' +echo status=$? -## status: 1 +## status: 0 ## STDOUT: -1:[ --> +(List) [[...],2,3] +status=1 ## END #### json write of Dict in cycle @@ -253,18 +249,14 @@ echo 'should have failed' var d = {} setvar d.k = d -shopt -s ysh:upgrade -redir >tmp.txt { - pp test_ (d) -} -fgrep -n -o '{ -->' tmp.txt +pp test_ (d) json write (d) -echo 'should have failed' +echo status=$? -## status: 1 ## STDOUT: -1:{ --> +(Dict) {"k":{...}} +status=1 ## END #### json write of List/Dict referenced twice (bug fix) diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index da15354676..7164396d8b 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -1,5 +1,5 @@ ## our_shell: ysh -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 #### Object() creates prototype chain @@ -162,31 +162,6 @@ echo 'nope' ## STDOUT: ## END -#### pp test_ (obj_with_cycle) - -var d = {k: 42} -setvar d.cycle = d - -var two = [d, d] -pp test_ (two) - -pp test_ (d) - -# This doesn't quite work -var o = Object(null, d) - -pp test_ (o) - -var two = [o, o] -#pp test_ (two) - -var o2 = Object(o, {z: 99}) - -pp test_ (o2) - -## STDOUT: -## END - #### Can all builtin methods with s.upper() var s = 'foo' diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index b7ac40ae3b..130bae5fb5 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -1,5 +1,4 @@ -## oils_failures_allowed: 2 - +## oils_failures_allowed: 1 #### Int = -123 @@ -269,38 +268,116 @@ setvar dict["key_omega"] = omega } ## END -#### List cycle +#### pp test_: List cycle + +var no_cycle = [5, 6] +pp test_ (no_cycle) + +var two = [no_cycle, no_cycle] +pp test_ (two) +#pp value (two) + +echo var L = [42] call L->append(L) - -# BUG -#pp test_ (L) -pp value (L) +pp test_ (L) +#pp value (L) var two = [L, L] - -# BUG -#pp test_ (two) -pp value (two) +pp test_ (two) +#pp value (two) ## STDOUT: +(List) [5,6] +(List) [[5,6],[5,6]] + +(List) [42,[...]] +(List) [[42,[...]],[42,[...]]] ## END +#### pp test_: Dict cycle + +var no_cycle = {z: 99} +pp test_ (no_cycle) -#### Dict cycle +var two = [no_cycle, no_cycle] +pp test_ (two) + +#pp value (two) + +echo var d = {k: 42} setvar d.cycle = d pp test_ (d) -pp value (d) +#pp value (d) var two = [d, d] +pp test_ (two) +#pp value (two) + + +## STDOUT: +(Dict) {"z":99} +(List) [{"z":99},{"z":99}] + +(Dict) {"k":42,"cycle":{...}} +(List) [{"k":42,"cycle":{...}},{"k":42,"cycle":{...}}] +## END + +#### pp test_: Obj cycle + +var methods = Object(null, {__foo__: null}) +var obj = Object(methods, {z: 99}) +pp test_ (obj) -# BUG -#pp test_ (two) +setvar obj.cycle = obj +pp test_ (obj) + +echo -pp value (two) +var two = [obj, obj] +pp test_ (two) ## STDOUT: +(Obj) {"z":99} ==> {"__foo__":null} +(Obj) {"z":99,"cycle":{...}} ==> {"__foo__":null} + +(List) [{"z":99,"cycle":{...}} ==> {"__foo__":null},{"z":99,"cycle":{...}} ==> {"__foo__":null}] +## END + + + +#### pp test_: Obj with dict cycle + +var methods = Object(null, {__foo__: null}) +var no_cycle = Object(methods, {z: 99}) +pp test_ (no_cycle) + +var two = [no_cycle, no_cycle] +pp test_ (two) + +echo + +var d = {k: 42} +setvar d.cycle = d + +# This cycle detection doesn't quite work +# Because we're only considering the object itself + +var o = Object(null, d) +pp test_ (o) + +var two = [o, o] +pp test_ (two) + +#var o2 = Object(o, {z: 99}) +#pp test_ (o2) + +## STDOUT: +(Obj) {"z":99} ==> {"__foo__":null} +(List) [{"z":99} ==> {"__foo__":null},{"z":99} ==> {"__foo__":null}] + ## END + From 3db24cbcb4ec878a8ffb1ab1cf1097a6a34b8897 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 10 Oct 2024 23:21:36 -0400 Subject: [PATCH 315/506] [builtin/pp] Print Obj more distinctly from Dicts It looks like this now ("k":1,"m":1) --> ("__invoke__":null) Rather than using {} and being confused with dicts. --- data_lang/j8.py | 17 +++++++++-------- spec/ysh-builtin-module.test.sh | 10 +++++----- spec/ysh-object.test.sh | 16 ++++++++-------- spec/ysh-printing.test.sh | 12 +++++++----- spec/ysh-proc.test.sh | 2 +- 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/data_lang/j8.py b/data_lang/j8.py index 4034466d72..19f64a11f8 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -297,12 +297,13 @@ def _PrintList(self, val, level): self._BracketIndent(level) self.buf.write(']') - def _PrintMapping(self, d, level): - # type: (Dict[str, value_t], int) -> None + def _PrintMapping(self, d, left, right, level): + # type: (Dict[str, value_t], str, str, int) -> None if len(d) == 0: # Special case like Python/JS - self.buf.write('{}') + self.buf.write(left) + self.buf.write(right) else: - self.buf.write('{') + self.buf.write(left) self._MaybeNewline() i = 0 for k, v in iteritems(d): @@ -323,19 +324,19 @@ def _PrintMapping(self, d, level): self._MaybeNewline() self._BracketIndent(level) - self.buf.write('}') + self.buf.write(right) def _PrintDict(self, val, level): # type: (value.Dict, int) -> None - self._PrintMapping(val.d, level) + self._PrintMapping(val.d, '{', '}', level) def _PrintObj(self, val, level): # type: (Obj, int) -> None - self._PrintMapping(val.d, level) + self._PrintMapping(val.d, '(', ')', level) if val.prototype: - self.buf.write(' ==> ') + self.buf.write(' --> ') self._PrintObj(val.prototype, level) def _PrintBashPrefix(self, type_str, level): diff --git a/spec/ysh-builtin-module.test.sh b/spec/ysh-builtin-module.test.sh index 6d68a4aa7a..35644f10d3 100644 --- a/spec/ysh-builtin-module.test.sh +++ b/spec/ysh-builtin-module.test.sh @@ -148,9 +148,9 @@ echo "setglobal_noleak $[getVar('setglobal_noleak')]" ## STDOUT: caller_no_leak = null -(List) ["util",{"MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh","invokableObj":{"x":3,"y":4} ==> {"__invoke__":}} ==> {"__invoke__":}] -(List) ["repeated",{"MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh","invokableObj":{"x":3,"y":4} ==> {"__invoke__":}} ==> {"__invoke__":}] -(List) ["symlink",{"MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh","invokableObj":{"x":3,"y":4} ==> {"__invoke__":}} ==> {"__invoke__":}] +(List) ["util",("MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh","invokableObj":("x":3,"y":4) --> ("__invoke__":)) --> ("__invoke__":)] +(List) ["repeated",("MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh","invokableObj":("x":3,"y":4) --> ("__invoke__":)) --> ("__invoke__":)] +(List) ["symlink",("MY_INTEGER":42,"log":,"die":,"setvar_noleak":"util.ysh","setglobal_noleak":"util.ysh","invokableObj":("x":3,"y":4) --> ("__invoke__":)) --> ("__invoke__":)] setvar_noleak null setglobal_noleak null ## END @@ -413,8 +413,8 @@ pp test_ (cycle2) echo hi ## STDOUT: -(Obj) {"c1":"c1"} ==> {"__invoke__":} -(Obj) {"c2":"c2"} ==> {"__invoke__":} +(Obj) ("c1":"c1") --> ("__invoke__":) +(Obj) ("c2":"c2") --> ("__invoke__":) hi ## END diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index 7164396d8b..af5aa62e4b 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -53,7 +53,7 @@ pp test_ (prototype(obj)) ## STDOUT: (Null) null -(Obj) {"area":} +(Obj) ("area":) ## END #### propView() @@ -124,9 +124,9 @@ setvar d.x = 100 pp test_ (rect) pp test_ (d) ## STDOUT: -(Obj) {"x":3,"y":4} +(Obj) ("x":3,"y":4) (Dict) {"x":3,"y":4} -(Obj) {"x":99,"y":4} +(Obj) ("x":99,"y":4) (Dict) {"x":100,"y":4} ## END @@ -145,10 +145,10 @@ setvar rect.x *= 5 pp test_ (rect) ## STDOUT: -(Obj) {"x":3,"y":4} -(Obj) {"x":3,"y":99} -(Obj) {"x":3,"y":102} -(Obj) {"x":15,"y":102} +(Obj) ("x":3,"y":4) +(Obj) ("x":3,"y":99) +(Obj) ("x":3,"y":102) +(Obj) ("x":15,"y":102) ## END #### can't encode objects as JSON @@ -228,5 +228,5 @@ var instance = Object(methods, {foo: 1, bar: 2, x: 3}) pp test_ (instance) ## STDOUT: -(Obj) {"foo":1,"bar":2,"x":3} ==> {"foo":42,"bar":[1,2]} ==> {"foo":"zz"} +(Obj) ("foo":1,"bar":2,"x":3) --> ("foo":42,"bar":[1,2]) --> ("foo":"zz") ## END diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index 130bae5fb5..bef98117c0 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -341,10 +341,10 @@ var two = [obj, obj] pp test_ (two) ## STDOUT: -(Obj) {"z":99} ==> {"__foo__":null} -(Obj) {"z":99,"cycle":{...}} ==> {"__foo__":null} +(Obj) ("z":99) --> ("__foo__":null) +(Obj) ("z":99,"cycle":{...}) --> ("__foo__":null) -(List) [{"z":99,"cycle":{...}} ==> {"__foo__":null},{"z":99,"cycle":{...}} ==> {"__foo__":null}] +(List) [("z":99,"cycle":{...}) --> ("__foo__":null),("z":99,"cycle":{...}) --> ("__foo__":null)] ## END @@ -376,8 +376,10 @@ pp test_ (two) #pp test_ (o2) ## STDOUT: -(Obj) {"z":99} ==> {"__foo__":null} -(List) [{"z":99} ==> {"__foo__":null},{"z":99} ==> {"__foo__":null}] +(Obj) ("z":99) --> ("__foo__":null) +(List) [("z":99) --> ("__foo__":null),("z":99) --> ("__foo__":null)] +(Obj) ("k":42,"cycle":{"k":42,"cycle":{...}}) +(List) [("k":42,"cycle":{"k":42,"cycle":{...}}),("k":42,"cycle":{"k":42,"cycle":{...}})] ## END diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 0d1907b190..467047128c 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -768,7 +768,7 @@ sum = 1 (List) ["a","b",42,43] sum = 5 -(Obj) {"x":2,"y":3} ==> {"__invoke__":} +(Obj) ("x":2,"y":3) --> ("__invoke__":) (List) ["a","b",44,45] ## END From f3aca44f0b0e83153edfb0a29cf0f851432850bf Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 11 Oct 2024 00:13:08 -0400 Subject: [PATCH 316/506] [spec/ysh-printing] Adjust allowed failures --- spec/ysh-printing.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index bef98117c0..c859838563 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 #### Int = -123 From b8ec52527c29fad461379d3e9ec21477078058c8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 11 Oct 2024 00:45:16 -0400 Subject: [PATCH 317/506] [ysh] Remove option redefine_proc_func It inhibits metaprogramming, and is no longer helpful now that we have MODULES with namespaces! I added the "closures in a loop" example to spec/ysh-object, and it tickled this. --- builtin/module_ysh.py | 2 +- core/shell.py | 8 +++--- core/state.py | 10 +++---- doc/ref/chap-option.md | 3 +- frontend/option_def.py | 9 ++---- osh/cmd_eval.py | 34 +---------------------- spec/ysh-func.test.sh | 60 ++++++---------------------------------- spec/ysh-object.test.sh | 29 +++++++++++++++++++ spec/ysh-options.test.sh | 50 +-------------------------------- spec/ysh-proc.test.sh | 5 ++-- spec/ysh-stdlib.test.sh | 2 -- 11 files changed, 54 insertions(+), 158 deletions(-) diff --git a/builtin/module_ysh.py b/builtin/module_ysh.py index 8ed1d1f446..12c745c827 100644 --- a/builtin/module_ysh.py +++ b/builtin/module_ysh.py @@ -51,7 +51,7 @@ def Run(self, cmd_val): #log('guards %s', self.guards) if name in self.guards: # already defined - if self.exec_opts.redefine_module(): + if self.exec_opts.redefine_source(): self.errfmt.PrintMessage( '(interactive) Reloading source file %r' % name) return 0 diff --git a/core/shell.py b/core/shell.py index cb1857102a..1224b058ea 100644 --- a/core/shell.py +++ b/core/shell.py @@ -1022,8 +1022,8 @@ def Main( if flag.headless: state.InitInteractive(mem, lang) - mutable_opts.set_redefine_proc_func() - mutable_opts.set_redefine_module() + mutable_opts.set_redefine_const() + mutable_opts.set_redefine_source() # NOTE: rc files loaded AFTER _InitDefaultCompletions. for rc_path in rc_paths: @@ -1056,8 +1056,8 @@ def Main( state.InitInteractive(mem, lang) # bash: 'set -o emacs' is the default only in the interactive shell mutable_opts.set_emacs() - mutable_opts.set_redefine_proc_func() - mutable_opts.set_redefine_module() + mutable_opts.set_redefine_const() + mutable_opts.set_redefine_source() if readline: term_width = 0 diff --git a/core/state.py b/core/state.py index 9414f8c3b7..96ae7e7357 100644 --- a/core/state.py +++ b/core/state.py @@ -538,15 +538,15 @@ def set_interactive(self): # type: () -> None self._Set(option_i.interactive, True) - def set_redefine_proc_func(self): + def set_redefine_const(self): # type: () -> None """For interactive shells.""" - self._Set(option_i.redefine_proc_func, True) + self._Set(option_i.redefine_const, True) - def set_redefine_module(self): + def set_redefine_source(self): # type: () -> None - """For interactive shells.""" - self._Set(option_i.redefine_module, True) + """For interactive shells. For source-guard""" + self._Set(option_i.redefine_source, True) def set_emacs(self): # type: () -> None diff --git a/doc/ref/chap-option.md b/doc/ref/chap-option.md index f8365cba8a..81e21827ba 100644 --- a/doc/ref/chap-option.md +++ b/doc/ref/chap-option.md @@ -219,8 +219,7 @@ Details on options that are not in `ysh:upgrade` and `strict:all`: In the interactive shell, you can redefine procs and funcs. - redefine_module 'module' builtin always returns 0 - redefine_proc_func (-u) Can shell func, proc and func be redefined? + redefine_source 'source-guard' builtin always returns 0 X redefine_const Can consts be redefined? ### opts-internal diff --git a/frontend/option_def.py b/frontend/option_def.py index 7e5a32ada1..bd52bad619 100644 --- a/frontend/option_def.py +++ b/frontend/option_def.py @@ -127,10 +127,6 @@ def DoneWithImplementedOptions(self): # Whether status 141 in pipelines is turned into 0 ('sigpipe_status_ok', False), - - # This applies to shell functions too - # It's also turned on in interactive mode - ('redefine_proc_func', True), ] # TODO: Add strict_arg_parse? For example, 'trap 1 2 3' shouldn't be @@ -304,9 +300,8 @@ def _Init(opt_def): opt_def.Add('dynamic_scope', default=True) # On in interactive shell - opt_def.Add('redefine_module', default=False) - # Hm these aren't the same? - #opt_def.Add('redefine_proc_func', default=False), + opt_def.Add('redefine_const', default=False) + opt_def.Add('redefine_source', default=False) # For disabling strict_errexit while running traps. Because we run in the # main loop, the value can be "off". Prefix with _ because it's undocumented diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 56fe99a96f..be87a9c17c 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1295,11 +1295,6 @@ def _DoForExpr(self, node): def _DoShFunction(self, node): # type: (command.ShFunction) -> None - existing, _ = self.procs.GetInvokable(node.name) - if existing and not self.exec_opts.redefine_proc_func(): - e_die( - "Function %s was already defined (redefine_proc_func)" % - node.name, node.name_tok) # Note: shell functions can read vars from the file they're defined in # But they don't appear in the module itself -- rather it is __sh_funcs__ @@ -1312,18 +1307,6 @@ def _DoProc(self, node): # type: (Proc) -> None proc_name = lexer.TokenVal(node.name) - # Note: this is similar 'const x = 42' and redefine_const -- it's a - # dynamic check that it doesn't already exist - # Also modules make this less necessary, because there are fewer name - # conflicts - # We could also define procs as READ-ONLY, but that means we need - # Dict[str, Cell] and not Dict[str, value_t] - existing, _ = self.procs.GetInvokable(proc_name) - if existing and not self.exec_opts.redefine_proc_func(): - e_die( - "Proc %s was already defined (redefine_proc_func)" % proc_name, - node.name) - if node.sig.tag() == proc_sig_e.Closed: sig = cast(proc_sig.Closed, node.sig) proc_defaults = func_proc.EvalProcDefaults(self.expr_ev, sig) @@ -1340,27 +1323,12 @@ def _DoFunc(self, node): name = lexer.TokenVal(node.name) lval = location.LName(name) - # Check that we haven't already defined a function - cell = self.mem.GetCell(name, scope_e.LocalOnly) - if cell and cell.val.tag() == value_e.Func: - if self.exec_opts.redefine_proc_func(): - cell.readonly = False # Ensure we can unset the value - did_unset = self.mem.Unset(lval, scope_e.LocalOnly) - assert did_unset, name - else: - e_die( - "Func %s was already defined (redefine_proc_func)" % name, - node.name) - pos_defaults, named_defaults = func_proc.EvalFuncDefaults( self.expr_ev, node) func_val = value.Func(name, node, pos_defaults, named_defaults, self.mem.GlobalFrame()) - self.mem.SetNamed(lval, - func_val, - scope_e.LocalOnly, - flags=state.SetReadOnly) + self.mem.SetNamed(lval, func_val, scope_e.LocalOnly) def _DoIf(self, node): # type: (command.If) -> int diff --git a/spec/ysh-func.test.sh b/spec/ysh-func.test.sh index 326ecb36f4..53c9c9c9f3 100644 --- a/spec/ysh-func.test.sh +++ b/spec/ysh-func.test.sh @@ -136,43 +136,19 @@ proc t() { return (0) } ## STDOUT: ## END -#### Redefining functions is not allowed (with shopt -u redefine_proc_func) -shopt -u redefine_proc_func -func f() { return (0) } -func f() { return (1) } -## status: 1 -## STDOUT: -## END - -#### Redefining functions is allowed (with shopt -s redefine_proc_func) -shopt -s redefine_proc_func +#### Redefining functions is allowed func f() { return (0) } func f() { return (1) } ## status: 0 ## STDOUT: ## END -#### Functions cannot redefine readonly vars (even with shopt -s redefine_proc_func) -shopt -s redefine_proc_func -const f = 0 -func f() { return (1) } -## status: 1 -## STDOUT: -## END - -#### Functions can redefine non-readonly vars +#### Functions can redefine vars var f = 0 func f() { return (1) } -## status: 0 -## STDOUT: -## END - -#### Vars cannot redefine functions (even with shopt -s redefine_proc_func) -shopt -s redefine_proc_func -func f() { return (1) } -const f = 0 -## status: 1 +pp test_ (f) ## STDOUT: + ## END #### Multiple func calls @@ -510,31 +486,11 @@ func inAnotherScope() { } call inAnotherScope() -# We need a scope otherwise we'd overwrite `mysum` in the global scope -var mysum = mysum([1, 2, 3]) # will raise status=1 -## status: 1 +var mysum = mysum([0, 1]) +echo mysum=$mysum + ## STDOUT: 1 + 2 + 3 = 6 mysum=6 -## END - -#### Function names cannot be redeclared -# Behaves like: const f = ... -func f(x) { - return (x) -} - -var f = "some val" -## status: 1 -## STDOUT: -## END - -#### Functions cannot be mutated -func f(x) { - return (x) -} - -setvar f = "some val" -## status: 1 -## STDOUT: +mysum=1 ## END diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index af5aa62e4b..f8f12e0713 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -230,3 +230,32 @@ pp test_ (instance) ## STDOUT: (Obj) ("foo":1,"bar":2,"x":3) --> ("foo":42,"bar":[1,2]) --> ("foo":"zz") ## END + + +#### Closures in a loop idiom + +var procs = [] +for i in (0 .. 3) { + proc __invoke__ (; self) { + echo "i = $[self.i]" + } + var methods = Object(null, {__invoke__}) + var obj = Object(methods, {i}) + call procs->append(obj) +} + +for p in (procs) { + p +} + +# TODO: sugar +# proc p (; self) capture {i} { +# echo "i = $[self.i]" +# } +# call procs->append(p) + +## STDOUT: +i = 0 +i = 1 +i = 2 +## END diff --git a/spec/ysh-options.test.sh b/spec/ysh-options.test.sh index 011648c187..ae10c258b0 100644 --- a/spec/ysh-options.test.sh +++ b/spec/ysh-options.test.sh @@ -179,7 +179,6 @@ shopt -s parse_triple_quote shopt -s parse_ysh_string shopt -s pipefail shopt -s process_sub_fail -shopt -u redefine_proc_func shopt -s sigpipe_status_ok shopt -s simple_word_eval shopt -s verbose_errexit @@ -661,53 +660,6 @@ echo finished finished ## END -#### Shell functions can't be refined with YSH (redefine_proc_func off) - -f() { - echo 1 -} -echo 'first' - -f() { - echo 2 -} -echo 'second' - -shopt --set ysh:upgrade -f() { - echo 3 -} -echo 'third' -## STDOUT: -first -second -## END -## status: 1 - -#### redefine_proc for procs -shopt --set parse_proc - -proc p { - echo 1 -} -echo 'first' - -proc p { - echo 2 -} -echo 'second' - -shopt --set oil:upgrade -proc p { - echo 3 -} -echo 'third' -## STDOUT: -first -second -## END -## status: 1 - #### redefine_proc is on in interactive shell $SH -O oil:all -i --rcfile /dev/null -c " @@ -724,7 +676,7 @@ hi ## END -#### redefine_module is on in interactive shell +#### redefine_source is on in interactive shell $SH -O oil:all -i --rcfile /dev/null -c " source $REPO_ROOT/spec/testdata/module/common.ysh diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 467047128c..61da993bb2 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -267,10 +267,9 @@ g # g is defined in the local scope of f G ## END -#### Procs defined inside compound statements (with redefine_proc) +#### Procs defined inside compound statements shopt --set ysh:upgrade -shopt --set redefine_proc_func for x in 1 2 { proc p { @@ -473,7 +472,7 @@ status=127 ## END #### procs shadow sh-funcs -shopt -s ysh:upgrade redefine_proc_func +shopt -s ysh:upgrade f() { echo sh-func diff --git a/spec/ysh-stdlib.test.sh b/spec/ysh-stdlib.test.sh index cf11f61c85..0c95920c78 100644 --- a/spec/ysh-stdlib.test.sh +++ b/spec/ysh-stdlib.test.sh @@ -18,8 +18,6 @@ status=1 #### smoke test for stream.ysh and table.ysh -shopt --set redefine_proc_func # byo-maybe-main - source $LIB_YSH/stream.ysh source $LIB_YSH/table.ysh From 98b2adacefc030282d3bbbe32db6569b0ab89ee0 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 11 Oct 2024 12:15:07 -0400 Subject: [PATCH 318/506] [ysh] Blocks are bound with the stack frame they're created in Both command literals: p { echo $x } And expression literals: var c = ^( echo $x ) Upcoming work: - Block -> BoundCommand - value.Command vs value.BoundCommand - parseCommand() will return the UNBOUND version - similarly, we probably need value.Expr vs value.BoundExpr - "Closures in a loop" prototype - ctx_FrontFrame or ctx_Enclosing can be used to make the Hay test case work - TODO: write a failing test case for this --- builtin/func_hay.py | 9 ++-- builtin/func_reflect.py | 23 ++++++++-- builtin/method_io.py | 30 ++++++++++--- core/shell.py | 4 +- core/state.py | 18 +++++--- core/value.asdl | 18 ++++---- demo/survey-closure.sh | 73 +++++++++++++++++++++++++++--- frontend/typed_args.py | 55 ++++++++++++++++++++--- osh/cmd_eval.py | 6 ++- spec/ysh-builtin-eval.test.sh | 83 ++++++++++++++++++++++++----------- spec/ysh-builtin-meta.test.sh | 4 +- spec/ysh-proc-meta.test.sh | 21 ++++++++- spec/ysh-proc.test.sh | 2 +- ysh/expr_eval.py | 7 +-- ysh/func_proc.py | 10 +++-- 15 files changed, 278 insertions(+), 85 deletions(-) diff --git a/builtin/func_hay.py b/builtin/func_hay.py index 138dfe9c46..3245b26a30 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -3,7 +3,7 @@ from __future__ import print_function from _devbuild.gen.syntax_asdl import source, loc, command_t -from _devbuild.gen.value_asdl import value +from _devbuild.gen.value_asdl import value, block_val from builtin import hay_ysh from core import alloc from core import error @@ -28,10 +28,11 @@ class ParseHay(vm._Callable): """parseHay()""" - def __init__(self, fd_state, parse_ctx, errfmt): - # type: (process.FdState, parse_lib.ParseContext, ui.ErrorFormatter) -> None + def __init__(self, fd_state, parse_ctx, mem, errfmt): + # type: (process.FdState, parse_lib.ParseContext, state.Mem, ui.ErrorFormatter) -> None self.fd_state = fd_state self.parse_ctx = parse_ctx + self.mem = mem self.errfmt = errfmt def _Call(self, path): @@ -64,7 +65,7 @@ def _Call(self, path): self.errfmt.PrettyPrintError(e) return None - return value.Command(node) + return value.Block(block_val.Expr(node), self.mem.CurrentFrame()) def Call(self, rd): # type: (typed_args.Reader) -> value_t diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index af8fddc823..4a1cf9dfbf 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -6,7 +6,7 @@ from _devbuild.gen.runtime_asdl import (scope_e) from _devbuild.gen.syntax_asdl import source -from _devbuild.gen.value_asdl import (value, value_e, value_t) +from _devbuild.gen.value_asdl import (value, value_e, value_t, block_val) from core import alloc from core import error @@ -113,9 +113,10 @@ def Call(self, rd): class ParseCommand(vm._Callable): - def __init__(self, parse_ctx, errfmt): - # type: (parse_lib.ParseContext, ui.ErrorFormatter) -> None + def __init__(self, parse_ctx, mem, errfmt): + # type: (parse_lib.ParseContext, state.Mem, ui.ErrorFormatter) -> None self.parse_ctx = parse_ctx + self.mem = mem self.errfmt = errfmt def Call(self, rd): @@ -140,7 +141,21 @@ def Call(self, rd): raise error.Structured(3, "Syntax error in parseCommand()", rd.LeftParenToken()) - return value.Command(cmd) + # TODO: It's a little weird that this captures? + # We should have scoping like 'eval $mystr' + # Or we should have + # + # var c = parseCommand('echo hi') # raw AST + # var block = Block(c) # attachs the current frame + # + # Yeah we might need this for value.Expr too, to control evaluation of + # names + # + # value.Expr vs. value.BoundExpr - it's bound to the frame it's defined + # in + # value.Command vs. value.Block - BoundCommand? + + return value.Block(block_val.Expr(cmd), self.mem.CurrentFrame()) class ParseExpr(vm._Callable): diff --git a/builtin/method_io.py b/builtin/method_io.py index ca1bda6f54..c955a5be1d 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -7,12 +7,12 @@ from core import num from core import state from core import vm +from frontend import typed_args from mycpp.mylib import log, NewDict from osh import prompt from typing import Dict, List, cast, TYPE_CHECKING if TYPE_CHECKING: - from frontend import typed_args from osh import cmd_eval _ = log @@ -45,7 +45,18 @@ def __init__(self, cmd_ev, which): def Call(self, rd): # type: (typed_args.Reader) -> value_t unused = rd.PosValue() - cmd = rd.PosCommand() + + # TODO: Can we evaluated both: + # value.BoundCommand + # value.Command (unbound) + #cmd, val = rd.PosCommand2() + + bound = rd.PosBoundCommand() + captured_frame = bound.captured_frame + + cmd = typed_args.GetCommand(bound) + + #log('CAPTURED %r', captured_frame) dollar0 = rd.NamedStr("dollar0", None) pos_args_raw = rd.NamedList("pos_args", None) @@ -64,16 +75,21 @@ def Call(self, rd): pos_args.append(cast(value.Str, arg).s) if self.which == EVAL_NULL: - with state.ctx_Eval(self.cmd_ev.mem, dollar0, pos_args, vars_): - unused_status = self.cmd_ev.EvalCommand(cmd) + # TOOD: don't need bindings + bindings = NewDict() # type: Dict[str, value_t] + with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, + bindings): + with state.ctx_Eval(self.cmd_ev.mem, dollar0, pos_args, vars_): + unused_status = self.cmd_ev.EvalCommand(cmd) return value.Null elif self.which == EVAL_DICT: - # TODO: dollar0, pos_args, vars_ not supposed + # TODO: dollar0, pos_args, vars_ not supported # Does ctx_FrontFrame has different scoping rules? For "vars"? - bindings = NewDict() # type: Dict[str, value_t] - with state.ctx_FrontFrame(self.cmd_ev.mem, bindings): + bindings = NewDict() + with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, + bindings): unused_status = self.cmd_ev.EvalCommand(cmd) return value.Dict(bindings) diff --git a/core/shell.py b/core/shell.py index 1224b058ea..e1264c5e70 100644 --- a/core/shell.py +++ b/core/shell.py @@ -847,7 +847,7 @@ def Main( # Initialize Built-in Funcs # - parse_hay = func_hay.ParseHay(fd_state, parse_ctx, errfmt) + parse_hay = func_hay.ParseHay(fd_state, parse_ctx, mem, errfmt) eval_hay = func_hay.EvalHay(hay_state, mutable_opts, mem, cmd_ev) hay_func = func_hay.HayFunc(hay_state) @@ -868,7 +868,7 @@ def Main( _AddBuiltinFunc(mem, 'id', func_reflect.Id()) _AddBuiltinFunc(mem, 'parseCommand', - func_reflect.ParseCommand(parse_ctx, errfmt)) + func_reflect.ParseCommand(parse_ctx, mem, errfmt)) _AddBuiltinFunc(mem, 'parseExpr', func_reflect.ParseExpr(parse_ctx, errfmt)) _AddBuiltinFunc(mem, 'evalExpr', func_reflect.EvalExpr(expr_ev)) diff --git a/core/state.py b/core/state.py index 96ae7e7357..a63a5b2dc5 100644 --- a/core/state.py +++ b/core/state.py @@ -1174,19 +1174,18 @@ class ctx_FrontFrame(object): Or maybe we disallow the setvar lookup? """ - def __init__(self, mem, out_dict): - # type: (Mem, Dict[str, value_t]) -> None + def __init__(self, mem, rear_frame, out_dict): + # type: (Mem, Dict[str, Cell], Dict[str, value_t]) -> None self.mem = mem + self.rear_frame = rear_frame self.out_dict = out_dict - self.rear_frame = mem.var_stack[-1] - # __rear__ gets a lookup rule self.front_frame = NewDict() # type: Dict[str, Cell] self.front_frame['__rear__'] = Cell(False, False, False, - value.Frame(self.rear_frame)) + value.Frame(rear_frame)) - mem.var_stack[-1] = self.front_frame + mem.var_stack.append(self.front_frame) def __enter__(self): # type: () -> None @@ -1207,7 +1206,7 @@ def __exit__(self, type, value, traceback): self.out_dict[name] = cell.val # Restore - self.mem.var_stack[-1] = self.rear_frame + self.mem.var_stack.pop() class ctx_ModuleEval(object): @@ -1686,6 +1685,11 @@ def GlobalFrame(self): """ return self.var_stack[0] + def CurrentFrame(self): + # type: () -> Dict[str, Cell] + """For attaching a stack frame to a value.Block""" + return self.var_stack[-1] + def PushSource(self, source_name, argv): # type: (str, List[str]) -> None """ For 'source foo.sh 1 2 3' """ diff --git a/core/value.asdl b/core/value.asdl index 6ea31c088c..b0cfc6a068 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -58,6 +58,15 @@ module value No | Yes %RegexMatch + # TODO: + # - Consolidate value.Command and value.LiteralBlock. All Block instances + # should have backing lines. + # - use LiteralBlock %LiteralBlock, but ASDL doesn't support shared variants + # across files. + block_val = + Literal(LiteralBlock b) # p { echo hi } has backing lines + | Expr(command c) # var b = ^(echo hi) + # Arbitrary objects, where attributes are looked up on the prototype chain. Obj = (Obj? prototype, Dict[str, value] d) @@ -121,14 +130,7 @@ module value # ^(echo 1; echo 2) and cd { echo 1; echo 2 } | Command(command c) - # for Hay to get the backing lines - # TODO: Consolidate value.Command and value.LiteralBlock. All Command - # instance should have backing lines. - - # TODO: ASDL doesn't support shared variant across module - # This would be more efficient - # | LiteralBlock %LiteralBlock - | Block(LiteralBlock block) + | Block(block_val block, Dict[str, Cell] captured_frame) # A place has an additional stack frame where the value is evaluated. # The frame MUST be lower on the stack at the time of use. diff --git a/demo/survey-closure.sh b/demo/survey-closure.sh index a07511b02e..869bd744ac 100755 --- a/demo/survey-closure.sh +++ b/demo/survey-closure.sh @@ -114,18 +114,12 @@ loops() { echo 'LOOPS PYTHON' echo - # I think this is the thing that Go and C# changed! - # Gah - # # We would have to test multiple blocks in a loop # # for i in (0 .. 3) { # cd /tmp { # this will work # echo $i # } - # - # var b = ^(echo $i) - # call blocks->append(b) # won't work # } python3 -c ' @@ -147,6 +141,73 @@ print(functions[2]()) ' } +js-while-var() { + echo 'WHILE JS' + echo + + nodejs -e ' + function createFunctions() { + const funcs = []; + let i = 0; // for let is SPECIAL! + while (i < 3) { + funcs.push(function() { return i; }); + i++; + } + return funcs; + } + + const functions = createFunctions(); + + console.log(functions[0]()) + console.log(functions[1]()) + console.log(functions[2]()) + ' + + echo 'FOR VAR JS' + echo + + nodejs -e ' + function createFunctions() { + const funcs = []; + // var is not captured + for (var i = 0; i < 3; i++) { + funcs.push(function() { return i; }); + } + return funcs; + } + + const functions = createFunctions(); + + console.log(functions[0]()) + console.log(functions[1]()) + console.log(functions[2]()) + ' + + echo 'FOR LET' + echo + + nodejs -e ' + function createFunctions() { + const funcs = []; + for (let i = 0; i < 3; i++) { + // This is captured + // let j = i + 10; + + // This is not captured, I guess it is "hoisted" + var j = i + 10; + funcs.push(function() { return j; }); + } + return funcs; + } + + const functions = createFunctions(); + + console.log(functions[0]()) + console.log(functions[1]()) + console.log(functions[2]()) + ' +} + nested() { echo 'NESTED JS' echo diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 387350fcb4..af1f26aeff 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -4,13 +4,14 @@ from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, LiteralBlock, command_t, expr_t, Token) -from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, Obj) +from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, Obj, + block_val, block_val_e, block_val_str) from core import error from core.error import e_usage from frontend import location from mycpp import mops from mycpp import mylib -from mycpp.mylib import log +from mycpp.mylib import log, tagswitch from typing import Dict, List, Optional, cast @@ -47,6 +48,21 @@ def OptionalLiteralBlock(cmd_val): return block +def GetCommand(bound): + # type: (value.Block) -> command_t + + block = bound.block + with tagswitch(block) as case: + if case(block_val_e.Literal): + lit = cast(block_val.Literal, block) + return lit.b.brace_group + elif case(block_val_e.Expr): + expr = cast(block_val.Expr, block) + return expr.c + else: + raise AssertionError(block_val_str(block.tag())) + + def ReaderForProc(cmd_val): # type: (cmd_value.Argv) -> Reader @@ -325,9 +341,10 @@ def _ToCommand(self, val): if val.tag() == value_e.Command: return cast(value.Command, val).c - # eval (myblock) uses this + # io.eval(mycmd) uses this if val.tag() == value_e.Block: - return cast(value.Block, val).block.brace_group + bound = cast(value.Block, val) + return GetCommand(bound) raise error.TypeErr(val, 'Arg %d should be a Command' % self.pos_consumed, @@ -341,16 +358,32 @@ def _ToBlock(self, val): # Special case for hay # Foo { x = 1 } if val.tag() == value_e.Block: - return cast(value.Block, val).block.brace_group + bound = cast(value.Block, val) + return GetCommand(bound) raise error.TypeErr(val, - 'Arg %d should be a Command' % self.pos_consumed, + 'Arg %d should be a Block' % self.pos_consumed, self.BlamePos()) + def _ToBoundCommand(self, val): + # type: (value_t) -> value.Block + if val.tag() == value_e.Block: + return cast(value.Block, val) + raise error.TypeErr( + val, 'Arg %d should be a BoundCommand' % self.pos_consumed, + self.BlamePos()) + def _ToLiteralBlock(self, val): # type: (value_t) -> LiteralBlock + """ Used by Hay """ if val.tag() == value_e.Block: - return cast(value.Block, val).block + block = cast(value.Block, val).block + with tagswitch(block) as case: + if case(block_val_e.Literal): + lit = cast(block_val.Literal, block) + return lit.b + else: + raise AssertionError() raise error.TypeErr( val, 'Arg %d should be a LiteralBlock' % self.pos_consumed, @@ -435,6 +468,11 @@ def PosCommand(self): val = self.PosValue() return self._ToCommand(val) + def PosBoundCommand(self): + # type: () -> value.Block + val = self.PosValue() + return self._ToBoundCommand(val) + def PosExpr(self): # type: () -> expr_t val = self.PosValue() @@ -459,6 +497,9 @@ def OptionalBlock(self): def OptionalLiteralBlock(self): # type: () -> Optional[LiteralBlock] + """ + Used by Hay + """ if self.block_arg is None: return None return self._ToLiteralBlock(self.block_arg) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index be87a9c17c..4279a15ce3 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -810,8 +810,10 @@ def _DoSimple(self, node, cmd_st): if node.typed_args or node.block: # guard to avoid allocs cmd_val.proc_args = ProcArgs(node.typed_args, None, None, None) - func_proc.EvalTypedArgsToProc(self.expr_ev, self.mutable_opts, - node, cmd_val.proc_args) + func_proc.EvalTypedArgsToProc(self.expr_ev, + self.mem.CurrentFrame(), + self.mutable_opts, node, + cmd_val.proc_args) else: if node.block: e_die("ShAssignment builtins don't accept blocks", diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 7a415854cb..b372f57e20 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 5 +## oils_failures_allowed: 3 #### eval builtin does not take a literal block - can restore this later @@ -53,6 +53,8 @@ call io->eval(my_block) #### io->eval(block) can read variables like eval '' +# NO LONGER WORKS, but is this a feature rather than a bug? + proc p2(code_str) { var mylocal = 42 eval $code_str @@ -279,6 +281,7 @@ call io->eval(^(true), pos_args=[1, 2, 3]) ## status: 3 #### eval with vars follows same scoping as without + proc local-scope { var myVar = "foo" call io->eval(^(echo $myVar), vars={ someOtherVar: "bar" }) @@ -349,6 +352,7 @@ pp test_ (d) # Same thing in a local frame proc p (myparam) { var mylocal = 'local' + # TODO: ^() needs to capture var cmd = ^( var foo = 42 var g = "-$g" @@ -379,7 +383,7 @@ var d = io->evalToDict(cmd) pp test_ (d) ## STDOUT: - + hi (Dict) {"x":42,"y":"global"} ## END @@ -410,13 +414,18 @@ var k = 'k-shadowed' var k2 = 'k2-shadowed' Dict (&d) { + bare = 42 + + # uh these find the wrong one + # This is like redeclaring the one above, but WITHOUT the static error + # HM HM HM var k = 'k-block' setvar k = 'k-block-mutated' - # this is confusing - # because it doesn't find it in the local stack frame - # it doesn't have 'var without setvar' bug - setvar k2 = 'k2-block' # global, so not checked + # Finds the global, so not checked + setvar k2 = 'k2-block' + + # This one is allowed setvar k3 = 'k3' # do we allow this? @@ -425,6 +434,8 @@ Dict (&d) { pp test_ (d) +exit + # restored to the shadowed values echo k=$k echo k2=$k2 @@ -434,8 +445,8 @@ proc p { var k = 'k-proc' setvar k = 'k-proc-mutated' - # is this in the dict? - setvar k2 = 'k2-proc' # local, so it's checked + # Not allowed STATICALLY, because o fproc check + #setvar k2 = 'k2-proc' # local, so it's checked } } @@ -463,6 +474,7 @@ var mydict = f() pp test_ (mydict) ## STDOUT: +(Dict) {"y":43} ## END #### block in yb-capture Dict (&d) can read from outer scope @@ -494,18 +506,19 @@ var result = f() pp test_ (result) ## STDOUT: +(Dict) {"status":0,"stdout":"43\n"} ## END #### Dict (&d) and setvar proc Dict ( ; out; ; block) { + echo "Dict proc global outer=$outer" var d = io->evalToDict(block) - echo 'proc Dict frame after evalToDict' - pp frame_vars_ + #echo 'proc Dict frame after evalToDict' + #pp frame_vars_ - echo "Dict outer=$outer" #echo "Dict outer2=$outer2" call out->setValue(d) } @@ -516,33 +529,51 @@ Dict (&d) { # new variable in the front frame outer2 = 'outer2' - #var v = 'v' - #setvar v = 'v-mutated' - - # hm setvar is local ONLY, so it does NOT find the 'outer' - # because we're inside Dict! Gah - # - # Do we want to say there's no matching 'var', instead of mutating locally? - # - # And also plain io->eval() should be able to mutate outer... + echo "inside Dict outer=$outer" setvar outer = 'zz' setvar not_declared = 'yy' - echo 'inside Dict block' - pp frame_vars_ + #echo 'inside Dict block' + #pp frame_vars_ } pp test_ (d) -echo after outer=$outer +echo "after Dict outer=$outer" + +echo + -echo 'after Dict' -pp frame_vars_ +# Now do the same thing inside a proc + +proc p { + var outer = 'p-outer' + + Dict (&d) { + p = 99 + setvar outer = 'p-outer-mutated' + } + + pp test_ (d) + echo "[p] after Dict outer=$outer" +} + +p + +echo "after p outer=$outer" ## STDOUT: +Dict proc global outer=xx +inside Dict outer=xx +(Dict) {"outer2":"outer2","not_declared":"yy"} +after Dict outer=zz + +Dict proc global outer=zz +(Dict) {"p":99} +[p] after Dict outer=p-outer-mutated +after p outer=zz ## END - #### Dict (&d) and setglobal proc Dict ( ; out; ; block) { diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index ba8c95b3dd..b82c48ee3d 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -75,13 +75,13 @@ ty (Str) "a" (Int) 42 (Int) 99 -Command +Block ty (Str) "a" (Int) 42 (Int) 99 -Command +Block ty (Str) "a" diff --git a/spec/ysh-proc-meta.test.sh b/spec/ysh-proc-meta.test.sh index 1af0c8b663..66fea8174d 100644 --- a/spec/ysh-proc-meta.test.sh +++ b/spec/ysh-proc-meta.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 0 +## oils_failures_allowed: 1 ## our_shell: ysh # dynamically generate procs @@ -99,7 +99,24 @@ proc p { } """ var cmd = parseCommand(s) + #pp test_ (cmd) + pp asdl_ (cmd) + + # Oh so then echo_a is defined in the front frame + # And then the front frame is discarded? + # + # OK I see + # + # So you only use evalToDict()? + # + # Or parseCommand() returns something UNBOUND, so it has the same power + # as eval $mystr + call io->eval(cmd) + + #call io->evalToDict(cmd) + #pp (echo_a) + echo_a zz } echo_a prefix @@ -108,7 +125,7 @@ proc p { p -echo_a prefix +echo_a not_defined ## status: 127 ## STDOUT: diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 61da993bb2..736b1629f2 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -352,7 +352,7 @@ Block (List) ["a","b"] (List) ["c","d"] (Dict) {"n":99} -Command +Block ## END diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index a4335f17ee..3d34ab4973 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -46,7 +46,7 @@ ) from _devbuild.gen.value_asdl import (value, value_e, value_t, y_lvalue, y_lvalue_e, y_lvalue_t, IntBox, LeftName, - Obj) + Obj, block_val) from core import error from core.error import e_die, e_die_status from core import num @@ -1145,8 +1145,9 @@ def _EvalExpr(self, node): id_ = node.left_token.id if id_ == Id.Left_CaretParen: # ^(echo block literal) - # TODO: Propgate location info? - return value.Command(node.child) + # TODO: Propagate location info with ^( + return value.Block(block_val.Expr(node.child), + self.mem.CurrentFrame()) else: stdout_str = self.shell_ex.RunCommandSub(node) if id_ == Id.Left_AtParen: # @(seq 3) diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 7f66c85020..5538e3a237 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -5,12 +5,12 @@ from __future__ import print_function from _devbuild.gen.id_kind_asdl import Id -from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs +from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs, Cell from _devbuild.gen.syntax_asdl import (proc_sig, proc_sig_e, Param, ParamGroup, NamedArg, Func, loc, ArgList, expr, expr_e, expr_t) from _devbuild.gen.value_asdl import (value, value_e, value_t, ProcDefaults, - LeftName) + LeftName, block_val) from core import error from core.error import e_die @@ -209,6 +209,7 @@ def _EvalArgList( def EvalTypedArgsToProc( expr_ev, # type: expr_eval.ExprEvaluator + current_frame, # type: Dict[str, Cell] mutable_opts, # type: state.MutableOpts node, # type: command.Simple proc_args, # type: ProcArgs @@ -260,8 +261,9 @@ def EvalTypedArgsToProc( # p { echo hi } is an unevaluated block if node.block: - # TODO: conslidate value.Block (holds LiteralBlock) and value.Command - proc_args.block_arg = value.Block(node.block) + # Attach current frame to value.Block + proc_args.block_arg = value.Block(block_val.Literal(node.block), + current_frame) # Add location info so the cmd_val looks the same for both: # cd /tmp (; ; ^(echo hi)) From 78b56cfb1a99961ba22b29c322daf921df7edc48 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sat, 12 Oct 2024 13:08:21 -0400 Subject: [PATCH 319/506] [spec/ysh-builtin-eval] Failing test case for closures in a loop test/ysh-runtime-errors - Fix test with default. Need a SPEC TEST for this too --- spec/ysh-builtin-eval.test.sh | 26 +++++++++++++++++++++++++- ysh/func_proc.py | 5 ++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index b372f57e20..6f21f3db1a 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 3 +## oils_failures_allowed: 4 #### eval builtin does not take a literal block - can restore this later @@ -621,3 +621,27 @@ a=a inner=z inner2=z ## END + +#### Block Closures in a Loop ! + +proc task (; tasks; ; b) { + call tasks->append(b) +} + +func makeTasks() { + var tasks = [] + for i in (0 .. 3) { + task (tasks) { echo "i = $i" } + } + return (tasks) +} + +var blocks = makeTasks() +#= blocks + +for b in (blocks) { + call io->eval(b) +} + +## STDOUT: +## END diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 5538e3a237..5d75092313 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -124,7 +124,10 @@ def EvalProcDefaults(expr_ev, sig): if exp: block_default = expr_ev.EvalExpr(exp, sig.block_param.blame_tok) # It can only be ^() or null - if block_default.tag() not in (value_e.Null, value_e.Command): + if block_default.tag() not in (value_e.Null, value_e.Block): + + # TODO: This is a value.Command, not a value.BoundCommand/Block? + raise error.TypeErr( block_default, "Default value for block should be Command or Null", From 553fd0ce61fd25910cb813ab336fe0903b282a0e Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 01:08:07 -0400 Subject: [PATCH 320/506] [stdlib] Take advantage of fixed block scoping The block passed to yb-capture can now variables outside the block. --- stdlib/ysh/args-test.ysh | 98 +++++++++++++++++++++------------------- stdlib/ysh/yblocks.ysh | 2 +- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/stdlib/ysh/args-test.ysh b/stdlib/ysh/args-test.ysh index 86ee7bf266..b9c6be6c5c 100755 --- a/stdlib/ysh/args-test.ysh +++ b/stdlib/ysh/args-test.ysh @@ -7,7 +7,11 @@ use $LIB_YSH/args.ysh --pick parser flag arg rest parseArgs source $LIB_YSH/yblocks.ysh -# Change to 'use'? +# TODO: why doesn't this work? Is there a buffering problem wtih read --all? +# Why would it not show up with source though? +#use $LIB_YSH/yblocks.ysh --pick yb-capture + +# Can't be 'use' because we're using shell functions? source $LIB_OSH/byo-server.sh proc test-basic { @@ -77,20 +81,20 @@ proc test-default-values { } proc test-multiple-argv-arrays { - yb-capture (&r) { - parser (&spec) { - flag -v --verbose ('bool', default=false) - flag -c --count ('int', default=120) - arg file - } + parser (&spec) { + flag -v --verbose ('bool', default=false) + flag -c --count ('int', default=120) + arg file + } - # TODO: argCases should go above - var argsCases = [ - :| -v --count 120 example.sh |, - :| -v --count 120 example.sh -v |, # duplicate flags are ignored - :| -v --count 120 example.sh -v --count 150 |, # the last duplicate has precedence - ] + # TODO: argCases should go above + var argsCases = [ + :| -v --count 120 example.sh |, + :| -v --count 120 example.sh -v |, # duplicate flags are ignored + :| -v --count 120 example.sh -v --count 150 |, # the last duplicate has precedence + ] + yb-capture (&r) { for args in (argsCases) { var args_str = join(args, ' ') echo "---------- $args_str ----------" @@ -179,15 +183,15 @@ proc test-more-errors { proc test-print-spec { - yb-capture (&r) { - parser (&spec) { - flag -v --verbose ('bool') - arg src - arg dst + parser (&spec) { + flag -v --verbose ('bool') + arg src + arg dst - rest more # allow more args - } + rest more # allow more args + } + yb-capture (&r) { json write (spec) } @@ -221,38 +225,38 @@ proc test-print-spec { } proc test-vs-python3-argparse { - yb-capture (&r) { - var spec = { - flags: [ - {short: '-v', long: '--verbose', name: 'verbose', type: null, default: '', help: 'Enable verbose logging'}, - {short: '-c', long: '--count', name: 'count', type: 'int', default: 80, help: 'Maximum line length'}, - ], - args: [ - {name: 'file', type: 'str', help: 'File to check line lengths of'} - ], - rest: null, - } + var spec = { + flags: [ + {short: '-v', long: '--verbose', name: 'verbose', type: null, default: '', help: 'Enable verbose logging'}, + {short: '-c', long: '--count', name: 'count', type: 'int', default: 80, help: 'Maximum line length'}, + ], + args: [ + {name: 'file', type: 'str', help: 'File to check line lengths of'} + ], + rest: null, + } - var argsCases = [ - :| -v --count 120 example.sh |, - :| -v --count 120 example.sh -v |, # duplicate flags are ignored - :| -v --count 120 example.sh -v --count 150 |, # the last duplicate has precedence - ] + var argsCases = [ + :| -v --count 120 example.sh |, + :| -v --count 120 example.sh -v |, # duplicate flags are ignored + :| -v --count 120 example.sh -v --count 150 |, # the last duplicate has precedence + ] - var argparse_py = ''' - import argparse - import sys + var argparse_py = ''' + import argparse + import sys - spec = argparse.ArgumentParser() - spec.add_argument("filename") - spec.add_argument("-c", "--count", type=int) - spec.add_argument("-v", "--verbose", - action="store_true") + spec = argparse.ArgumentParser() + spec.add_argument("filename") + spec.add_argument("-c", "--count", type=int) + spec.add_argument("-v", "--verbose", + action="store_true") - result = spec.parse_args(sys.argv[1:]) - print([result.filename, result.count, result.verbose]) - ''' + result = spec.parse_args(sys.argv[1:]) + print([result.filename, result.count, result.verbose]) + ''' + yb-capture (&r) { for args in (argsCases) { var args_str = args => join(" ") echo "---------- $args_str ----------" diff --git a/stdlib/ysh/yblocks.ysh b/stdlib/ysh/yblocks.ysh index a218a1fa9e..e8a2754750 100644 --- a/stdlib/ysh/yblocks.ysh +++ b/stdlib/ysh/yblocks.ysh @@ -4,7 +4,7 @@ # # Capture status/stdout/stderr, and nq-assert those values. -#module yblocks || return 0 +const __provide__ = :| yb-capture yb-capture-2 | : ${LIB_OSH=stdlib/osh} source $LIB_OSH/two.sh From 399994495d7b3db30744557d3f3b6de7d5f27ac5 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 01:21:27 -0400 Subject: [PATCH 321/506] [ysh demo] Proof of concept for "closures in a loop" problem This was pretty easy - just create a new front frame/rear frame. And fix the rear frame lookup to be recurisve. This is better known as a Scheme-like environment with a persistent data structure. TODO: - This is hidden behind a '__hack__' - I think we should do some analysis in the parser to selectively enable it. - Write a benchmark like fibonacci to measure the difference between the Python-style mutation behavior and the new binding --- builtin/method_io.py | 16 +++++++++++ core/state.py | 37 ++++++++++++++++++++----- core/value.asdl | 2 ++ osh/cmd_eval.py | 51 ++++++++++++++++++----------------- spec/ysh-builtin-eval.test.sh | 12 ++++++--- 5 files changed, 84 insertions(+), 34 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index c955a5be1d..ca9866c3d8 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -14,6 +14,7 @@ from typing import Dict, List, cast, TYPE_CHECKING if TYPE_CHECKING: from osh import cmd_eval + from _devbuild.gen.runtime_asdl import Cell _ = log @@ -21,6 +22,18 @@ EVAL_DICT = 2 +def _PrintFrame(prefix, frame): + # type: (str, Dict[str, Cell]) -> None + print('%s %s' % (prefix, ' '.join(frame.keys()))) + + rear = frame.get('__rear__') + if rear: + rear_val = rear.val + if rear_val.tag() == value_e.Frame: + r = cast(value.Frame, rear_val) + _PrintFrame('--> ' + prefix, r.frame) + + class Eval(vm._Callable): """ These are similar: @@ -75,10 +88,13 @@ def Call(self, rd): pos_args.append(cast(value.Str, arg).s) if self.which == EVAL_NULL: + # _PrintFrame('[captured]', captured_frame) + # TOOD: don't need bindings bindings = NewDict() # type: Dict[str, value_t] with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, bindings): + # _PrintFrame('[new]', self.cmd_ev.mem.var_stack[-1]) with state.ctx_Eval(self.cmd_ev.mem, dollar0, pos_args, vars_): unused_status = self.cmd_ev.EvalCommand(cmd) return value.Null diff --git a/core/state.py b/core/state.py index a63a5b2dc5..a280938180 100644 --- a/core/state.py +++ b/core/state.py @@ -1154,6 +1154,31 @@ def _MakeArgvCell(argv): return Cell(False, False, False, value.List(items)) +class ctx_LoopFrame(object): + + def __init__(self, mem, name1): + # type: (Mem, str) -> None + self.mem = mem + self.name1 = name1 + self.do_new_frame = name1 == '__hack__' + + if self.do_new_frame: + rear_frame = self.mem.var_stack[-1] + self.front_frame = NewDict() # type: Dict[str, Cell] + self.front_frame['__rear__'] = Cell(False, False, False, + value.Frame(rear_frame)) + mem.var_stack.append(self.front_frame) + + def __enter__(self): + # type: () -> None + pass + + def __exit__(self, type, value, traceback): + # type: (Any, Any, Any) -> None + if self.do_new_frame: + self.mem.var_stack.pop() + + class ctx_FrontFrame(object): """ For use by io->evalToDict(), which is a primitive used for Hay and the Dict @@ -1353,6 +1378,8 @@ def _FrameLookup(frame, name): # type: (Dict[str, Cell], str) -> Tuple[Optional[Cell], Dict[str, Cell]] """ Look in the frame itself, then the __rear__ frame if it exists + + TODO: Need to recursively look at __rear__ """ cell = frame.get(name) if cell: @@ -1361,12 +1388,10 @@ def _FrameLookup(frame, name): rear_cell = frame.get('__rear__') # ctx_FrontFrame() sets this if rear_cell: rear_val = rear_cell.val - if rear_val and rear_val.tag() == value_e.Frame: - frame = cast(value.Frame, rear_val).frame - cell = frame.get(name) - if cell: - #return cell, frame - return cell, None + assert rear_val, rear_val + if rear_val.tag() == value_e.Frame: + rear_frame = cast(value.Frame, rear_val).frame + return _FrameLookup(rear_frame, name) # recursive call return None, None diff --git a/core/value.asdl b/core/value.asdl index b0cfc6a068..500481bfdf 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -127,9 +127,11 @@ module value # ^[42 + a[i]] | Expr(expr e) + # TODO: this is an UnboundCommand? # ^(echo 1; echo 2) and cd { echo 1; echo 2 } | Command(command c) + # TODO: this is BoundCommand | Block(block_val block, Dict[str, Cell] captured_frame) # A place has an additional stack frame where the value is evaluated. diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 4279a15ce3..a5b33c0d4b 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -1231,35 +1231,36 @@ def _DoForEach(self, node): status = 0 # in case we loop zero times with ctx_LoopLevel(self): while True: - first = it2.FirstValue() - #log('first %s', first) - if first is None: # for StdinIterator - #log('first is None') - break + with state.ctx_LoopFrame(self.mem, name1.name): + first = it2.FirstValue() + #log('first %s', first) + if first is None: # for StdinIterator + #log('first is None') + break - if first.tag() == value_e.Interrupted: - self.RunPendingTraps() - #log('Done running traps') - continue + if first.tag() == value_e.Interrupted: + self.RunPendingTraps() + #log('Done running traps') + continue - self.mem.SetLocalName(name1, first) - if name2: - self.mem.SetLocalName(name2, it2.SecondValue()) - if i_name: - self.mem.SetLocalName(i_name, num.ToBig(it2.Index())) + self.mem.SetLocalName(name1, first) + if name2: + self.mem.SetLocalName(name2, it2.SecondValue()) + if i_name: + self.mem.SetLocalName(i_name, num.ToBig(it2.Index())) - # increment index before handling continue, etc. - it2.Next() + # increment index before handling continue, etc. + it2.Next() - try: - status = self._Execute(node.body) # last one wins - except vm.IntControlFlow as e: - status = 0 - action = e.HandleLoop() - if action == flow_e.Break: - break - elif action == flow_e.Raise: - raise + try: + status = self._Execute(node.body) # last one wins + except vm.IntControlFlow as e: + status = 0 + action = e.HandleLoop() + if action == flow_e.Break: + break + elif action == flow_e.Raise: + raise return status diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 6f21f3db1a..d9cb35ec7c 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 4 +## oils_failures_allowed: 3 #### eval builtin does not take a literal block - can restore this later @@ -630,8 +630,11 @@ proc task (; tasks; ; b) { func makeTasks() { var tasks = [] - for i in (0 .. 3) { - task (tasks) { echo "i = $i" } + var x = 'x' + for __hack__ in (0 .. 3) { + var i = __hack__ + var j = i + 2 + task (tasks) { echo "$x: i = $i, j = $j" } } return (tasks) } @@ -644,4 +647,7 @@ for b in (blocks) { } ## STDOUT: +x: i = 0, j = 2 +x: i = 1, j = 3 +x: i = 2, j = 4 ## END From 217a846e0af7fac1d3814669f59b2ffcc96ac8a6 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 16:20:42 -0400 Subject: [PATCH 322/506] [benchmarks] Benchmark for "closures in a loop" Surprisingly, it's not that much slower. I'm seeing 630 ms with mutation, and 756 ms with closures. There are many more allocations, but the overall time isn't bad. Also, YSH idioms are much faster than OSH, and OSH is faster than bash! --- benchmarks/ysh-for.sh | 96 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100755 benchmarks/ysh-for.sh diff --git a/benchmarks/ysh-for.sh b/benchmarks/ysh-for.sh new file mode 100755 index 0000000000..95f15f6fa6 --- /dev/null +++ b/benchmarks/ysh-for.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# Benchmarks for YSH for loop +# +# Usage: +# benchmarks/ysh-for.sh + +set -o nounset +set -o pipefail +set -o errexit + +YSH=_bin/cxx-opt/ysh +OSH=_bin/cxx-opt/osh + +sum() { + echo " YSH for loop" + + time $YSH -c ' + var n = int($1) + var sum = 0 + for i in (0 .. n) { + setvar sum += i + } + echo "i = $i" + echo "sum = $sum" + ' dummy "$@" +} + +sum-closures() { + echo " YSH closures" + + time $YSH -c ' + var n = int($1) + var sum = 0 + for __hack__ in (0 .. n) { # trigger allocation + setvar sum += __hack__ + } + # Does not leak! + #echo "__hack__ = $__hack__" + echo "sum = $sum" + ' dummy "$@" +} + +sum-py() { + echo ' PY' + time python3 -c ' +import sys +n = int(sys.argv[1]) +sum = 0 +for i in range(n): + sum += i +print(f"sum = {sum}") + ' "$@" +} + +sum-sh() { + local sh=$1 + local n=$2 + + echo " $sh" + time $sh -c ' +n=$1 +sum=0 +i=0 +while test $i -lt $n; do + sum=$(( sum + i )) + i=$(( i + 1 )) +done +echo "sum = $sum" + ' "$@" +} + +compare() { + local n=${1:-1000000} + local OILS_GC_STATS=${2:-} + + ninja $OSH $YSH + + sum-py $n + echo + + sum-sh bash $n + echo + + sum-sh $OSH $n + echo + + export OILS_GC_STATS + sum $n + echo + + sum-closures $n + echo +} + +"$@" From 25202d1901598a7be099fcd65b1db85db15b71f1 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 20:19:29 -0400 Subject: [PATCH 323/506] [demo/survey-closure] Add Ruby examples It has many styles! --- demo/survey-closure.sh | 121 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/demo/survey-closure.sh b/demo/survey-closure.sh index 869bd744ac..4bb14b80ef 100755 --- a/demo/survey-closure.sh +++ b/demo/survey-closure.sh @@ -373,4 +373,125 @@ print(g()) # - Python didn't change it, but people mostly write blog posts about it, and # don't hit it? + +ruby-blocks() { + ruby -e ' +def create_multiplier(factor) + ->(x) { x * factor } +end + +double = create_multiplier(2) +triple = create_multiplier(3) + +puts double.call(5) # Output: 10 +puts triple.call(5) # Output: 15 +' + echo + + ruby -e ' +def use_multiplier(factor) + # This method yields to a block + yield factor +end + +multiplier = 3 + +# The block captures the outer multiplier variable +result = use_multiplier(5) { |x| x * multiplier } +puts result # Output: 15 + +# alternative syntax +result = use_multiplier(5) do |x| + x * multiplier +end + +puts result # Output: 15 +' + echo + + ruby -e ' +# alternative syntax +def use_multiplier(factor, &block) + block.call(factor) +end + +multiplier = 3 + +result = use_multiplier(5) { |x| x * multiplier } +puts result # Output: 15 + +# alterantive syntax +result = use_multiplier(5) do |x| + x * multiplier +end + +puts result # Output: 15 +' +} + +ruby-mine() { + ruby -e ' +# Two styles + +# Implicit block arg +def run_it + yield 2 +end + +# explicit proc arg +def run_it2 (&block) # interchangeable + block.call(2) +end + +# 2 Styles of Block + +factor = 3 + +block1 = ->(x) { x * factor } +puts block1.call(5) + +result = run_it(&block1) +puts result + +puts + +block2 = lambda do |x| + x * factor +end +puts block2.call(5) + +result = run_it(&block2) +puts result + +puts + +# 2 styles of Proc + +proc1 = proc { |x| x * factor } +puts proc1.call(5) + +result = run_it(&proc1) +puts result + +puts + +proc2 = Proc.new do |x| + x * factor +end +puts proc2.call(5) + +result = run_it(&proc2) +puts result + +puts + +# Now do a literal style + +result = run_it do |x| + x * factor +end +puts result +' +} + "$@" From 71917d90c69e26755cc1f1e0aac85f36c1bea866 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 20:41:01 -0400 Subject: [PATCH 324/506] [ASDL refactor] Move LiteralBlock from syntax.asdl -> value.asdl Work around "shared variant" issue --- builtin/func_hay.py | 4 ++-- builtin/func_reflect.py | 4 ++-- core/runtime.asdl | 2 +- core/value.asdl | 10 +++++++--- frontend/syntax.asdl | 5 +---- frontend/typed_args.py | 21 +++++++++++---------- osh/cmd_parse.py | 2 +- ysh/expr_eval.py | 4 ++-- ysh/func_proc.py | 4 ++-- 9 files changed, 29 insertions(+), 27 deletions(-) diff --git a/builtin/func_hay.py b/builtin/func_hay.py index 3245b26a30..960926a3ae 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -3,7 +3,7 @@ from __future__ import print_function from _devbuild.gen.syntax_asdl import source, loc, command_t -from _devbuild.gen.value_asdl import value, block_val +from _devbuild.gen.value_asdl import value, cmd_frag from builtin import hay_ysh from core import alloc from core import error @@ -65,7 +65,7 @@ def _Call(self, path): self.errfmt.PrettyPrintError(e) return None - return value.Block(block_val.Expr(node), self.mem.CurrentFrame()) + return value.Block(cmd_frag.Expr(node), self.mem.CurrentFrame()) def Call(self, rd): # type: (typed_args.Reader) -> value_t diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index 4a1cf9dfbf..be90e565ff 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -6,7 +6,7 @@ from _devbuild.gen.runtime_asdl import (scope_e) from _devbuild.gen.syntax_asdl import source -from _devbuild.gen.value_asdl import (value, value_e, value_t, block_val) +from _devbuild.gen.value_asdl import (value, value_e, value_t, cmd_frag) from core import alloc from core import error @@ -155,7 +155,7 @@ def Call(self, rd): # in # value.Command vs. value.Block - BoundCommand? - return value.Block(block_val.Expr(cmd), self.mem.CurrentFrame()) + return value.Block(cmd_frag.Expr(cmd), self.mem.CurrentFrame()) class ParseExpr(vm._Callable): diff --git a/core/runtime.asdl b/core/runtime.asdl index e26a090086..cfb5e2a95a 100644 --- a/core/runtime.asdl +++ b/core/runtime.asdl @@ -8,7 +8,7 @@ module runtime expr word command CompoundWord DoubleQuoted ArgList re redir_loc proc_sig - LiteralBlock Func + Func } use core value { diff --git a/core/value.asdl b/core/value.asdl index 500481bfdf..aabd60da0e 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -8,9 +8,10 @@ module value expr command DoubleQuoted re proc_sig - LiteralBlock Func + Func NameType EggexFlag + BraceGroup SourceLine } use core runtime { @@ -58,12 +59,15 @@ module value No | Yes %RegexMatch + # Retain references to lines + LiteralBlock = (BraceGroup brace_group, List[SourceLine] lines) + # TODO: # - Consolidate value.Command and value.LiteralBlock. All Block instances # should have backing lines. # - use LiteralBlock %LiteralBlock, but ASDL doesn't support shared variants # across files. - block_val = + cmd_frag = Literal(LiteralBlock b) # p { echo hi } has backing lines | Expr(command c) # var b = ^(echo hi) @@ -132,7 +136,7 @@ module value | Command(command c) # TODO: this is BoundCommand - | Block(block_val block, Dict[str, Cell] captured_frame) + | Block(cmd_frag block, Dict[str, Cell] captured_frame) # A place has an additional stack frame where the value is evaluated. # The frame MUST be lower on the stack at the time of use. diff --git a/frontend/syntax.asdl b/frontend/syntax.asdl index 146797bac9..a0d5372499 100644 --- a/frontend/syntax.asdl +++ b/frontend/syntax.asdl @@ -28,7 +28,7 @@ module syntax { use core value { - value + value LiteralBlock } # More efficient than the List[bool] pattern we've been using @@ -357,9 +357,6 @@ module syntax command body ) - # Retain references to lines - LiteralBlock = (BraceGroup brace_group, List[SourceLine] lines) - # Represents all these case: s=1 s+=1 s[x]=1 ... ParsedAssignment = (Token? left, Token? close, int part_offset, CompoundWord w) diff --git a/frontend/typed_args.py b/frontend/typed_args.py index af1f26aeff..bd0e775651 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -2,10 +2,11 @@ from __future__ import print_function from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs -from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, LiteralBlock, - command_t, expr_t, Token) +from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, command_t, expr_t, + Token) from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, Obj, - block_val, block_val_e, block_val_str) + cmd_frag, cmd_frag_e, cmd_frag_str, + LiteralBlock) from core import error from core.error import e_usage from frontend import location @@ -53,14 +54,14 @@ def GetCommand(bound): block = bound.block with tagswitch(block) as case: - if case(block_val_e.Literal): - lit = cast(block_val.Literal, block) + if case(cmd_frag_e.Literal): + lit = cast(cmd_frag.Literal, block) return lit.b.brace_group - elif case(block_val_e.Expr): - expr = cast(block_val.Expr, block) + elif case(cmd_frag_e.Expr): + expr = cast(cmd_frag.Expr, block) return expr.c else: - raise AssertionError(block_val_str(block.tag())) + raise AssertionError(cmd_frag_str(block.tag())) def ReaderForProc(cmd_val): @@ -379,8 +380,8 @@ def _ToLiteralBlock(self, val): if val.tag() == value_e.Block: block = cast(value.Block, val).block with tagswitch(block) as case: - if case(block_val_e.Literal): - lit = cast(block_val.Literal, block) + if case(cmd_frag_e.Literal): + lit = cast(cmd_frag.Literal, block) return lit.b else: raise AssertionError() diff --git a/osh/cmd_parse.py b/osh/cmd_parse.py index c9c810e755..a02d895f35 100644 --- a/osh/cmd_parse.py +++ b/osh/cmd_parse.py @@ -25,7 +25,6 @@ for_iter, ArgList, BraceGroup, - LiteralBlock, CaseArm, case_arg, IfArm, @@ -55,6 +54,7 @@ Proc, Func, ) +from _devbuild.gen.value_asdl import LiteralBlock from core import alloc from core import error from core.error import p_die diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 3d34ab4973..34b9f60896 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -46,7 +46,7 @@ ) from _devbuild.gen.value_asdl import (value, value_e, value_t, y_lvalue, y_lvalue_e, y_lvalue_t, IntBox, LeftName, - Obj, block_val) + Obj, cmd_frag) from core import error from core.error import e_die, e_die_status from core import num @@ -1146,7 +1146,7 @@ def _EvalExpr(self, node): id_ = node.left_token.id if id_ == Id.Left_CaretParen: # ^(echo block literal) # TODO: Propagate location info with ^( - return value.Block(block_val.Expr(node.child), + return value.Block(cmd_frag.Expr(node.child), self.mem.CurrentFrame()) else: stdout_str = self.shell_ex.RunCommandSub(node) diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 5d75092313..203804c283 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -10,7 +10,7 @@ NamedArg, Func, loc, ArgList, expr, expr_e, expr_t) from _devbuild.gen.value_asdl import (value, value_e, value_t, ProcDefaults, - LeftName, block_val) + LeftName, cmd_frag) from core import error from core.error import e_die @@ -265,7 +265,7 @@ def EvalTypedArgsToProc( # p { echo hi } is an unevaluated block if node.block: # Attach current frame to value.Block - proc_args.block_arg = value.Block(block_val.Literal(node.block), + proc_args.block_arg = value.Block(cmd_frag.Literal(node.block), current_frame) # Add location info so the cmd_val looks the same for both: From 65acd157ea5b94076696fdf8fe0f7093aca7eb1d Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 20:47:49 -0400 Subject: [PATCH 325/506] [value.asdl refactor] Introduce cmd_frag, rename to CommandFrag These are unbound commands. Used shared variant for the LiteralBlock type. --- core/shell.py | 2 +- core/value.asdl | 16 ++++++---------- demo/survey-closure.rb | 23 +++++++++++++++++++++++ demo/survey-closure.sh | 4 ++++ frontend/typed_args.py | 36 ++++++++++++++++++------------------ ysh/func_proc.py | 5 ++--- ysh/val_ops.py | 6 +++--- 7 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 demo/survey-closure.rb diff --git a/core/shell.py b/core/shell.py index e1264c5e70..fa910483ba 100644 --- a/core/shell.py +++ b/core/shell.py @@ -836,7 +836,7 @@ def Main( 'M/setValue': method_other.SetValue(mem), } - methods[value_e.Command] = { + methods[value_e.CommandFrag] = { # var x = ^(echo hi) # Export source code and line number # Useful for test frameworks and so forth diff --git a/core/value.asdl b/core/value.asdl index aabd60da0e..20f6547d71 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -62,14 +62,10 @@ module value # Retain references to lines LiteralBlock = (BraceGroup brace_group, List[SourceLine] lines) - # TODO: - # - Consolidate value.Command and value.LiteralBlock. All Block instances - # should have backing lines. - # - use LiteralBlock %LiteralBlock, but ASDL doesn't support shared variants - # across files. + # TODO: should Expr also have backing lines? cmd_frag = - Literal(LiteralBlock b) # p { echo hi } has backing lines - | Expr(command c) # var b = ^(echo hi) + LiteralBlock %LiteralBlock # p { echo hi } has backing lines + | Expr(command c) # var b = ^(echo hi) # Arbitrary objects, where attributes are looked up on the prototype chain. Obj = (Obj? prototype, Dict[str, value] d) @@ -131,12 +127,12 @@ module value # ^[42 + a[i]] | Expr(expr e) - # TODO: this is an UnboundCommand? + # This is an UNBOUND command, like # ^(echo 1; echo 2) and cd { echo 1; echo 2 } - | Command(command c) + | CommandFrag(command c) # TODO: this is BoundCommand - | Block(cmd_frag block, Dict[str, Cell] captured_frame) + | Block(cmd_frag frag, Dict[str, Cell] captured_frame) # A place has an additional stack frame where the value is evaluated. # The frame MUST be lower on the stack at the time of use. diff --git a/demo/survey-closure.rb b/demo/survey-closure.rb new file mode 100644 index 0000000000..d53cdbd035 --- /dev/null +++ b/demo/survey-closure.rb @@ -0,0 +1,23 @@ + +def create_context(x) + y = 20 + binding +end + +binding = create_context(10) +puts binding +puts binding.class + +vars = binding.eval("local_variables") +puts vars +puts vars.class + +# Why doesn't this work? +#myproc = proc { puts "x: #{x}, y: #{y}" } + +# You need this longer thing +myproc = proc { puts "x: #{binding.eval("x")}, y: #{binding.eval("y")}" } + +# Execute the block in the context +binding.instance_exec(&myproc) + diff --git a/demo/survey-closure.sh b/demo/survey-closure.sh index 4bb14b80ef..ae0fd4298c 100755 --- a/demo/survey-closure.sh +++ b/demo/survey-closure.sh @@ -494,4 +494,8 @@ puts result ' } +ruby-binding() { + ruby demo/survey-closure.rb +} + "$@" diff --git a/frontend/typed_args.py b/frontend/typed_args.py index bd0e775651..9d634f070f 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -52,16 +52,16 @@ def OptionalLiteralBlock(cmd_val): def GetCommand(bound): # type: (value.Block) -> command_t - block = bound.block - with tagswitch(block) as case: - if case(cmd_frag_e.Literal): - lit = cast(cmd_frag.Literal, block) - return lit.b.brace_group + frag = bound.frag + with tagswitch(frag) as case: + if case(cmd_frag_e.LiteralBlock): + lit = cast(LiteralBlock, frag) + return lit.brace_group elif case(cmd_frag_e.Expr): - expr = cast(cmd_frag.Expr, block) + expr = cast(cmd_frag.Expr, frag) return expr.c else: - raise AssertionError(cmd_frag_str(block.tag())) + raise AssertionError(cmd_frag_str(frag.tag())) def ReaderForProc(cmd_val): @@ -337,10 +337,10 @@ def _ToExpr(self, val): raise error.TypeErr(val, 'Arg %d should be a Expr' % self.pos_consumed, self.BlamePos()) - def _ToCommand(self, val): + def _ToCommandFrag(self, val): # type: (value_t) -> command_t - if val.tag() == value_e.Command: - return cast(value.Command, val).c + if val.tag() == value_e.CommandFrag: + return cast(value.CommandFrag, val).c # io.eval(mycmd) uses this if val.tag() == value_e.Block: @@ -353,8 +353,8 @@ def _ToCommand(self, val): def _ToBlock(self, val): # type: (value_t) -> command_t - if val.tag() == value_e.Command: - return cast(value.Command, val).c + if val.tag() == value_e.CommandFrag: + return cast(value.CommandFrag, val).c # Special case for hay # Foo { x = 1 } @@ -378,11 +378,11 @@ def _ToLiteralBlock(self, val): # type: (value_t) -> LiteralBlock """ Used by Hay """ if val.tag() == value_e.Block: - block = cast(value.Block, val).block - with tagswitch(block) as case: - if case(cmd_frag_e.Literal): - lit = cast(cmd_frag.Literal, block) - return lit.b + frag = cast(value.Block, val).frag + with tagswitch(frag) as case: + if case(cmd_frag_e.LiteralBlock): + lit = cast(LiteralBlock, frag) + return lit else: raise AssertionError() @@ -467,7 +467,7 @@ def PosMatch(self): def PosCommand(self): # type: () -> command_t val = self.PosValue() - return self._ToCommand(val) + return self._ToCommandFrag(val) def PosBoundCommand(self): # type: () -> value.Block diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 203804c283..25a72de438 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -10,7 +10,7 @@ NamedArg, Func, loc, ArgList, expr, expr_e, expr_t) from _devbuild.gen.value_asdl import (value, value_e, value_t, ProcDefaults, - LeftName, cmd_frag) + LeftName) from core import error from core.error import e_die @@ -265,8 +265,7 @@ def EvalTypedArgsToProc( # p { echo hi } is an unevaluated block if node.block: # Attach current frame to value.Block - proc_args.block_arg = value.Block(cmd_frag.Literal(node.block), - current_frame) + proc_args.block_arg = value.Block(node.block, current_frame) # Add location info so the cmd_val looks the same for both: # cd /tmp (; ; ^(echo hi)) diff --git a/ysh/val_ops.py b/ysh/val_ops.py index 9da110afce..7c1d9ad670 100644 --- a/ysh/val_ops.py +++ b/ysh/val_ops.py @@ -74,11 +74,11 @@ def ToDict(val, msg, blame_loc): raise error.TypeErr(val, msg, blame_loc) -def ToCommand(val, msg, blame_loc): +def ToCommandFrag(val, msg, blame_loc): # type: (value_t, str, loc_t) -> command_t UP_val = val - if val.tag() == value_e.Command: - val = cast(value.Command, UP_val) + if val.tag() == value_e.CommandFrag: + val = cast(value.CommandFrag, UP_val) return val.c raise error.TypeErr(val, msg, blame_loc) From be3af71c4775155a8aab3f2bc925082412bad77d Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 21:37:38 -0400 Subject: [PATCH 326/506] [value.asdl refactor] Distinguish between Command and CommandFrag This revealed that io.captureStdout() should probably take a Command, not CommandFrag. So you can access variables from the outer scope, like io->eval(). That might make pure functions a little less ergonomic though. Also evalHay(). --- builtin/func_hay.py | 4 ++-- builtin/func_reflect.py | 2 +- builtin/method_io.py | 4 ++-- core/value.asdl | 4 ++-- frontend/typed_args.py | 42 ++++++++++++++++++++--------------------- ysh/expr_eval.py | 4 ++-- ysh/func_proc.py | 9 +++------ 7 files changed, 33 insertions(+), 36 deletions(-) diff --git a/builtin/func_hay.py b/builtin/func_hay.py index 960926a3ae..036f1c497c 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -65,7 +65,7 @@ def _Call(self, path): self.errfmt.PrettyPrintError(e) return None - return value.Block(cmd_frag.Expr(node), self.mem.CurrentFrame()) + return value.Command(cmd_frag.Expr(node), self.mem.CurrentFrame()) def Call(self, rd): # type: (typed_args.Reader) -> value_t @@ -105,7 +105,7 @@ def _Call(self, cmd): def Call(self, rd): # type: (typed_args.Reader) -> value_t - cmd = rd.PosCommand() + cmd = rd.PosCommandFrag() rd.Done() return value.Dict(self._Call(cmd)) diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index be90e565ff..67cad77c19 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -155,7 +155,7 @@ def Call(self, rd): # in # value.Command vs. value.Block - BoundCommand? - return value.Block(cmd_frag.Expr(cmd), self.mem.CurrentFrame()) + return value.Command(cmd_frag.Expr(cmd), self.mem.CurrentFrame()) class ParseExpr(vm._Callable): diff --git a/builtin/method_io.py b/builtin/method_io.py index ca9866c3d8..d8fbe30109 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -67,7 +67,7 @@ def Call(self, rd): bound = rd.PosBoundCommand() captured_frame = bound.captured_frame - cmd = typed_args.GetCommand(bound) + cmd = typed_args.GetCommandFrag(bound) #log('CAPTURED %r', captured_frame) @@ -123,7 +123,7 @@ def Call(self, rd): # type: (typed_args.Reader) -> value_t unused = rd.PosValue() - cmd = rd.PosCommand() + cmd = rd.PosCommandFrag() # TODO: Use bound command? rd.Done() # no more args status, stdout_str = self.shell_ex.CaptureStdout(cmd) diff --git a/core/value.asdl b/core/value.asdl index 20f6547d71..619e584708 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -131,8 +131,8 @@ module value # ^(echo 1; echo 2) and cd { echo 1; echo 2 } | CommandFrag(command c) - # TODO: this is BoundCommand - | Block(cmd_frag frag, Dict[str, Cell] captured_frame) + # Bound command + | Command(cmd_frag frag, Dict[str, Cell] captured_frame) # A place has an additional stack frame where the value is evaluated. # The frame MUST be lower on the stack at the time of use. diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 9d634f070f..92fe8ae041 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -49,8 +49,8 @@ def OptionalLiteralBlock(cmd_val): return block -def GetCommand(bound): - # type: (value.Block) -> command_t +def GetCommandFrag(bound): + # type: (value.Command) -> command_t frag = bound.frag with tagswitch(frag) as case: @@ -343,13 +343,13 @@ def _ToCommandFrag(self, val): return cast(value.CommandFrag, val).c # io.eval(mycmd) uses this - if val.tag() == value_e.Block: - bound = cast(value.Block, val) - return GetCommand(bound) + if val.tag() == value_e.Command: + bound = cast(value.Command, val) + return GetCommandFrag(bound) - raise error.TypeErr(val, - 'Arg %d should be a Command' % self.pos_consumed, - self.BlamePos()) + raise error.TypeErr( + val, 'Arg %d should be a CommandFrag' % self.pos_consumed, + self.BlamePos()) def _ToBlock(self, val): # type: (value_t) -> command_t @@ -358,27 +358,27 @@ def _ToBlock(self, val): # Special case for hay # Foo { x = 1 } - if val.tag() == value_e.Block: - bound = cast(value.Block, val) - return GetCommand(bound) + if val.tag() == value_e.Command: + bound = cast(value.Command, val) + return GetCommandFrag(bound) raise error.TypeErr(val, 'Arg %d should be a Block' % self.pos_consumed, self.BlamePos()) def _ToBoundCommand(self, val): - # type: (value_t) -> value.Block - if val.tag() == value_e.Block: - return cast(value.Block, val) - raise error.TypeErr( - val, 'Arg %d should be a BoundCommand' % self.pos_consumed, - self.BlamePos()) + # type: (value_t) -> value.Command + if val.tag() == value_e.Command: + return cast(value.Command, val) + raise error.TypeErr(val, + 'Arg %d should be a Command' % self.pos_consumed, + self.BlamePos()) def _ToLiteralBlock(self, val): # type: (value_t) -> LiteralBlock """ Used by Hay """ - if val.tag() == value_e.Block: - frag = cast(value.Block, val).frag + if val.tag() == value_e.Command: + frag = cast(value.Command, val).frag with tagswitch(frag) as case: if case(cmd_frag_e.LiteralBlock): lit = cast(LiteralBlock, frag) @@ -464,13 +464,13 @@ def PosMatch(self): val = self.PosValue() return self._ToMatch(val) - def PosCommand(self): + def PosCommandFrag(self): # type: () -> command_t val = self.PosValue() return self._ToCommandFrag(val) def PosBoundCommand(self): - # type: () -> value.Block + # type: () -> value.Command val = self.PosValue() return self._ToBoundCommand(val) diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 34b9f60896..c4be1af6c9 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -1146,8 +1146,8 @@ def _EvalExpr(self, node): id_ = node.left_token.id if id_ == Id.Left_CaretParen: # ^(echo block literal) # TODO: Propagate location info with ^( - return value.Block(cmd_frag.Expr(node.child), - self.mem.CurrentFrame()) + return value.Command(cmd_frag.Expr(node.child), + self.mem.CurrentFrame()) else: stdout_str = self.shell_ex.RunCommandSub(node) if id_ == Id.Left_AtParen: # @(seq 3) diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 25a72de438..84478a9b59 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -124,10 +124,7 @@ def EvalProcDefaults(expr_ev, sig): if exp: block_default = expr_ev.EvalExpr(exp, sig.block_param.blame_tok) # It can only be ^() or null - if block_default.tag() not in (value_e.Null, value_e.Block): - - # TODO: This is a value.Command, not a value.BoundCommand/Block? - + if block_default.tag() not in (value_e.Null, value_e.Command): raise error.TypeErr( block_default, "Default value for block should be Command or Null", @@ -264,8 +261,8 @@ def EvalTypedArgsToProc( # p { echo hi } is an unevaluated block if node.block: - # Attach current frame to value.Block - proc_args.block_arg = value.Block(node.block, current_frame) + # Attach current frame to command fragment + proc_args.block_arg = value.Command(node.block, current_frame) # Add location info so the cmd_val looks the same for both: # cd /tmp (; ; ^(echo hi)) From 2b015fcf04309637fea144b9d10a893fe0aa371b Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 22:32:11 -0400 Subject: [PATCH 327/506] [test/spec] Fix assertions after Block -> Command rename --- spec/ysh-builtin-eval.test.sh | 2 +- spec/ysh-builtin-meta.test.sh | 6 +++--- spec/ysh-builtin-module.test.sh | 4 ++-- spec/ysh-proc.test.sh | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index d9cb35ec7c..7ae45e7f25 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -383,7 +383,7 @@ var d = io->evalToDict(cmd) pp test_ (d) ## STDOUT: - + hi (Dict) {"x":42,"y":"global"} ## END diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index b82c48ee3d..9f0d58d23b 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -75,19 +75,19 @@ ty (Str) "a" (Int) 42 (Int) 99 -Block +Command ty (Str) "a" (Int) 42 (Int) 99 -Block +Command ty (Str) "a" (Int) 42 (Int) 99 -Block +Command ## END diff --git a/spec/ysh-builtin-module.test.sh b/spec/ysh-builtin-module.test.sh index 35644f10d3..c14ee2fa69 100644 --- a/spec/ysh-builtin-module.test.sh +++ b/spec/ysh-builtin-module.test.sh @@ -291,7 +291,7 @@ util2 echo-args w1 w2 (3, 4, n3=9) { (List) [7,8] (Dict) {"n3":9} - + --- (List) ["w1","w2"] (List) [] @@ -302,7 +302,7 @@ util2 echo-args w1 w2 (3, 4, n3=9) { (List) [42,43] (Dict) {"n3":9} - + ## END #### module-with-hyphens diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 736b1629f2..3d76d1cbbc 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -347,12 +347,12 @@ p2 a b ('c', 'd'; n=99; block) { (List) ["a","b"] (List) ["c","d"] (Dict) {"n":99} -Block +Command (List) ["a","b"] (List) ["c","d"] (Dict) {"n":99} -Block +Command ## END @@ -444,7 +444,7 @@ p word (42, n=99) { (Str) "word" (Int) 42 (Int) 99 -Block +Command ## END #### can unset procs without -f From 72602275e3df79ee21c86aef4257f4bc2f745110 Mon Sep 17 00:00:00 2001 From: Andy C Date: Sun, 13 Oct 2024 23:01:48 -0400 Subject: [PATCH 328/506] [test/spec] Failing tests for block scope, value.Command should capture Clean up typed_args.py interface. Builtins are using RequiredBlock/OptionalBlock, with value.CommandFrag. To fix the cd bug, this should be value.Command. We also need figure out how globals are bound. --- builtin/method_io.py | 8 +------ frontend/typed_args.py | 25 +++++---------------- spec/ysh-blocks.test.sh | 48 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index d8fbe30109..7b61d2b083 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -58,15 +58,9 @@ def __init__(self, cmd_ev, which): def Call(self, rd): # type: (typed_args.Reader) -> value_t unused = rd.PosValue() + bound = rd.PosCommand() - # TODO: Can we evaluated both: - # value.BoundCommand - # value.Command (unbound) - #cmd, val = rd.PosCommand2() - - bound = rd.PosBoundCommand() captured_frame = bound.captured_frame - cmd = typed_args.GetCommandFrag(bound) #log('CAPTURED %r', captured_frame) diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 92fe8ae041..181842d0b3 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -351,22 +351,7 @@ def _ToCommandFrag(self, val): val, 'Arg %d should be a CommandFrag' % self.pos_consumed, self.BlamePos()) - def _ToBlock(self, val): - # type: (value_t) -> command_t - if val.tag() == value_e.CommandFrag: - return cast(value.CommandFrag, val).c - - # Special case for hay - # Foo { x = 1 } - if val.tag() == value_e.Command: - bound = cast(value.Command, val) - return GetCommandFrag(bound) - - raise error.TypeErr(val, - 'Arg %d should be a Block' % self.pos_consumed, - self.BlamePos()) - - def _ToBoundCommand(self, val): + def _ToCommand(self, val): # type: (value_t) -> value.Command if val.tag() == value_e.Command: return cast(value.Command, val) @@ -469,10 +454,10 @@ def PosCommandFrag(self): val = self.PosValue() return self._ToCommandFrag(val) - def PosBoundCommand(self): + def PosCommand(self): # type: () -> value.Command val = self.PosValue() - return self._ToBoundCommand(val) + return self._ToCommand(val) def PosExpr(self): # type: () -> expr_t @@ -488,13 +473,13 @@ def RequiredBlock(self): if self.block_arg is None: raise error.TypeErrVerbose('Expected a block arg', self.LeastSpecificLocation()) - return self._ToBlock(self.block_arg) + return self._ToCommandFrag(self.block_arg) def OptionalBlock(self): # type: () -> Optional[command_t] if self.block_arg is None: return None - return self._ToBlock(self.block_arg) + return self._ToCommandFrag(self.block_arg) def OptionalLiteralBlock(self): # type: () -> Optional[LiteralBlock] diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh index a7604a40f6..a7008f68dd 100644 --- a/spec/ysh-blocks.test.sh +++ b/spec/ysh-blocks.test.sh @@ -1,3 +1,5 @@ +## oils_failures_allowed: 2 + #### cd accepts a block, runs it in different dir shopt -s ysh:all @@ -58,6 +60,52 @@ echo 'not reached' block ## END +#### cd passed a block defined in a different scope +shopt --set ysh:upgrade + +proc my-cd (; b) { + cd /tmp ( ; ; b) +} + +proc p { + var i = 42 + var b = ^(echo "i = $i") + + my-cd (b) +} + +p + +## STDOUT: +## END + +#### io->eval() and io.captureStdout() passed a block in different scope +shopt --set ysh:upgrade + +proc my-cd (; b) { + call io->eval(b) + + var d = io->evalToDict(b) + + pp test_ (d) + + # Yup, this is a problem + var s = io.captureStdout(b) + echo "stdout $s" +} + +proc p { + var i = 42 + var b = ^(var x = 'x'; echo "i = $i") + + my-cd (b) +} + +p + +## STDOUT: +## END + #### block doesn't have its own scope shopt -s ysh:all var x = 1 From 7e50f2f5b13c84566a79059fffe5d633a21f4ac1 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 14 Oct 2024 00:04:47 -0400 Subject: [PATCH 329/506] [ysh reflection] thisFrame() and bindCommand() Working on CommandFrag vs. Command distinction. I want to fix 'cd' as well. --- builtin/dirs_osh.py | 2 +- builtin/error_ysh.py | 2 +- builtin/func_hay.py | 2 +- builtin/func_reflect.py | 27 +++++++++++++++++++++++++++ builtin/hay_ysh.py | 4 ++-- builtin/io_ysh.py | 2 +- builtin/method_io.py | 4 ++-- builtin/pure_osh.py | 2 +- builtin/pure_ysh.py | 6 +++--- core/shell.py | 8 ++++++++ frontend/typed_args.py | 16 +++++++++++++++- osh/cmd_eval.py | 2 +- spec/ysh-func-builtin.test.sh | 14 ++++++++++++++ 13 files changed, 77 insertions(+), 14 deletions(-) diff --git a/builtin/dirs_osh.py b/builtin/dirs_osh.py index 0d7a715455..811fe272b7 100644 --- a/builtin/dirs_osh.py +++ b/builtin/dirs_osh.py @@ -166,7 +166,7 @@ def Run(self, cmd_val): out_errs = [] # type: List[bool] with ctx_CdBlock(self.dir_stack, real_dest_dir, self.mem, self.errfmt, out_errs): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) if len(out_errs): return 1 diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index c34ce88ae3..ecd6ee39ed 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -104,7 +104,7 @@ def Run(self, cmd_val): status = 0 # success by default try: with ctx_Try(self.mutable_opts): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) except error.Expr as e: status = e.ExitStatus() except error.ErrExit as e: diff --git a/builtin/func_hay.py b/builtin/func_hay.py index 036f1c497c..e1ee9f41c6 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -95,7 +95,7 @@ def _Call(self, cmd): # type: (command_t) -> Dict[str, value_t] with hay_ysh.ctx_HayEval(self.hay_state, self.mutable_opts, self.mem): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) return self.hay_state.Result() diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index 67cad77c19..13236817e4 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -61,6 +61,33 @@ def Call(self, rd): raise AssertionError() +class ThisFrame(vm._Callable): + + def __init__(self, mem): + # type: (state.Mem) -> None + vm._Callable.__init__(self) + self.mem = mem + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + rd.Done() + return value.Frame(self.mem.CurrentFrame()) + + +class BindCommand(vm._Callable): + + def __init__(self): + # type: () -> None + vm._Callable.__init__(self) + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + frag = rd.PosCommandFrag() + frame = rd.PosFrame() + rd.Done() + return value.Command(cmd_frag.Expr(frag), frame) + + class Shvar_get(vm._Callable): """Look up with dynamic scope.""" diff --git a/builtin/hay_ysh.py b/builtin/hay_ysh.py index e9bc3ba187..4049dd0abb 100644 --- a/builtin/hay_ysh.py +++ b/builtin/hay_ysh.py @@ -291,7 +291,7 @@ def Run(self, cmd_val): with ctx_HayEval(self.hay_state, self.mutable_opts, self.mem): # Note: we want all haynode invocations in the block to appear as # our 'children', recursively - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) result = self.hay_state.Result() @@ -411,7 +411,7 @@ def Run(self, cmd_val): with ctx_HayNode(self.hay_state, hay_name): # Note: we want all haynode invocations in the block to appear as # our 'children', recursively - self.cmd_ev.EvalCommand(lit_block.brace_group) + self.cmd_ev.EvalCommandFrag(lit_block.brace_group) # Treat the vars as a Dict block_attrs = self.mem.TopNamespace() diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 4520068e76..efdf71e587 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -314,5 +314,5 @@ def Run(self, cmd_val): if not cmd: raise error.Usage('expected a block', loc.Missing) - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) return 0 diff --git a/builtin/method_io.py b/builtin/method_io.py index 7b61d2b083..1f18172965 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -90,7 +90,7 @@ def Call(self, rd): bindings): # _PrintFrame('[new]', self.cmd_ev.mem.var_stack[-1]) with state.ctx_Eval(self.cmd_ev.mem, dollar0, pos_args, vars_): - unused_status = self.cmd_ev.EvalCommand(cmd) + unused_status = self.cmd_ev.EvalCommandFrag(cmd) return value.Null elif self.which == EVAL_DICT: @@ -100,7 +100,7 @@ def Call(self, rd): bindings = NewDict() with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, bindings): - unused_status = self.cmd_ev.EvalCommand(cmd) + unused_status = self.cmd_ev.EvalCommandFrag(cmd) return value.Dict(bindings) else: diff --git a/builtin/pure_osh.py b/builtin/pure_osh.py index ff881e2239..c007d77abe 100644 --- a/builtin/pure_osh.py +++ b/builtin/pure_osh.py @@ -256,7 +256,7 @@ def Run(self, cmd_val): opt_nums.append(index) with state.ctx_Option(self.mutable_opts, opt_nums, b): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) return 0 # cd also returns 0 # Otherwise, set options. diff --git a/builtin/pure_ysh.py b/builtin/pure_ysh.py index 4e2bcccf7b..8fe247e3a1 100644 --- a/builtin/pure_ysh.py +++ b/builtin/pure_ysh.py @@ -58,7 +58,7 @@ def Run(self, cmd_val): self.search_path.ClearCache() with state.ctx_Eval(self.mem, None, None, vars): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) return 0 @@ -99,7 +99,7 @@ def _GetContext(self): def _Push(self, context, block): # type: (Dict[str, value_t], command_t) -> int with ctx_Context(self.mem, context): - return self.cmd_ev.EvalCommand(block) + return self.cmd_ev.EvalCommandFrag(block) def _Set(self, updates): # type: (Dict[str, value_t]) -> int @@ -181,7 +181,7 @@ def Run(self, cmd_val): raise error.Usage('expected a block', loc.Missing) with state.ctx_Registers(self.mem): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) # make it "SILENT" in terms of not mutating $? # TODO: Revisit this. It might be better to provide the headless shell diff --git a/core/shell.py b/core/shell.py index fa910483ba..cab6d47c51 100644 --- a/core/shell.py +++ b/core/shell.py @@ -877,6 +877,14 @@ def Main( _AddBuiltinFunc(mem, 'getVar', func_reflect.GetVar(mem)) _AddBuiltinFunc(mem, 'setVar', func_reflect.SetVar(mem)) + # TODO: implement + # and then parseCommand() and parseHay will not depend on mem; they will + # not bind a frame yet + # + # what about newFrame() and globalFrame()? + _AddBuiltinFunc(mem, 'thisFrame', func_reflect.ThisFrame(mem)) + _AddBuiltinFunc(mem, 'bindCommand', func_reflect.BindCommand()) + _AddBuiltinFunc(mem, 'Object', func_misc.Object()) _AddBuiltinFunc(mem, 'prototype', func_misc.Prototype()) _AddBuiltinFunc(mem, 'propView', func_misc.PropView()) diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 181842d0b3..1ad13bf9e2 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -1,7 +1,7 @@ #!/usr/bin/env python2 from __future__ import print_function -from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs +from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs, Cell from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, command_t, expr_t, Token) from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, Obj, @@ -337,6 +337,15 @@ def _ToExpr(self, val): raise error.TypeErr(val, 'Arg %d should be a Expr' % self.pos_consumed, self.BlamePos()) + def _ToFrame(self, val): + # type: (value_t) -> Dict[str, Cell] + if val.tag() == value_e.Frame: + return cast(value.Frame, val).frame + + raise error.TypeErr(val, + 'Arg %d should be a Frame' % self.pos_consumed, + self.BlamePos()) + def _ToCommandFrag(self, val): # type: (value_t) -> command_t if val.tag() == value_e.CommandFrag: @@ -449,6 +458,11 @@ def PosMatch(self): val = self.PosValue() return self._ToMatch(val) + def PosFrame(self): + # type: () -> Dict[str, Cell] + val = self.PosValue() + return self._ToFrame(val) + def PosCommandFrag(self): # type: () -> command_t val = self.PosValue() diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index a5b33c0d4b..83df898c46 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -2074,7 +2074,7 @@ def ExecuteAndCatch(self, node, cmd_flags): self.mem.SetLastStatus(status) return is_return, is_fatal - def EvalCommand(self, block): + def EvalCommandFrag(self, block): # type: (command_t) -> int """For builtins to evaluate command args. diff --git a/spec/ysh-func-builtin.test.sh b/spec/ysh-func-builtin.test.sh index fae66f5d9e..9a81ccc488 100644 --- a/spec/ysh-func-builtin.test.sh +++ b/spec/ysh-func-builtin.test.sh @@ -181,3 +181,17 @@ echo $[y => lower()] ÀÈ áé ## END + +#### thisFrame() + +var fr = thisFrame() +pp test_ (fr) +#= fr + +#var bound = bindCommand(null, fr) +#pp test_ (bound) + +## STDOUT: + +## END + From 1203758b35caca26b85124680390b58586d521b4 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 14 Oct 2024 00:51:44 -0400 Subject: [PATCH 330/506] [builtin/cd breaking] The cd block arg now captures variables We now use EvalCommand(), not EvalCommandFrag() This will be done for all builtin commands that take blocks. This makes them behave like user-defined procs! --- builtin/dirs_osh.py | 4 ++-- builtin/method_io.py | 6 +----- core/state.py | 19 ++++++++++--------- frontend/typed_args.py | 17 +++++++++++++++++ osh/cmd_eval.py | 11 +++++++++-- spec/ysh-blocks.test.sh | 3 ++- spec/ysh-scope.test.sh | 9 ++++----- 7 files changed, 45 insertions(+), 24 deletions(-) diff --git a/builtin/dirs_osh.py b/builtin/dirs_osh.py index 811fe272b7..15ccabaa32 100644 --- a/builtin/dirs_osh.py +++ b/builtin/dirs_osh.py @@ -102,7 +102,7 @@ def Run(self, cmd_val): arg = arg_types.cd(attrs.attrs) # If a block is passed, we do additional syntax checks - cmd = typed_args.OptionalBlock(cmd_val) + cmd = typed_args.OptionalCommandBlock(cmd_val) dest_dir, arg_loc = arg_r.Peek2() if dest_dir is None: @@ -166,7 +166,7 @@ def Run(self, cmd_val): out_errs = [] # type: List[bool] with ctx_CdBlock(self.dir_stack, real_dest_dir, self.mem, self.errfmt, out_errs): - unused = self.cmd_ev.EvalCommandFrag(cmd) + unused = self.cmd_ev.EvalCommand(cmd) if len(out_errs): return 1 diff --git a/builtin/method_io.py b/builtin/method_io.py index 1f18172965..5604aa76e8 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -83,11 +83,7 @@ def Call(self, rd): if self.which == EVAL_NULL: # _PrintFrame('[captured]', captured_frame) - - # TOOD: don't need bindings - bindings = NewDict() # type: Dict[str, value_t] - with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, - bindings): + with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, None): # _PrintFrame('[new]', self.cmd_ev.mem.var_stack[-1]) with state.ctx_Eval(self.cmd_ev.mem, dollar0, pos_args, vars_): unused_status = self.cmd_ev.EvalCommandFrag(cmd) diff --git a/core/state.py b/core/state.py index a280938180..bafc0cbece 100644 --- a/core/state.py +++ b/core/state.py @@ -1200,7 +1200,7 @@ class ctx_FrontFrame(object): """ def __init__(self, mem, rear_frame, out_dict): - # type: (Mem, Dict[str, Cell], Dict[str, value_t]) -> None + # type: (Mem, Dict[str, Cell], Optional[Dict[str, value_t]]) -> None self.mem = mem self.rear_frame = rear_frame self.out_dict = out_dict @@ -1219,16 +1219,17 @@ def __enter__(self): def __exit__(self, type, value, traceback): # type: (Any, Any, Any) -> None - for name, cell in iteritems(self.front_frame): - #log('name %r', name) - #log('cell %r', cell) + if self.out_dict is not None: + for name, cell in iteritems(self.front_frame): + #log('name %r', name) + #log('cell %r', cell) - # User can hide variables with _ suffix - # e.g. for i_ in foo bar { echo $i_ } - if name.endswith('_'): - continue + # User can hide variables with _ suffix + # e.g. for i_ in foo bar { echo $i_ } + if name.endswith('_'): + continue - self.out_dict[name] = cell.val + self.out_dict[name] = cell.val # Restore self.mem.var_stack.pop() diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 1ad13bf9e2..92dc9d6c07 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -25,6 +25,17 @@ def DoesNotAccept(proc_args): e_usage('got unexpected typed args', proc_args.typed_args.left) +def OptionalCommandBlock(cmd_val): + # type: (cmd_value.Argv) -> Optional[value.Command] + + cmd = None # type: Optional[value.Command] + if cmd_val.proc_args: + r = ReaderForProc(cmd_val) + cmd = r.OptionalCommandBlock() + r.Done() + return cmd + + def OptionalBlock(cmd_val): # type: (cmd_value.Argv) -> Optional[command_t] """Helper for shopt, etc.""" @@ -482,6 +493,12 @@ def PosExpr(self): # Block arg # + def OptionalCommandBlock(self): + # type: () -> Optional[value.Command] + if self.block_arg is None: + return None + return self._ToCommand(self.block_arg) + def RequiredBlock(self): # type: () -> command_t if self.block_arg is None: diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 83df898c46..3b12990f67 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -81,6 +81,7 @@ from frontend import consts from frontend import lexer from frontend import location +from frontend import typed_args from osh import braces from osh import sh_expr_eval from osh import word_eval @@ -2074,7 +2075,7 @@ def ExecuteAndCatch(self, node, cmd_flags): self.mem.SetLastStatus(status) return is_return, is_fatal - def EvalCommandFrag(self, block): + def EvalCommandFrag(self, frag): # type: (command_t) -> int """For builtins to evaluate command args. @@ -2090,7 +2091,13 @@ def EvalCommandFrag(self, block): (Should those be more like eval 'mystring'?) """ - return self._Execute(block) # can raise FatalRuntimeError, etc. + return self._Execute(frag) # can raise FatalRuntimeError, etc. + + def EvalCommand(self, cmd): + # type: (value.Command) -> int + frag = typed_args.GetCommandFrag(cmd) + with state.ctx_FrontFrame(self.mem, cmd.captured_frame, None): + return self.EvalCommandFrag(frag) def RunTrapsOnExit(self, mut_status): # type: (IntParamBox) -> None diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh index a7008f68dd..a1d0fd32eb 100644 --- a/spec/ysh-blocks.test.sh +++ b/spec/ysh-blocks.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 2 +## oils_failures_allowed: 1 #### cd accepts a block, runs it in different dir shopt -s ysh:all @@ -77,6 +77,7 @@ proc p { p ## STDOUT: +i = 42 ## END #### io->eval() and io.captureStdout() passed a block in different scope diff --git a/spec/ysh-scope.test.sh b/spec/ysh-scope.test.sh index 901b91c1a4..65165844fc 100644 --- a/spec/ysh-scope.test.sh +++ b/spec/ysh-scope.test.sh @@ -550,8 +550,8 @@ inline FOO= bar ## END -#### cd blocks don't introduce new scopes -shopt --set oil:upgrade +#### cd blocks introduce new scopes +shopt --set ysh:upgrade var x = 42 cd / { @@ -560,12 +560,11 @@ cd / { echo $x $y $z setvar y = 43 } -setvar z = 44 -echo $x $y $z +echo $x $[getVar('y')] $[getVar('z')] ## STDOUT: 42 0 1 -42 43 44 +42 null null ## END #### IFS=: myproc exports when it doesn't need to From 9239ee89cfbd5ea9eb82b8c22fbe9fccc7d5f36b Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 14 Oct 2024 01:28:25 -0400 Subject: [PATCH 331/506] [translation] Fix build --- builtin/method_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index 5604aa76e8..d578ac0c1a 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -93,7 +93,7 @@ def Call(self, rd): # TODO: dollar0, pos_args, vars_ not supported # Does ctx_FrontFrame has different scoping rules? For "vars"? - bindings = NewDict() + bindings = NewDict() # type: Dict[str, value_t] with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, bindings): unused_status = self.cmd_ev.EvalCommandFrag(cmd) From 2af369b6f91919680715c23e29bbed3403ba17dd Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 14 Oct 2024 01:32:58 -0400 Subject: [PATCH 332/506] [ysh] Lexical scope for block passed to io.captureStdout() --- builtin/method_io.py | 23 +++++++++++++---------- core/shell.py | 6 +++--- spec/ysh-blocks.test.sh | 10 +++++++--- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index d578ac0c1a..604387c3e2 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -50,8 +50,9 @@ class Eval(vm._Callable): The CALLER must handle errors. """ - def __init__(self, cmd_ev, which): - # type: (cmd_eval.CommandEvaluator, int) -> None + def __init__(self, mem, cmd_ev, which): + # type: (state.Mem, cmd_eval.CommandEvaluator, int) -> None + self.mem = mem self.cmd_ev = cmd_ev self.which = which @@ -83,9 +84,9 @@ def Call(self, rd): if self.which == EVAL_NULL: # _PrintFrame('[captured]', captured_frame) - with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, None): + with state.ctx_FrontFrame(self.mem, captured_frame, None): # _PrintFrame('[new]', self.cmd_ev.mem.var_stack[-1]) - with state.ctx_Eval(self.cmd_ev.mem, dollar0, pos_args, vars_): + with state.ctx_Eval(self.mem, dollar0, pos_args, vars_): unused_status = self.cmd_ev.EvalCommandFrag(cmd) return value.Null @@ -94,8 +95,7 @@ def Call(self, rd): # Does ctx_FrontFrame has different scoping rules? For "vars"? bindings = NewDict() # type: Dict[str, value_t] - with state.ctx_FrontFrame(self.cmd_ev.mem, captured_frame, - bindings): + with state.ctx_FrontFrame(self.mem, captured_frame, bindings): unused_status = self.cmd_ev.EvalCommandFrag(cmd) return value.Dict(bindings) @@ -105,18 +105,21 @@ def Call(self, rd): class CaptureStdout(vm._Callable): - def __init__(self, shell_ex): - # type: (vm._Executor) -> None + def __init__(self, mem, shell_ex): + # type: (state.Mem, vm._Executor) -> None + self.mem = mem self.shell_ex = shell_ex def Call(self, rd): # type: (typed_args.Reader) -> value_t unused = rd.PosValue() - cmd = rd.PosCommandFrag() # TODO: Use bound command? + cmd = rd.PosCommand() rd.Done() # no more args - status, stdout_str = self.shell_ex.CaptureStdout(cmd) + frag = typed_args.GetCommandFrag(cmd) + with state.ctx_FrontFrame(self.mem, cmd.captured_frame, None): + status, stdout_str = self.shell_ex.CaptureStdout(frag) if status != 0: # Note that $() raises error.ErrExit with the status. # But I think that results in a more confusing error message, so we diff --git a/core/shell.py b/core/shell.py index cab6d47c51..3949ef932c 100644 --- a/core/shell.py +++ b/core/shell.py @@ -567,13 +567,13 @@ def Main( # The M/ prefix means it's io->eval() io_methods['M/eval'] = value.BuiltinFunc( - method_io.Eval(cmd_ev, method_io.EVAL_NULL)) + method_io.Eval(mem, cmd_ev, method_io.EVAL_NULL)) io_methods['M/evalToDict'] = value.BuiltinFunc( - method_io.Eval(cmd_ev, method_io.EVAL_DICT)) + method_io.Eval(mem, cmd_ev, method_io.EVAL_DICT)) # Identical to command sub io_methods['captureStdout'] = value.BuiltinFunc( - method_io.CaptureStdout(shell_ex)) + method_io.CaptureStdout(mem, shell_ex)) # TODO: io_methods['time'] = value.BuiltinFunc(method_io.Time()) diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh index a1d0fd32eb..5537129e45 100644 --- a/spec/ysh-blocks.test.sh +++ b/spec/ysh-blocks.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 #### cd accepts a block, runs it in different dir shopt -s ysh:all @@ -83,7 +83,7 @@ i = 42 #### io->eval() and io.captureStdout() passed a block in different scope shopt --set ysh:upgrade -proc my-cd (; b) { +proc my-eval (; b) { call io->eval(b) var d = io->evalToDict(b) @@ -99,12 +99,16 @@ proc p { var i = 42 var b = ^(var x = 'x'; echo "i = $i") - my-cd (b) + my-eval (b) } p ## STDOUT: +i = 42 +i = 42 +(Dict) {"x":"x"} +stdout i = 42 ## END #### block doesn't have its own scope From 6ea4be14c1761e71ea0b44bba6167089df78523b Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 14 Oct 2024 01:54:54 -0400 Subject: [PATCH 333/506] [ysh] Migrate hay and shopt to Command, over CommandFrag --- builtin/func_hay.py | 6 +++--- builtin/pure_osh.py | 4 ++-- frontend/typed_args.py | 2 +- spec/ysh-blocks.test.sh | 30 ++++++++++++++++++------------ 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/builtin/func_hay.py b/builtin/func_hay.py index e1ee9f41c6..c2c49838cb 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -92,10 +92,10 @@ def __init__( self.cmd_ev = cmd_ev def _Call(self, cmd): - # type: (command_t) -> Dict[str, value_t] + # type: (value.Command) -> Dict[str, value_t] with hay_ysh.ctx_HayEval(self.hay_state, self.mutable_opts, self.mem): - unused = self.cmd_ev.EvalCommandFrag(cmd) + unused = self.cmd_ev.EvalCommand(cmd) return self.hay_state.Result() @@ -105,7 +105,7 @@ def _Call(self, cmd): def Call(self, rd): # type: (typed_args.Reader) -> value_t - cmd = rd.PosCommandFrag() + cmd = rd.PosCommand() rd.Done() return value.Dict(self._Call(cmd)) diff --git a/builtin/pure_osh.py b/builtin/pure_osh.py index c007d77abe..0bb9fc1cb6 100644 --- a/builtin/pure_osh.py +++ b/builtin/pure_osh.py @@ -230,7 +230,7 @@ def Run(self, cmd_val): return 0 # shopt --set x { my-block } - cmd = typed_args.OptionalBlock(cmd_val) + cmd = typed_args.OptionalCommandBlock(cmd_val) if cmd: opt_nums = [] # type: List[int] for opt_name in opt_names: @@ -256,7 +256,7 @@ def Run(self, cmd_val): opt_nums.append(index) with state.ctx_Option(self.mutable_opts, opt_nums, b): - unused = self.cmd_ev.EvalCommandFrag(cmd) + unused = self.cmd_ev.EvalCommand(cmd) return 0 # cd also returns 0 # Otherwise, set options. diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 92dc9d6c07..8b5371835a 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -362,7 +362,7 @@ def _ToCommandFrag(self, val): if val.tag() == value_e.CommandFrag: return cast(value.CommandFrag, val).c - # io.eval(mycmd) uses this + # TODO: remove this. Many builtin commands rely on it. if val.tag() == value_e.Command: bound = cast(value.Command, val) return GetCommandFrag(bound) diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh index 5537129e45..30fc80dd5d 100644 --- a/spec/ysh-blocks.test.sh +++ b/spec/ysh-blocks.test.sh @@ -111,20 +111,26 @@ i = 42 stdout i = 42 ## END -#### block doesn't have its own scope -shopt -s ysh:all -var x = 1 -echo "x=$x" -cd / { - #set y = 5 # This would be an error because set doesn't do dynamic lookup - var x = 42 - echo "x=$x" +#### builtins like shopt with block arg +shopt --set ysh:upgrade + +proc my-eval (; b) { + shopt --unset nounset (; ; b) + #shopt --unset errexit (; ; b) } -echo "x=$x" + +proc p { + var i = 42 + var b = ^(var x = 'x'; echo "i = $i, undef = [$undef]") + + my-eval (b) +} + +p + + ## STDOUT: -x=1 -x=42 -x=42 +i = 42, undef = [] ## END #### redirects allowed in words, typed args, and after block From ae4fa93140932ae00541e2204565206a71d8e781 Mon Sep 17 00:00:00 2001 From: Andy C Date: Mon, 14 Oct 2024 10:39:08 -0400 Subject: [PATCH 334/506] [test/lint] Fix build --- builtin/func_hay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/func_hay.py b/builtin/func_hay.py index c2c49838cb..ac1d5322d7 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -2,7 +2,7 @@ """func_hay.py.""" from __future__ import print_function -from _devbuild.gen.syntax_asdl import source, loc, command_t +from _devbuild.gen.syntax_asdl import source, loc from _devbuild.gen.value_asdl import value, cmd_frag from builtin import hay_ysh from core import alloc From dbd65c215dbaa237121668c868c7bf66d41f3dee Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 11:59:55 -0400 Subject: [PATCH 335/506] [builtin/cd] Revert scope change cd doesn't have lexical scope, it's more like an "inline proc", which all builtins will probably be This is because we want to be able to use variables after the block: cd /tmp { var listing = $(ls -x -y -z) } echo $listing --- builtin/dirs_osh.py | 4 ++-- demo/survey-closure.sh | 9 ++++++++- spec/ysh-blocks.test.sh | 41 +++++++++++++++-------------------------- spec/ysh-scope.test.sh | 9 +++++---- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/builtin/dirs_osh.py b/builtin/dirs_osh.py index 15ccabaa32..811fe272b7 100644 --- a/builtin/dirs_osh.py +++ b/builtin/dirs_osh.py @@ -102,7 +102,7 @@ def Run(self, cmd_val): arg = arg_types.cd(attrs.attrs) # If a block is passed, we do additional syntax checks - cmd = typed_args.OptionalCommandBlock(cmd_val) + cmd = typed_args.OptionalBlock(cmd_val) dest_dir, arg_loc = arg_r.Peek2() if dest_dir is None: @@ -166,7 +166,7 @@ def Run(self, cmd_val): out_errs = [] # type: List[bool] with ctx_CdBlock(self.dir_stack, real_dest_dir, self.mem, self.errfmt, out_errs): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) if len(out_errs): return 1 diff --git a/demo/survey-closure.sh b/demo/survey-closure.sh index ae0fd4298c..df26fc2b71 100755 --- a/demo/survey-closure.sh +++ b/demo/survey-closure.sh @@ -455,9 +455,16 @@ puts result puts +g = 9 # visible + block2 = lambda do |x| - x * factor + x * factor * g + h = 20 end + +# Not visible, but in YSH we may want it to be, e.g. for try { } and shopt { } +# puts h + puts block2.call(5) result = run_it(&block2) diff --git a/spec/ysh-blocks.test.sh b/spec/ysh-blocks.test.sh index 30fc80dd5d..a7008f68dd 100644 --- a/spec/ysh-blocks.test.sh +++ b/spec/ysh-blocks.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 0 +## oils_failures_allowed: 2 #### cd accepts a block, runs it in different dir shopt -s ysh:all @@ -77,13 +77,12 @@ proc p { p ## STDOUT: -i = 42 ## END #### io->eval() and io.captureStdout() passed a block in different scope shopt --set ysh:upgrade -proc my-eval (; b) { +proc my-cd (; b) { call io->eval(b) var d = io->evalToDict(b) @@ -99,38 +98,28 @@ proc p { var i = 42 var b = ^(var x = 'x'; echo "i = $i") - my-eval (b) + my-cd (b) } p ## STDOUT: -i = 42 -i = 42 -(Dict) {"x":"x"} -stdout i = 42 ## END -#### builtins like shopt with block arg -shopt --set ysh:upgrade - -proc my-eval (; b) { - shopt --unset nounset (; ; b) - #shopt --unset errexit (; ; b) -} - -proc p { - var i = 42 - var b = ^(var x = 'x'; echo "i = $i, undef = [$undef]") - - my-eval (b) +#### block doesn't have its own scope +shopt -s ysh:all +var x = 1 +echo "x=$x" +cd / { + #set y = 5 # This would be an error because set doesn't do dynamic lookup + var x = 42 + echo "x=$x" } - -p - - +echo "x=$x" ## STDOUT: -i = 42, undef = [] +x=1 +x=42 +x=42 ## END #### redirects allowed in words, typed args, and after block diff --git a/spec/ysh-scope.test.sh b/spec/ysh-scope.test.sh index 65165844fc..901b91c1a4 100644 --- a/spec/ysh-scope.test.sh +++ b/spec/ysh-scope.test.sh @@ -550,8 +550,8 @@ inline FOO= bar ## END -#### cd blocks introduce new scopes -shopt --set ysh:upgrade +#### cd blocks don't introduce new scopes +shopt --set oil:upgrade var x = 42 cd / { @@ -560,11 +560,12 @@ cd / { echo $x $y $z setvar y = 43 } -echo $x $[getVar('y')] $[getVar('z')] +setvar z = 44 +echo $x $y $z ## STDOUT: 42 0 1 -42 null null +42 43 44 ## END #### IFS=: myproc exports when it doesn't need to From 9fcc21af82a4268e824a94c00274d1c2e4086eb9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 12:02:20 -0400 Subject: [PATCH 336/506] Revert "[ysh] Migrate hay and shopt to Command, over CommandFrag" This reverts commit 6ea4be14c1761e71ea0b44bba6167089df78523b. Also fix lint error. --- builtin/func_hay.py | 9 ++++----- builtin/pure_osh.py | 4 ++-- frontend/typed_args.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/builtin/func_hay.py b/builtin/func_hay.py index ac1d5322d7..b2ec8b3f3e 100644 --- a/builtin/func_hay.py +++ b/builtin/func_hay.py @@ -1,8 +1,7 @@ #!/usr/bin/env python2 -"""func_hay.py.""" from __future__ import print_function -from _devbuild.gen.syntax_asdl import source, loc +from _devbuild.gen.syntax_asdl import source, loc, command_t from _devbuild.gen.value_asdl import value, cmd_frag from builtin import hay_ysh from core import alloc @@ -92,10 +91,10 @@ def __init__( self.cmd_ev = cmd_ev def _Call(self, cmd): - # type: (value.Command) -> Dict[str, value_t] + # type: (command_t) -> Dict[str, value_t] with hay_ysh.ctx_HayEval(self.hay_state, self.mutable_opts, self.mem): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) return self.hay_state.Result() @@ -105,7 +104,7 @@ def _Call(self, cmd): def Call(self, rd): # type: (typed_args.Reader) -> value_t - cmd = rd.PosCommand() + cmd = rd.PosCommandFrag() rd.Done() return value.Dict(self._Call(cmd)) diff --git a/builtin/pure_osh.py b/builtin/pure_osh.py index 0bb9fc1cb6..c007d77abe 100644 --- a/builtin/pure_osh.py +++ b/builtin/pure_osh.py @@ -230,7 +230,7 @@ def Run(self, cmd_val): return 0 # shopt --set x { my-block } - cmd = typed_args.OptionalCommandBlock(cmd_val) + cmd = typed_args.OptionalBlock(cmd_val) if cmd: opt_nums = [] # type: List[int] for opt_name in opt_names: @@ -256,7 +256,7 @@ def Run(self, cmd_val): opt_nums.append(index) with state.ctx_Option(self.mutable_opts, opt_nums, b): - unused = self.cmd_ev.EvalCommand(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd) return 0 # cd also returns 0 # Otherwise, set options. diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 8b5371835a..92dc9d6c07 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -362,7 +362,7 @@ def _ToCommandFrag(self, val): if val.tag() == value_e.CommandFrag: return cast(value.CommandFrag, val).c - # TODO: remove this. Many builtin commands rely on it. + # io.eval(mycmd) uses this if val.tag() == value_e.Command: bound = cast(value.Command, val) return GetCommandFrag(bound) From 3f52f6283f7a9e312bd08e04a822757f6d173b2c Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 12:08:24 -0400 Subject: [PATCH 337/506] [refactor] Introduce typed_args.RequiredBlock() I think the API should be: typed_args.RequiredBlockFrag() typed_args.OptionalBlockFrag() --- builtin/dirs_osh.py | 8 +++---- builtin/func_reflect.py | 7 ++++-- builtin/io_ysh.py | 12 +++------- builtin/method_io.py | 22 ++++++++++++++++-- core/shell.py | 6 +++-- frontend/typed_args.py | 50 +++++++++++++++++++++++++---------------- 6 files changed, 67 insertions(+), 38 deletions(-) diff --git a/builtin/dirs_osh.py b/builtin/dirs_osh.py index 811fe272b7..df77898ec6 100644 --- a/builtin/dirs_osh.py +++ b/builtin/dirs_osh.py @@ -102,11 +102,11 @@ def Run(self, cmd_val): arg = arg_types.cd(attrs.attrs) # If a block is passed, we do additional syntax checks - cmd = typed_args.OptionalBlock(cmd_val) + cmd_frag = typed_args.OptionalBlock(cmd_val) dest_dir, arg_loc = arg_r.Peek2() if dest_dir is None: - if cmd: + if cmd_frag: raise error.Usage( 'requires an argument when a block is passed', cmd_val.arg_locs[0]) @@ -162,11 +162,11 @@ def Run(self, cmd_val): # PWD. Other shells use global variables. self.mem.SetPwd(real_dest_dir) - if cmd: + if cmd_frag: out_errs = [] # type: List[bool] with ctx_CdBlock(self.dir_stack, real_dest_dir, self.mem, self.errfmt, out_errs): - unused = self.cmd_ev.EvalCommandFrag(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd_frag) if len(out_errs): return 1 diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index 13236817e4..3cd7b090a7 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -61,7 +61,7 @@ def Call(self, rd): raise AssertionError() -class ThisFrame(vm._Callable): +class GetFrame(vm._Callable): def __init__(self, mem): # type: (state.Mem) -> None @@ -70,11 +70,14 @@ def __init__(self, mem): def Call(self, rd): # type: (typed_args.Reader) -> value_t + index = rd.PosInt() rd.Done() + + # TODO: 0 is global, -1 is current, -2 is parent return value.Frame(self.mem.CurrentFrame()) -class BindCommand(vm._Callable): +class BindFrame(vm._Callable): def __init__(self): # type: () -> None diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index efdf71e587..cb4b41de0a 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -293,10 +293,7 @@ class RunBlock(vm._Builtin): """Used for 'redir' builtin It's used solely for its redirects. - fopen >out.txt { echo hi } - - It's a subset of eval - eval >out.txt { echo hi } + redir >out.txt { echo hi } """ def __init__(self, mem, cmd_ev): @@ -310,9 +307,6 @@ def Run(self, cmd_val): cmd_val, accept_typed_args=True) - cmd = typed_args.OptionalBlock(cmd_val) - if not cmd: - raise error.Usage('expected a block', loc.Missing) - - unused = self.cmd_ev.EvalCommandFrag(cmd) + cmd_frag = typed_args.RequiredBlock(cmd_val) + unused = self.cmd_ev.EvalCommandFrag(cmd_frag) return 0 diff --git a/builtin/method_io.py b/builtin/method_io.py index 604387c3e2..ceafe9ef24 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -34,6 +34,26 @@ def _PrintFrame(prefix, frame): _PrintFrame('--> ' + prefix, r.frame) +class EvalInFrame(vm._Callable): + """ + For making "inline procs" + """ + + def __init__(self, mem, cmd_ev): + # type: (state.Mem, cmd_eval.CommandEvaluator) -> None + self.mem = mem + self.cmd_ev = cmd_ev + + def Call(self, rd): + # type: (typed_args.Reader) -> value_t + frag = rd.PosCommandFrag() + bound = rd.PosFrame() + + # TODO: EvalCommandFrag() + + return value.Null + + class Eval(vm._Callable): """ These are similar: @@ -45,8 +65,6 @@ class Eval(vm._Callable): call io->evalToDict(cmd) - TODO: remove eval (c) - The CALLER must handle errors. """ diff --git a/core/shell.py b/core/shell.py index 3949ef932c..27333fca2d 100644 --- a/core/shell.py +++ b/core/shell.py @@ -570,6 +570,8 @@ def Main( method_io.Eval(mem, cmd_ev, method_io.EVAL_NULL)) io_methods['M/evalToDict'] = value.BuiltinFunc( method_io.Eval(mem, cmd_ev, method_io.EVAL_DICT)) + io_methods['M/evalInFrame'] = value.BuiltinFunc( + method_io.EvalInFrame(mem, cmd_ev)) # Identical to command sub io_methods['captureStdout'] = value.BuiltinFunc( @@ -882,8 +884,8 @@ def Main( # not bind a frame yet # # what about newFrame() and globalFrame()? - _AddBuiltinFunc(mem, 'thisFrame', func_reflect.ThisFrame(mem)) - _AddBuiltinFunc(mem, 'bindCommand', func_reflect.BindCommand()) + _AddBuiltinFunc(mem, 'getFrame', func_reflect.GetFrame(mem)) + _AddBuiltinFunc(mem, 'bindFrame', func_reflect.BindFrame()) _AddBuiltinFunc(mem, 'Object', func_misc.Object()) _AddBuiltinFunc(mem, 'prototype', func_misc.Prototype()) diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 92dc9d6c07..7caaf1bce6 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -25,26 +25,37 @@ def DoesNotAccept(proc_args): e_usage('got unexpected typed args', proc_args.typed_args.left) -def OptionalCommandBlock(cmd_val): - # type: (cmd_value.Argv) -> Optional[value.Command] +if 0: + def OptionalCommandBlock(cmd_val): + # type: (cmd_value.Argv) -> Optional[value.Command] + """ + Unused, the builtins don't take value.Command - they take a command_t CommandFrag + """ + cmd = None # type: Optional[value.Command] + if cmd_val.proc_args: + r = ReaderForProc(cmd_val) + cmd = r.OptionalCommandBlock() + r.Done() + return cmd - cmd = None # type: Optional[value.Command] - if cmd_val.proc_args: - r = ReaderForProc(cmd_val) - cmd = r.OptionalCommandBlock() - r.Done() + +def OptionalBlock(cmd_val): + # type: (cmd_value.Argv) -> Optional[command_t] + """Helper for cd, etc.""" + + r = ReaderForProc(cmd_val) + cmd = r.OptionalBlock() + r.Done() return cmd -def OptionalBlock(cmd_val): +def RequiredBlock(cmd_val): # type: (cmd_value.Argv) -> Optional[command_t] - """Helper for shopt, etc.""" + """Helper for try, shopt, etc.""" - cmd = None # type: Optional[command_t] - if cmd_val.proc_args: - r = ReaderForProc(cmd_val) - cmd = r.OptionalBlock() - r.Done() + r = ReaderForProc(cmd_val) + cmd = r.RequiredBlock() + r.Done() return cmd @@ -493,11 +504,12 @@ def PosExpr(self): # Block arg # - def OptionalCommandBlock(self): - # type: () -> Optional[value.Command] - if self.block_arg is None: - return None - return self._ToCommand(self.block_arg) + if 0: + def OptionalCommandBlock(self): + # type: () -> Optional[value.Command] + if self.block_arg is None: + return None + return self._ToCommand(self.block_arg) def RequiredBlock(self): # type: () -> command_t From 0b83066c379e3cdcf259531062b7179a053da9ec Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 12:30:36 -0400 Subject: [PATCH 338/506] [test/spec] Fix tests, add cases for "inline proc" reflection --- spec/ysh-builtin-eval.test.sh | 35 ++++++++++++++++++++++++++++++++++- spec/ysh-func-builtin.test.sh | 22 +++++++++++++++++++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 7ae45e7f25..6e4d767e3b 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -1,7 +1,7 @@ # YSH specific features of eval ## our_shell: ysh -## oils_failures_allowed: 3 +## oils_failures_allowed: 4 #### eval builtin does not take a literal block - can restore this later @@ -651,3 +651,36 @@ x: i = 0, j = 2 x: i = 1, j = 3 x: i = 2, j = 4 ## END + + + +#### io->evalInFrame() can express try, cd builtins + +var frag = ^(echo $i) + +proc my-cd (new_dir; ; ; block) { + pushd $new_dir + + # could call this "unbound"? or unbind()? What about procs and funcs and + # exprs? + var frag = getCommandFrag(block) + + var calling_frame = getFrame(-2) + call io->evalInFrame(frag, calling_frame) + + popd +} + +var i = 42 +my-cd /tmp { + echo $PWD + var j = i + 1 +} +echo "j = $j" + +## STDOUT: +x: i = 0, j = 2 +x: i = 1, j = 3 +x: i = 2, j = 4 +## END + diff --git a/spec/ysh-func-builtin.test.sh b/spec/ysh-func-builtin.test.sh index 9a81ccc488..d2b1791f9b 100644 --- a/spec/ysh-func-builtin.test.sh +++ b/spec/ysh-func-builtin.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 3 ## our_shell: ysh #### join() @@ -182,9 +182,11 @@ echo $[y => lower()] áé ## END -#### thisFrame() +#### getFrame() -var fr = thisFrame() +# TODO: vm.getFrame() + +var fr = getFrame(null) pp test_ (fr) #= fr @@ -195,3 +197,17 @@ pp test_ (fr) ## END + +#### bindFrame() + +var frag = ^(echo $i) + +# TODO: should be fragment +pp test_ (frag) + +var cmd = bindFrame(frag, getFrame(0)) + +pp test_ (cmd) + +## STDOUT: +## END From a27b5685b29e18d9fb15daaf3a7010bf98c869c0 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 12:36:53 -0400 Subject: [PATCH 339/506] [errors] Change RequiredBlock error to UsageError, status 2 Not TypeError, status 3. This gives a better error message. --- builtin/hay_ysh.py | 4 +--- builtin/io_ysh.py | 2 +- frontend/typed_args.py | 6 ++++-- test/ysh-runtime-errors.sh | 25 +++++++++++++++++++++++++ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/builtin/hay_ysh.py b/builtin/hay_ysh.py index 4049dd0abb..93f2c7215e 100644 --- a/builtin/hay_ysh.py +++ b/builtin/hay_ysh.py @@ -284,9 +284,7 @@ def Run(self, cmd_val): var_name = var_name[1:] # TODO: This could be fatal? - cmd = typed_args.OptionalBlock(cmd_val) - if not cmd: # 'package foo' is OK - e_usage('eval expected a block', loc.Missing) + cmd = typed_args.RequiredBlock(cmd_val) with ctx_HayEval(self.hay_state, self.mutable_opts, self.mem): # Note: we want all haynode invocations in the block to appear as diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index cb4b41de0a..798061bf57 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -6,7 +6,7 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value -from _devbuild.gen.syntax_asdl import command_e, BraceGroup, loc +from _devbuild.gen.syntax_asdl import command_e, BraceGroup from _devbuild.gen.value_asdl import value, value_e, value_t from asdl import format as fmt from core import error diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 7caaf1bce6..5ff96b7f57 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -26,6 +26,7 @@ def DoesNotAccept(proc_args): if 0: + def OptionalCommandBlock(cmd_val): # type: (cmd_value.Argv) -> Optional[value.Command] """ @@ -505,6 +506,7 @@ def PosExpr(self): # if 0: + def OptionalCommandBlock(self): # type: () -> Optional[value.Command] if self.block_arg is None: @@ -514,8 +516,8 @@ def OptionalCommandBlock(self): def RequiredBlock(self): # type: () -> command_t if self.block_arg is None: - raise error.TypeErrVerbose('Expected a block arg', - self.LeastSpecificLocation()) + raise error.Usage('expected a block arg', + self.LeastSpecificLocation()) return self._ToCommandFrag(self.block_arg) def OptionalBlock(self): diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 7dbfae1787..0c60b71440 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -1008,6 +1008,31 @@ test-module() { _ysh-error-X 3 'use spec/testdata/module2/util2.ysh; util2 badObj otherproc' } +test-required-blocks() { + + # These are procs, which normally give usage errors + # The usage error prints the builtin name + # + # Funcs give you type errors though? Is that inconsistent? + + _ysh-error-X 2 'redir' + _ysh-error-X 2 'redir (42)' + _ysh-error-X 2 'hay eval :myvar' + _ysh-error-X 2 'hay eval :myvar (42)' + _ysh-error-X 2 'try' + _ysh-error-X 2 'ctx push ({})' + + _ysh-error-X 2 'haynode Foo' + + # Hm this isn't a usage error + _ysh-error-X 3 'haynode Foo (42)' + + # This neither + _ysh-error-X 3 'haynode Foo ( ; ; 42)' + + _ysh-should-run 'haynode Foo a { echo hi }' +} + soil-run-py() { run-test-funcs } From 14943459bb21f540da0a0cfafa814b248baf5e51 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 12:44:11 -0400 Subject: [PATCH 340/506] [refactor] Migrate some builtins to typed_args.RequiredBlock() --- builtin/process_osh.py | 14 ++++---------- builtin/pure_osh.py | 6 +++--- builtin/pure_ysh.py | 16 ++++++---------- test/ysh-runtime-errors.sh | 11 ++++++++++- 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/builtin/process_osh.py b/builtin/process_osh.py index 24c3a3ac62..c8d9eede41 100644 --- a/builtin/process_osh.py +++ b/builtin/process_osh.py @@ -159,11 +159,8 @@ def Run(self, cmd_val): if arg is not None: e_usage('got unexpected argument %r' % arg, location) - cmd = typed_args.OptionalBlock(cmd_val) - if cmd is None: - e_usage('expected a block', loc.Missing) - - return self.shell_ex.RunBackgroundJob(cmd) + cmd_frag = typed_args.RequiredBlock(cmd_val) + return self.shell_ex.RunBackgroundJob(cmd_frag) class ForkWait(vm._Builtin): @@ -181,11 +178,8 @@ def Run(self, cmd_val): if arg is not None: e_usage('got unexpected argument %r' % arg, location) - cmd = typed_args.OptionalBlock(cmd_val) - if cmd is None: - e_usage('expected a block', loc.Missing) - - return self.shell_ex.RunSubshell(cmd) + cmd_frag = typed_args.RequiredBlock(cmd_val) + return self.shell_ex.RunSubshell(cmd_frag) class Exec(vm._Builtin): diff --git a/builtin/pure_osh.py b/builtin/pure_osh.py index c007d77abe..7832fdd6af 100644 --- a/builtin/pure_osh.py +++ b/builtin/pure_osh.py @@ -230,8 +230,8 @@ def Run(self, cmd_val): return 0 # shopt --set x { my-block } - cmd = typed_args.OptionalBlock(cmd_val) - if cmd: + cmd_frag = typed_args.OptionalBlock(cmd_val) + if cmd_frag: opt_nums = [] # type: List[int] for opt_name in opt_names: # TODO: could consolidate with checks in core/state.py and option @@ -256,7 +256,7 @@ def Run(self, cmd_val): opt_nums.append(index) with state.ctx_Option(self.mutable_opts, opt_nums, b): - unused = self.cmd_ev.EvalCommandFrag(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd_frag) return 0 # cd also returns 0 # Otherwise, set options. diff --git a/builtin/pure_ysh.py b/builtin/pure_ysh.py index 8fe247e3a1..b544a5d1b9 100644 --- a/builtin/pure_ysh.py +++ b/builtin/pure_ysh.py @@ -35,11 +35,9 @@ def Run(self, cmd_val): cmd_val, accept_typed_args=True) - cmd = typed_args.OptionalBlock(cmd_val) - if not cmd: - # TODO: I think shvar LANG=C should just mutate - # But should there be a whitelist? - raise error.Usage('expected a block', loc.Missing) + # TODO: I think shvar LANG=C should just mutate + # But should there be a whitelist? + cmd_frag = typed_args.RequiredBlock(cmd_val) vars = NewDict() # type: Dict[str, value_t] args, arg_locs = arg_r.Rest2() @@ -58,7 +56,7 @@ def Run(self, cmd_val): self.search_path.ClearCache() with state.ctx_Eval(self.mem, None, None, vars): - unused = self.cmd_ev.EvalCommandFrag(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd_frag) return 0 @@ -176,12 +174,10 @@ def Run(self, cmd_val): cmd_val, accept_typed_args=True) - cmd = typed_args.OptionalBlock(cmd_val) - if not cmd: - raise error.Usage('expected a block', loc.Missing) + cmd_frag = typed_args.RequiredBlock(cmd_val) with state.ctx_Registers(self.mem): - unused = self.cmd_ev.EvalCommandFrag(cmd) + unused = self.cmd_ev.EvalCommandFrag(cmd_frag) # make it "SILENT" in terms of not mutating $? # TODO: Revisit this. It might be better to provide the headless shell diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 0c60b71440..4eeac1c8a0 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -908,7 +908,7 @@ test-append-usage-error() { # Bad error location test-try-usage-error() { - _ysh-expr-error ' + _ysh-error-X 2 ' var s = "README" case (s) { README { echo hi } @@ -1015,6 +1015,9 @@ test-required-blocks() { # # Funcs give you type errors though? Is that inconsistent? + _ysh-error-X 2 'shvar' + _ysh-error-X 2 'push-registers' + _ysh-error-X 2 'redir' _ysh-error-X 2 'redir (42)' _ysh-error-X 2 'hay eval :myvar' @@ -1022,6 +1025,12 @@ test-required-blocks() { _ysh-error-X 2 'try' _ysh-error-X 2 'ctx push ({})' + _ysh-error-X 2 'fork' + _ysh-error-X 2 'forkwait' + + # OK this is a type error + _ysh-error-X 3 'forkwait ( ; ; 42)' + _ysh-error-X 2 'haynode Foo' # Hm this isn't a usage error From d5e3a5c6a11f7b492c98e962224b9bc22ed6d1c9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 12:51:34 -0400 Subject: [PATCH 341/506] [spec/ysh-builtin-error] Remove duplicate test This was adjusted in test/ysh-runtime-errors.sh --- frontend/typed_args.py | 8 +++++++- spec/ysh-builtin-error.test.sh | 9 --------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 5ff96b7f57..82c49d3545 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -374,7 +374,13 @@ def _ToCommandFrag(self, val): if val.tag() == value_e.CommandFrag: return cast(value.CommandFrag, val).c - # io.eval(mycmd) uses this + # Builtins like shopt, cd, try rely on this, because proc argument + # evaluation gives you a value.Command, yet they operate on a + # CommandFrag. + # + # In YSH, we do this with the getCommandFrag() builtin, which returns + # an UNBOUND version of the command. Hm. + if val.tag() == value_e.Command: bound = cast(value.Command, val) return GetCommandFrag(bound) diff --git a/spec/ysh-builtin-error.test.sh b/spec/ysh-builtin-error.test.sh index 7c5031c389..c83ba8f955 100644 --- a/spec/ysh-builtin-error.test.sh +++ b/spec/ysh-builtin-error.test.sh @@ -2,15 +2,6 @@ ## our_shell: ysh -#### try requires an argument - -try -echo status=$? - -## status: 3 -## STDOUT: -## END - #### User errors behave like builtin errors func divide(a, b) { if (b === 0) { From 75a1c123f2e72c31c1f2aa22aa1ad15f3d60b58f Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 13:08:24 -0400 Subject: [PATCH 342/506] [rename] ctx_FrontFrame -> ctx_EnclosedFrame And __rear__ -> __E, the enclosing frame. The lookup is changed. --- builtin/method_io.py | 10 +++++----- core/state.py | 28 +++++++++++++++------------- osh/cmd_eval.py | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index ceafe9ef24..4c8923f8b2 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -26,7 +26,7 @@ def _PrintFrame(prefix, frame): # type: (str, Dict[str, Cell]) -> None print('%s %s' % (prefix, ' '.join(frame.keys()))) - rear = frame.get('__rear__') + rear = frame.get('__E') if rear: rear_val = rear.val if rear_val.tag() == value_e.Frame: @@ -102,7 +102,7 @@ def Call(self, rd): if self.which == EVAL_NULL: # _PrintFrame('[captured]', captured_frame) - with state.ctx_FrontFrame(self.mem, captured_frame, None): + with state.ctx_EnclosedFrame(self.mem, captured_frame, None): # _PrintFrame('[new]', self.cmd_ev.mem.var_stack[-1]) with state.ctx_Eval(self.mem, dollar0, pos_args, vars_): unused_status = self.cmd_ev.EvalCommandFrag(cmd) @@ -110,10 +110,10 @@ def Call(self, rd): elif self.which == EVAL_DICT: # TODO: dollar0, pos_args, vars_ not supported - # Does ctx_FrontFrame has different scoping rules? For "vars"? + # Does ctx_EnclosedFrame has different scoping rules? For "vars"? bindings = NewDict() # type: Dict[str, value_t] - with state.ctx_FrontFrame(self.mem, captured_frame, bindings): + with state.ctx_EnclosedFrame(self.mem, captured_frame, bindings): unused_status = self.cmd_ev.EvalCommandFrag(cmd) return value.Dict(bindings) @@ -136,7 +136,7 @@ def Call(self, rd): rd.Done() # no more args frag = typed_args.GetCommandFrag(cmd) - with state.ctx_FrontFrame(self.mem, cmd.captured_frame, None): + with state.ctx_EnclosedFrame(self.mem, cmd.captured_frame, None): status, stdout_str = self.shell_ex.CaptureStdout(frag) if status != 0: # Note that $() raises error.ErrExit with the status. diff --git a/core/state.py b/core/state.py index bafc0cbece..3db1b341b0 100644 --- a/core/state.py +++ b/core/state.py @@ -1165,8 +1165,8 @@ def __init__(self, mem, name1): if self.do_new_frame: rear_frame = self.mem.var_stack[-1] self.front_frame = NewDict() # type: Dict[str, Cell] - self.front_frame['__rear__'] = Cell(False, False, False, - value.Frame(rear_frame)) + self.front_frame['__E'] = Cell(False, False, False, + value.Frame(rear_frame)) mem.var_stack.append(self.front_frame) def __enter__(self): @@ -1179,10 +1179,13 @@ def __exit__(self, type, value, traceback): self.mem.var_stack.pop() -class ctx_FrontFrame(object): +class ctx_EnclosedFrame(object): """ - For use by io->evalToDict(), which is a primitive used for Hay and the Dict - proc + Usages: + + - io->evalToDict(), which is a primitive used for Hay and the Dict proc + - lexical scope aka static scope for block args to user-defined procs + - Including the "closures in a loop" problem, which will be used for Hay var mutated = 'm' var shadowed = 's' @@ -1205,10 +1208,10 @@ def __init__(self, mem, rear_frame, out_dict): self.rear_frame = rear_frame self.out_dict = out_dict - # __rear__ gets a lookup rule + # __E gets a lookup rule self.front_frame = NewDict() # type: Dict[str, Cell] - self.front_frame['__rear__'] = Cell(False, False, False, - value.Frame(rear_frame)) + self.front_frame['__E'] = Cell(False, False, False, + value.Frame(rear_frame)) mem.var_stack.append(self.front_frame) @@ -1240,7 +1243,7 @@ class ctx_ModuleEval(object): e.g. setglobal in the new module doesn't leak - Different from ctx_FrontFrame because the new code can't see variables in + Different from ctx_EnclosedFrame because the new code can't see variables in the old frame. """ @@ -1378,15 +1381,14 @@ def _Pop(self): def _FrameLookup(frame, name): # type: (Dict[str, Cell], str) -> Tuple[Optional[Cell], Dict[str, Cell]] """ - Look in the frame itself, then the __rear__ frame if it exists - - TODO: Need to recursively look at __rear__ + Look for a name in the frame, then recursively into the enclosing __E + frame, if it exists """ cell = frame.get(name) if cell: return cell, frame - rear_cell = frame.get('__rear__') # ctx_FrontFrame() sets this + rear_cell = frame.get('__E') # ctx_EnclosedFrame() sets this if rear_cell: rear_val = rear_cell.val assert rear_val, rear_val diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 3b12990f67..96181be201 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -2096,7 +2096,7 @@ def EvalCommandFrag(self, frag): def EvalCommand(self, cmd): # type: (value.Command) -> int frag = typed_args.GetCommandFrag(cmd) - with state.ctx_FrontFrame(self.mem, cmd.captured_frame, None): + with state.ctx_EnclosedFrame(self.mem, cmd.captured_frame, None): return self.EvalCommandFrag(frag) def RunTrapsOnExit(self, mut_status): From 031220a438d11de9a83a9279c2f674325663a77d Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 13:14:00 -0400 Subject: [PATCH 343/506] [rename] OptionalBlock() -> OptionalBlockAsFrag() Likewise for Required* To clarify that we're dealing with "unbound" blocks in builtins. --- builtin/dirs_osh.py | 2 +- builtin/error_ysh.py | 2 +- builtin/hay_ysh.py | 2 +- builtin/io_ysh.py | 2 +- builtin/process_osh.py | 4 ++-- builtin/pure_osh.py | 2 +- builtin/pure_ysh.py | 6 +++--- frontend/typed_args.py | 12 ++++++------ 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/builtin/dirs_osh.py b/builtin/dirs_osh.py index df77898ec6..ae7c6fb682 100644 --- a/builtin/dirs_osh.py +++ b/builtin/dirs_osh.py @@ -102,7 +102,7 @@ def Run(self, cmd_val): arg = arg_types.cd(attrs.attrs) # If a block is passed, we do additional syntax checks - cmd_frag = typed_args.OptionalBlock(cmd_val) + cmd_frag = typed_args.OptionalBlockAsFrag(cmd_val) dest_dir, arg_loc = arg_r.Peek2() if dest_dir is None: diff --git a/builtin/error_ysh.py b/builtin/error_ysh.py index ecd6ee39ed..fd439ee683 100644 --- a/builtin/error_ysh.py +++ b/builtin/error_ysh.py @@ -96,7 +96,7 @@ def Run(self, cmd_val): accept_typed_args=True) rd = typed_args.ReaderForProc(cmd_val) - cmd = rd.RequiredBlock() + cmd = rd.RequiredBlockAsFrag() rd.Done() error_dict = None # type: value.Dict diff --git a/builtin/hay_ysh.py b/builtin/hay_ysh.py index 93f2c7215e..9675ca0be8 100644 --- a/builtin/hay_ysh.py +++ b/builtin/hay_ysh.py @@ -284,7 +284,7 @@ def Run(self, cmd_val): var_name = var_name[1:] # TODO: This could be fatal? - cmd = typed_args.RequiredBlock(cmd_val) + cmd = typed_args.RequiredBlockAsFrag(cmd_val) with ctx_HayEval(self.hay_state, self.mutable_opts, self.mem): # Note: we want all haynode invocations in the block to appear as diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index 798061bf57..f20be00c51 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -307,6 +307,6 @@ def Run(self, cmd_val): cmd_val, accept_typed_args=True) - cmd_frag = typed_args.RequiredBlock(cmd_val) + cmd_frag = typed_args.RequiredBlockAsFrag(cmd_val) unused = self.cmd_ev.EvalCommandFrag(cmd_frag) return 0 diff --git a/builtin/process_osh.py b/builtin/process_osh.py index c8d9eede41..443b207da8 100644 --- a/builtin/process_osh.py +++ b/builtin/process_osh.py @@ -159,7 +159,7 @@ def Run(self, cmd_val): if arg is not None: e_usage('got unexpected argument %r' % arg, location) - cmd_frag = typed_args.RequiredBlock(cmd_val) + cmd_frag = typed_args.RequiredBlockAsFrag(cmd_val) return self.shell_ex.RunBackgroundJob(cmd_frag) @@ -178,7 +178,7 @@ def Run(self, cmd_val): if arg is not None: e_usage('got unexpected argument %r' % arg, location) - cmd_frag = typed_args.RequiredBlock(cmd_val) + cmd_frag = typed_args.RequiredBlockAsFrag(cmd_val) return self.shell_ex.RunSubshell(cmd_frag) diff --git a/builtin/pure_osh.py b/builtin/pure_osh.py index 7832fdd6af..b3c3286951 100644 --- a/builtin/pure_osh.py +++ b/builtin/pure_osh.py @@ -230,7 +230,7 @@ def Run(self, cmd_val): return 0 # shopt --set x { my-block } - cmd_frag = typed_args.OptionalBlock(cmd_val) + cmd_frag = typed_args.OptionalBlockAsFrag(cmd_val) if cmd_frag: opt_nums = [] # type: List[int] for opt_name in opt_names: diff --git a/builtin/pure_ysh.py b/builtin/pure_ysh.py index b544a5d1b9..d68857ee96 100644 --- a/builtin/pure_ysh.py +++ b/builtin/pure_ysh.py @@ -37,7 +37,7 @@ def Run(self, cmd_val): # TODO: I think shvar LANG=C should just mutate # But should there be a whitelist? - cmd_frag = typed_args.RequiredBlock(cmd_val) + cmd_frag = typed_args.RequiredBlockAsFrag(cmd_val) vars = NewDict() # type: Dict[str, value_t] args, arg_locs = arg_r.Rest2() @@ -135,7 +135,7 @@ def Run(self, cmd_val): if verb == "push": context = rd.PosDict() - block = rd.RequiredBlock() + block = rd.RequiredBlockAsFrag() rd.Done() arg_r.AtEnd() @@ -174,7 +174,7 @@ def Run(self, cmd_val): cmd_val, accept_typed_args=True) - cmd_frag = typed_args.RequiredBlock(cmd_val) + cmd_frag = typed_args.RequiredBlockAsFrag(cmd_val) with state.ctx_Registers(self.mem): unused = self.cmd_ev.EvalCommandFrag(cmd_frag) diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 82c49d3545..4ef3ee65c7 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -40,22 +40,22 @@ def OptionalCommandBlock(cmd_val): return cmd -def OptionalBlock(cmd_val): +def OptionalBlockAsFrag(cmd_val): # type: (cmd_value.Argv) -> Optional[command_t] """Helper for cd, etc.""" r = ReaderForProc(cmd_val) - cmd = r.OptionalBlock() + cmd = r.OptionalBlockAsFrag() r.Done() return cmd -def RequiredBlock(cmd_val): +def RequiredBlockAsFrag(cmd_val): # type: (cmd_value.Argv) -> Optional[command_t] """Helper for try, shopt, etc.""" r = ReaderForProc(cmd_val) - cmd = r.RequiredBlock() + cmd = r.RequiredBlockAsFrag() r.Done() return cmd @@ -519,14 +519,14 @@ def OptionalCommandBlock(self): return None return self._ToCommand(self.block_arg) - def RequiredBlock(self): + def RequiredBlockAsFrag(self): # type: () -> command_t if self.block_arg is None: raise error.Usage('expected a block arg', self.LeastSpecificLocation()) return self._ToCommandFrag(self.block_arg) - def OptionalBlock(self): + def OptionalBlockAsFrag(self): # type: () -> Optional[command_t] if self.block_arg is None: return None From 641fcc268f3098bfd7ade87e3b04354e9a0c6ae8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 13:23:15 -0400 Subject: [PATCH 344/506] [core] Rename enclosing scope from __E to __E__ The Hay evalToDict() rule of "hidden vars" relies on s.endswith('_') Because the _ prefix is for registers, like _error and _match() --- builtin/method_io.py | 2 +- core/state.py | 10 +++++----- spec/ysh-builtin-eval.test.sh | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/builtin/method_io.py b/builtin/method_io.py index 4c8923f8b2..c1b452b0ad 100644 --- a/builtin/method_io.py +++ b/builtin/method_io.py @@ -26,7 +26,7 @@ def _PrintFrame(prefix, frame): # type: (str, Dict[str, Cell]) -> None print('%s %s' % (prefix, ' '.join(frame.keys()))) - rear = frame.get('__E') + rear = frame.get('__E__') if rear: rear_val = rear.val if rear_val.tag() == value_e.Frame: diff --git a/core/state.py b/core/state.py index 3db1b341b0..59917985c2 100644 --- a/core/state.py +++ b/core/state.py @@ -1165,7 +1165,7 @@ def __init__(self, mem, name1): if self.do_new_frame: rear_frame = self.mem.var_stack[-1] self.front_frame = NewDict() # type: Dict[str, Cell] - self.front_frame['__E'] = Cell(False, False, False, + self.front_frame['__E__'] = Cell(False, False, False, value.Frame(rear_frame)) mem.var_stack.append(self.front_frame) @@ -1208,9 +1208,9 @@ def __init__(self, mem, rear_frame, out_dict): self.rear_frame = rear_frame self.out_dict = out_dict - # __E gets a lookup rule + # __E__ gets a lookup rule self.front_frame = NewDict() # type: Dict[str, Cell] - self.front_frame['__E'] = Cell(False, False, False, + self.front_frame['__E__'] = Cell(False, False, False, value.Frame(rear_frame)) mem.var_stack.append(self.front_frame) @@ -1381,14 +1381,14 @@ def _Pop(self): def _FrameLookup(frame, name): # type: (Dict[str, Cell], str) -> Tuple[Optional[Cell], Dict[str, Cell]] """ - Look for a name in the frame, then recursively into the enclosing __E + Look for a name in the frame, then recursively into the enclosing __E__ frame, if it exists """ cell = frame.get(name) if cell: return cell, frame - rear_cell = frame.get('__E') # ctx_EnclosedFrame() sets this + rear_cell = frame.get('__E__') # ctx_EnclosedFrame() sets this if rear_cell: rear_val = rear_cell.val assert rear_val, rear_val diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index 6e4d767e3b..faf3f97782 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -597,7 +597,7 @@ echo g=$g #pp frame_vars_ ## STDOUT: - [frame_vars_] __rear__ a + [frame_vars_] __E__ a (Dict) {"a":42} g=zz From cd417835a6a26cc9940bb80ae4a3ed171531d20d Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 20:20:29 -0400 Subject: [PATCH 345/506] [stdlib/ysh/stream] Got some things working, grep-like test cases It seems like it's working! Testing out io->eval(pos_args=, vars=) I think io->evalToDict() might be different, and won't take those arguments. Both of them take a value.Command, not a CommandFrag. --- spec/ysh-proc.test.sh | 1 + stdlib/ysh/stream.ysh | 114 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 103 insertions(+), 12 deletions(-) diff --git a/spec/ysh-proc.test.sh b/spec/ysh-proc.test.sh index 3d76d1cbbc..acc05cbfef 100644 --- a/spec/ysh-proc.test.sh +++ b/spec/ysh-proc.test.sh @@ -794,6 +794,7 @@ sum = 53 #### Stateful proc with counter shopt --set ysh:upgrade + proc invokeCounter(; self, inc) { setvar self.i += inc echo "counter = $[self.i]" diff --git a/stdlib/ysh/stream.ysh b/stdlib/ysh/stream.ysh index 0aa86b8787..102c7f8f3e 100644 --- a/stdlib/ysh/stream.ysh +++ b/stdlib/ysh/stream.ysh @@ -11,12 +11,26 @@ source $LIB_OSH/byo-server.sh source $LIB_YSH/args.ysh proc slurp-by (; num_lines) { - # TODO: (stdin) - for line in (stdin) { - echo TODO + var buf = [] + for line in (io.stdin) { + call buf->append(line) + if (len(buf) === num_lines) { + json write (buf, space=0) + + # TODO: + #call buf->clear() + setvar buf = [] + } + } + if (buf) { + json write (buf, space=0) } } +proc test-slurp-by { + seq 8 | slurp-by (3) +} + # Note: # - these are all the same algorithm # - also word, block, etc. are all optional @@ -46,33 +60,109 @@ proc test-each-line { # ysh-tool test stream.ysh # # Col - - } proc each-row (; ; block) { echo TODO } -proc split-by (; ifs=null; block) { - echo TODO +proc split-by (; delim; ifs=null; block) { + + # TODO: provide the option to bind names? Or is that a separate thing? + # The output of this is "ragged" + + for line in (io.stdin) { + #pp (line) + var parts = line.split(delim) + pp (parts) + + # variable number + call io->eval(block, dollar0=line, pos_args=parts) + } } -proc if-split-by (; ifs=null; block) { +proc chop () { + ### alias for split-by echo TODO } -proc chop () { - ### alias for if-split-by +proc test-split-by { + var z = 'z' # test out scoping + var count = 0 # test out mutation + + # TODO: need split by space + # Where the leading and trailing are split + # if-split-by(' ') doesn't work well + + line-data | split-by (/s+/) { + + # how do we deal with nonexistent? + # should we also bind _parts or _words? + + echo "$z | $0 | $1 | $z" + + setvar count += 1 + } + echo "count = $count" +} + +proc must-split-by (; ; ifs=null; block) { + ### like if-split-by + echo TODO } +proc if-match (; pattern; ; block) { + ### like 'grep' but with submatches + + for line in (io.stdin) { + var m = line.search(pattern) + if (m) { + #pp asdl_ (m) + #var groups = m.groups() + + # Should we also pass _line? + call io->eval(block, dollar0=m.group(0)) + } + } + + # always succeeds - I think must-match is the one that can fail +} + proc must-match (; pattern; block) { + ### like if-match + echo TODO } -proc if-match (; pattern; block) { - echo TODO +proc line-data { + # note: trailing ''' issue, I should probably get rid of the last line + + echo ''' + prefix 30 foo + oils + /// 42 bar''' +} + +proc test-if-match { + var z = 'z' # test out scoping + var count = 0 # test out mutation + + # Test cases should be like: + # grep: print the matches, or just count them + # sed: print a new line based on submatches + # awk: re-arrange the cols, and also accumulate counters + + var pat = / s+ / + line-data | if-match (pat) { + echo "$z $0 $z" + # TODO: need pos_args + + #echo "-- $2 $1 --" + + setvar count += 1 + } + echo "count = $count" } # Protocol: From ed2e2ac0731158f32c346c0a7a6d28b36ca442db Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 15 Oct 2024 23:01:04 -0400 Subject: [PATCH 346/506] [doc] Document (( vs ( ( parsing issue And clarify $(( vs $( ( as well --- doc/known-differences.md | 44 ++++++++++++++++++++++--- stdlib/ysh/stream.ysh | 69 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/doc/known-differences.md b/doc/known-differences.md index 9f4c76c9cf..b3c8fb1d1e 100644 --- a/doc/known-differences.md +++ b/doc/known-differences.md @@ -88,12 +88,14 @@ Undecidable](https://www.oilshell.org/blog/2016/10/20.html) (2016). - [OILS-ERR-101](error-catalog.html#oils-err-101) explains more ways to fix this. -### Subshell in command sub +### Subshell in command sub - `$((` versus `$( (` -You can have a subshell in a command sub, but it usually doesn't make sense. +You can have a subshell `(` in a command sub `$(`, but it usually doesn't make +sense. -In OSH you need a space after `$(`. The characters `$((` always start an -arith sub. +In OSH you need a space after `$(`, so it would be `$( (`. + +characters `$((` always start an arith sub. No: @@ -105,6 +107,40 @@ Yes: $({ cd / && ls; }) # Use {} for grouping, not (). Note trailing ; $(cd / && ls) # Even better +### Nested Subshells - `((` versus `( (` + +You should never need nested subshells with `((` in Bourne shell or Oils. + +If you do, you should add a space with `( (` instead of `((`, similar to the +issue above. + +In OSH, `((` always starts bash-style arithmetic. + +--- + +The only place I see `((` arise is when shell users try to use `( )` to mean +**grouping**, because they are used to C or Python. + +But it means **subshell**, not grouping. In shell, `{ }` is the way to group +commands. + +No: + + if ((test -f a || test -f b) && grep foo c); then + echo ok + fi + +Allowed, but not what you want: + + if ( (test -f a || test -f b) && grep foo c); then + echo ok + fi + +Yes: + + if { test -f a || test -f b; } && grep foo c; then + echo ok + fi ### Extended glob vs. Negation of boolean expression diff --git a/stdlib/ysh/stream.ysh b/stdlib/ysh/stream.ysh index 102c7f8f3e..a790822f0a 100644 --- a/stdlib/ysh/stream.ysh +++ b/stdlib/ysh/stream.ysh @@ -31,9 +31,30 @@ proc test-slurp-by { seq 8 | slurp-by (3) } -# Note: -# - these are all the same algorithm -# - also word, block, etc. are all optional +### Awk + +# Naming +# +# TEXT INPUT +# each-word # this doesn't go by lines, it does a global regex split or something? +# +# LINE INPUT +# each-line --j8 { echo "-- $_line" } # similar to @() +# each-line --j8 (^"-- $_line") # is this superfluous? +# +# each-split name1 name2 +# (delim=' ') +# (ifs=' ') +# (pat=/d+/) +# # also assign names for each part? +# +# each-match # regex match +# must-match # assert that every line matches +# +# TABLE INPUT +# each-row # TSV and TSV8 input? +# +# They all take templates or blocks? proc each-line (...words; template=null; ; block=null) { # TODO: @@ -112,7 +133,9 @@ proc must-split-by (; ; ifs=null; block) { echo TODO } -proc if-match (; pattern; ; block) { +# Naming: each-match, each-split? + +proc if-match (; pattern, template=null; ; block=null) { ### like 'grep' but with submatches for line in (io.stdin) { @@ -122,7 +145,14 @@ proc if-match (; pattern; ; block) { #var groups = m.groups() # Should we also pass _line? - call io->eval(block, dollar0=m.group(0)) + + if (block) { + call io->eval(block, dollar0=m.group(0)) + } elif (template) { + echo TEMPLATE + } else { + echo TSV + } } } @@ -144,6 +174,8 @@ proc line-data { /// 42 bar''' } +const pat = / s+ / + proc test-if-match { var z = 'z' # test out scoping var count = 0 # test out mutation @@ -153,7 +185,6 @@ proc test-if-match { # sed: print a new line based on submatches # awk: re-arrange the cols, and also accumulate counters - var pat = / s+ / line-data | if-match (pat) { echo "$z $0 $z" # TODO: need pos_args @@ -165,6 +196,32 @@ proc test-if-match { echo "count = $count" } +proc test-if-match-2 { + # If there's no block or template, it should print out a TSV with: + # + # $0 ... + # $1 $2 + # $_line maybe? + + #line-data | if-match (pat) + + var z = 'z' # scoping + line-data | if-match (pat, ^"$z $0 $z") + line-data | if-match (pat, ^"-- $0 --") +} + +# might be a nice way to write it, not sure if byo.sh can discover it +if false { +tests 'if-match' { + proc case-block { + echo TODO + } + proc case-template { + echo TODO + } +} +} + # Protocol: # # - The file lists its tests the "actions" From 11f8dffa3c4975c602b9ea8723dc4f408a1bb23f Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 16 Oct 2024 12:46:16 -0400 Subject: [PATCH 347/506] [shopt -s strict_errexit] Allow && || ! expressions i.e. compound conditionals Grouping with { } is still an issue. This can be done with chained if statements for now. [doc] Intro to process model doc [doc] YSH FAQ: "subshells by surprise" and placeholder for test --true --false --- doc/process-model.md | 71 +++++++++++++++++++++++++++++++++------- doc/ysh-faq.md | 39 ++++++++++++++++++++++ osh/cmd_eval.py | 8 +++++ spec/errexit-osh.test.sh | 56 +++++++++++++++++++++++++++++++ test/runtime-errors.sh | 4 +-- 5 files changed, 165 insertions(+), 13 deletions(-) diff --git a/doc/process-model.md b/doc/process-model.md index a0994d231c..3aab2b24c5 100644 --- a/doc/process-model.md +++ b/doc/process-model.md @@ -2,10 +2,34 @@ in_progress: yes --- -Process Model +The Unix Shell Process Model - When Are Processes Created? ============= -Why does a Unix shell start processes? How many processes are started? +OSH and YSH are both extensions of POSIX shell, and share its underlying "process model". + +Each Unix process has its **own** memory, that is not shared with other +processes. (It's created by `fork()`, which means that the memory is +"copy-on-write".) + +Understanding when a shell starts processes will make you a better shell +programmer. + +As a concrete example, here is some code that behaves differently in +[bash]($xref) and [zsh]($xref): + + + $ bash -c 'echo hi | read x; echo x=$x' + x= + + $ zsh -c 'echo hi | read x; echo x=$x' + x=hi + +If you understand why they are different, then that means you understand the +process model! + +(OSH behaves like zsh.) + +--- Related: [Interpreter State](interpreter-state.html). These two docs are the missing documentation for shell! @@ -15,14 +39,14 @@ missing documentation for shell! ## Shell Constructs That Start Processes -### Pipelines +### Pipelines `myproc | wc -l` - `shopt -s lastpipe` - `set -o pipefail` -#### Functions Can Be Transparently Put in Pipelines +Note that functions Can Be Transparently Put in Pipelines: -Implicit subshell: +Hidden subshell: { echo 1; echo 2; } | wc -l @@ -44,8 +68,31 @@ Explicit Subshells are Rarely Needed. - prefer `pushd` / `popd`, or `cd { }` in YSH. + +## FAQ: "Subshells By Surprise" + +Sometimes subshells have no syntax. + +Common issues: + +### shopt -s lastpipe + +Mentioned in the intro: + + $ bash -c 'echo hi | read x; echo x=$x' + x= + + $ zsh -c 'echo hi | read x; echo x=$x' + x=hi + +### Other Pipelines + + myproc (&p) | grep foo + ## Process Optimizations - `noforklast` +Why does a Unix shell start processes? How many processes are started? + Bugs / issues - job control: @@ -64,11 +111,11 @@ Oils/YSH specific: - because we don't get to test if it failed - stats / tracing - counting exit codes + ## Process State ### Redirects - ## Builtins ### [wait]($help) @@ -82,9 +129,11 @@ Oils/YSH specific: ## Appendix: Non-Shell Tools -- `xargs` and `xargs -P` +These Unix tools start processes: + +- `xargs` + - `xargs -P` starts parallel processes (but doesn't buffer output) - `find -exec` -- `make -j` - - doesn't do anything smart with output -- `ninja` - - buffers output too +- `make` + - `make -j` starts parallel processes (but doesn't buffer output) +- `ninja` (buffers output) diff --git a/doc/ysh-faq.md b/doc/ysh-faq.md index a100f5d6a7..39cccd650a 100644 --- a/doc/ysh-faq.md +++ b/doc/ysh-faq.md @@ -205,6 +205,45 @@ not `${}`. --> +## How do I combine conditional commands and expressions: `if (myvar)` versus `if test`? + +TODO: `test --true --false` + +This happens in `while` too. + +## Why do I lose the value of `p` in `myproc (&p) | grep foo`? + +In a pipeline, most components are **forked**. This means that `myproc (&p)` +runs in a different process from the main shell. + +The main shell can't see the memory of a subshell. + +--- + +In general, you have to restructure your code to avoid this. You could use a proc with multiple outputs: + + myproc (&p, &grepped_output) + +Or you could use a function: + + var out1, out2 = myfunc(io) + +--- + +[The Unix Shell Process Model - When Are Processes +Created?](process-model.html) may help. + +This issue is similar to the `shopt -s lastpipe` issue: + + $ bash -c 'echo hi | read x; echo x=$x' + x= + + $ zsh -c 'echo hi | read x; echo x=$x' + x=hi + +In bash, `read` runs in a subshell, but in `zsh` and OSH, it runs in the main +shell. + ## Related - [Oil Language FAQ]($wiki) on the wiki has more answers. They may be migrated diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 96181be201..3bc82ca452 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -168,6 +168,14 @@ def _HasManyStatuses(node): # Multiple parts like 'ls | wc' is disallowed return True + elif case(command_e.AndOr): + node = cast(command.AndOr, UP_node) + for c in node.children: + if _HasManyStatuses(c): + return True + return False # otherwise allow 'if true && true; ...' + + # - ShAssignment could be allowed, though its exit code will always be # 0 without command subs # - Naively, (non-singleton) pipelines could be allowed because pipefail. diff --git a/spec/errexit-osh.test.sh b/spec/errexit-osh.test.sh index 72f0f77c1b..5cd035818f 100644 --- a/spec/errexit-osh.test.sh +++ b/spec/errexit-osh.test.sh @@ -123,6 +123,62 @@ fi yes ## END +#### strict_errexit with && || ! +set -o errexit +shopt -s strict_errexit || true + +if true && true; then + echo A +fi + +if true || false; then + echo B +fi + +if ! false && ! false; then + echo C +fi + +## STDOUT: +A +B +C +## END + +#### strict_errexit detects proc in && || ! +set -o errexit +shopt -s strict_errexit || true + +myfunc() { + echo 'failing' + false + echo 'should not get here' +} + +if true && ! myfunc; then + echo B +fi + +if ! myfunc; then + echo A +fi + +## status: 1 +## STDOUT: +## END + +# POSIX shell behavior: + +## OK bash/dash/mksh/ash status: 0 +## OK bash/dash/mksh/ash STDOUT: +failing +should not get here +failing +should not get here +## END + + + #### strict_errexit without errexit proc myproc() { echo myproc diff --git a/test/runtime-errors.sh b/test/runtime-errors.sh index 859edd5c52..bbeeb2aaf9 100755 --- a/test/runtime-errors.sh +++ b/test/runtime-errors.sh @@ -387,7 +387,7 @@ done # OLD WAY OF BLAMING # Note: most of these don't fail -test-strict_errexit_old() { +test-strict-errexit-old() { # Test out all the location info # command.Pipeline. @@ -398,7 +398,7 @@ test-strict_errexit_old() { #_strict-errexit-case 'if ! ls; then echo Pipeline; fi' # command.AndOr - _strict-errexit-case 'if echo a && echo b; then echo AndOr; fi' + #_strict-errexit-case 'if echo a && echo b; then echo AndOr; fi' # command.DoGroup _strict-errexit-case '! for x in a; do echo $x; done' From a2cd18a59fcd147a59c97bfefea1f3de0eddd636 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 16 Oct 2024 13:19:42 -0400 Subject: [PATCH 348/506] [builtin/test] Implement test --true; test --false The other part of issue #2094. Based on feedback from Will Clardy. Also add a FAQ entry on this, since Julian also ran into it. --- builtin/bracket_osh.py | 4 ++ doc/ref/chap-builtin-cmd.md | 35 ++++++++++++--- doc/ref/toc-ysh.md | 1 + doc/ysh-faq.md | 17 ++++++-- frontend/id_kind_def.py | 18 +++++--- osh/sh_expr_eval.py | 4 ++ spec/ysh-builtins.test.sh | 86 +++++++++++++++++++++++++++++++++++++ 7 files changed, 150 insertions(+), 15 deletions(-) diff --git a/builtin/bracket_osh.py b/builtin/bracket_osh.py index c95b89a5b8..4500dca1a3 100644 --- a/builtin/bracket_osh.py +++ b/builtin/bracket_osh.py @@ -127,6 +127,10 @@ def _TwoArgs(w_parser): unary_id = Id.BoolUnary_f elif s0 == '--symlink': unary_id = Id.BoolUnary_L + elif s0 == '--true': + unary_id = Id.BoolUnary_true + elif s0 == '--false': + unary_id = Id.BoolUnary_false if unary_id == Id.Undefined_Tok: unary_id = match.BracketUnary(w0.s) diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index fa0915c1aa..d37cfdabae 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -513,6 +513,30 @@ See the [YSH FAQ][echo-en] for details. [simple_echo]: chap-option.html#ysh:all [echo-en]: ../ysh-faq.html#how-do-i-write-the-equivalent-of-echo-e-or-echo-n +### ysh-test + +The YSH [test](#test) builtin supports these long flags: + + --dir same as -d + --exists same as -e + --file same as -f + --symlink same as -L + + --true Is the argument equal to the string "true"? + --false Is the argument equal to the string "false"? + +The `--true` and `--false` flags can be used to combine commands and +expressions: + + if test --file a && test --true $[bool(mydict)] { + echo ok + } + +This works because the boolean `true` *stringifies* to `"true"`, and likewise +with `false`. + +That is, `$[true] === "true"` and `$[false] === "false"`. + ### write write fixes problems with shell's `echo` builtin. @@ -1108,8 +1132,8 @@ JOB: Evaluates a conditional expression and returns 0 (true) or 1 (false). -Note that [ is the name of a builtin, not an operator in the language. Use -'test' to avoid this confusion. +Note that `[` is the name of a builtin, not an operator in the language. Use +`test` to avoid this confusion. String expressions: @@ -1168,12 +1192,9 @@ these are discouraged. -Oils supports these long flags: +--- - --dir same as -d - --exists same as -e - --file same as -f - --symlink same as -L +See [ysh-test](#ysh-test) for log flags like `--file` and `--true`. ### getopts diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 3425a1e21d..ba121ef74b 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -126,6 +126,7 @@ X [Wok] _field() use create a module Obj from a source file [I/O] ysh-read flags --all, -0 ysh-echo no -e -n with simple_echo + ysh-test --file --true etc. write Like echo, with --, --sep, --end fork forkwait Replace & and (), and takes a block fopen Open multiple streams, takes a block diff --git a/doc/ysh-faq.md b/doc/ysh-faq.md index 39cccd650a..ad167343dd 100644 --- a/doc/ysh-faq.md +++ b/doc/ysh-faq.md @@ -205,11 +205,22 @@ not `${}`. --> -## How do I combine conditional commands and expressions: `if (myvar)` versus `if test`? +## How do I combine conditional commands and expressions: `if (myvar)` and `if test -f`? -TODO: `test --true --false` +You can use the `--true` and `--false` flags to the [YSH test][ysh-test] +builtin: + + if test --true $[myvar] && test --file x { + echo ok + } + +They test if their argument is literally the string `"true"` or `"false"`. + +This works because the boolean `true` *stringifies* to `"true"`, and likewise +with `false`. + +[ysh-test]: ref/chap-builtin-cmd.html#ysh-test -This happens in `while` too. ## Why do I lose the value of `p` in `myproc (&p) | grep foo`? diff --git a/frontend/id_kind_def.py b/frontend/id_kind_def.py index 21a9a8e523..02726b76d6 100755 --- a/frontend/id_kind_def.py +++ b/frontend/id_kind_def.py @@ -109,10 +109,10 @@ def AddBoolKind( ): # type: (...) -> None """ - Args: - kind_name: string - arg_type_pairs: dictionary of bool_arg_type_e -> [] - """ + Args: + kind_name: string + arg_type_pairs: dictionary of bool_arg_type_e -> [] + """ lexer_pairs = [] num_tokens = 0 for arg_type, pairs in arg_type_pairs: @@ -733,6 +733,15 @@ def AddBoolKinds(spec): (bool_arg_type_e.Path, _Dash(list(_UNARY_PATH_CHARS))), ]) + Id = spec.id_str2int + + # test --true and test --false have no single letter flags. They need no + # lexing. + for long_flag in ('true', 'false'): + id_name = 'BoolUnary_%s' % long_flag + spec._AddId(id_name) + spec.AddBoolOp(Id[id_name], bool_arg_type_e.Str) + spec.AddBoolKind('BoolBinary', [ (bool_arg_type_e.Str, [ ('GlobEqual', '='), @@ -744,7 +753,6 @@ def AddBoolKinds(spec): (bool_arg_type_e.Int, _Dash(_BINARY_INT)), ]) - Id = spec.id_str2int # logical, arity, arg_type spec.AddBoolOp(Id['Op_DAmp'], bool_arg_type_e.Undefined) spec.AddBoolOp(Id['Op_DPipe'], bool_arg_type_e.Undefined) diff --git a/osh/sh_expr_eval.py b/osh/sh_expr_eval.py index be93ac5b8e..1ccbfac230 100644 --- a/osh/sh_expr_eval.py +++ b/osh/sh_expr_eval.py @@ -1112,6 +1112,10 @@ def EvalB(self, node): return not bool(s) if op_id == Id.BoolUnary_n: return bool(s) + if op_id == Id.BoolUnary_true: + return s == 'true' + if op_id == Id.BoolUnary_false: + return s == 'false' raise AssertionError(op_id) # should never happen diff --git a/spec/ysh-builtins.test.sh b/spec/ysh-builtins.test.sh index 7511cc5b2d..1fbfc32b6c 100644 --- a/spec/ysh-builtins.test.sh +++ b/spec/ysh-builtins.test.sh @@ -435,6 +435,92 @@ status=1 status=2 ## END +#### test --true; test --false +shopt --set ysh:upgrade + +for expr in (true, false, '', 'other') { + pp test_ (expr) + + try { + test --true $[expr] + } + echo true=$[_error.code] + + try { + test --false $[expr] + } + echo false=$[_error.code] + echo +} + +## STDOUT: +(Bool) true +true=0 +false=1 + +(Bool) false +true=1 +false=0 + +(Str) "" +true=1 +false=1 + +(Str) "other" +true=1 +false=1 + +## END + +#### More test --true --false +shopt --set ysh:upgrade + +var d = {} + +try { + test --true $[bool(d)] +} +echo dict=$[_error.code] + +setvar d.key = 'val' + +try { + test --true $[bool(d)] +} +echo dict=$[_error.code] + +echo + +if test --true $[bool(d)] && ! test -f / { + echo AndOr +} + +## STDOUT: +dict=1 +dict=0 + +AndOr +## END + + +#### Make sure [[ is not affected by --true --false + +set +o errexit + +$SH +o ysh:all -c '[[ --true ]]; echo dbracket=$?' +$SH +o ysh:all -c '[[ --false ]]; echo dbracket=$?' + +$SH +o ysh:all -c '[[ --true true ]]; echo dbracket=$?' +echo "parse error $?" +$SH +o ysh:all -c '[[ --false false ]]; echo dbracket=$?' +echo "parse error $?" + +## STDOUT: +dbracket=0 +dbracket=0 +parse error 2 +parse error 2 +## END #### push-registers shopt --set ysh:upgrade From 1a7e2e32d470de0ef842edfa07899bbeefef0762 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 16 Oct 2024 17:18:03 -0400 Subject: [PATCH 349/506] [builtin/test] Disallow typed args Based on feedback from Will Clardy --- builtin/bracket_osh.py | 3 +++ test/ysh-runtime-errors.sh | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/builtin/bracket_osh.py b/builtin/bracket_osh.py index 4500dca1a3..3b7f6ce817 100644 --- a/builtin/bracket_osh.py +++ b/builtin/bracket_osh.py @@ -10,6 +10,7 @@ from core.error import e_usage, p_die from core import vm from frontend import match +from frontend import typed_args from mycpp.mylib import log from osh import bool_parse from osh import sh_expr_eval @@ -189,6 +190,8 @@ def Run(self, cmd_val): The only difference between test and [ is that [ needs a matching ]. """ + typed_args.DoesNotAccept(cmd_val.proc_args) # Disallow test (42) + if self.need_right_bracket: # Preprocess right bracket if self.exec_opts.simple_test_builtin(): e_usage("should be invoked as 'test' (simple_test_builtin)", diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 4eeac1c8a0..583371c6a1 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -15,12 +15,17 @@ source test/sh-assert.sh # _assert-sh-status # test-no-typed-args() { - # Hm these could both be J8 notation - #_ysh-error-1 'echo (42)' - #_ysh-error-1 'write (42)' + _ysh-error-X 2 'echo (42)' + _ysh-error-X 2 'echo { echo hi }' + + # Hm write could be J8 notation? like json8 write (x)? + _ysh-error-X 2 'write (42)' _ysh-error-X 2 'true (42)' _ysh-error-X 2 'false { echo hi }' + + _ysh-error-X 2 'test x (42)' + _ysh-error-X 2 'test x { echo hi }' } test-undefined-vars() { From 3e22e818372f35be87ccb1895f664eb9d2fc4c16 Mon Sep 17 00:00:00 2001 From: Andy C Date: Wed, 16 Oct 2024 22:01:30 -0400 Subject: [PATCH 350/506] [doc/ref] Document new reflection - evalToDict() - Frame - Command, CommandFrag - Expr, ExprFrag - not implemented Also sketch out reflection - vm.getFrame(-1) - Might still need to change parseCommand() and parseHay() - could be parseString() parseFile()? - and parseExpr() or parseExprString() --- builtin/func_reflect.py | 4 ++ core/shell.py | 14 +++-- doc/ref/chap-type-method.md | 65 ++++++++++++++------- doc/ref/toc-ysh.md | 6 +- spec/ysh-builtin-eval.test.sh | 107 ++++++++++++++++++---------------- 5 files changed, 119 insertions(+), 77 deletions(-) diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index 3cd7b090a7..e160aaca08 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -70,6 +70,7 @@ def __init__(self, mem): def Call(self, rd): # type: (typed_args.Reader) -> value_t + unused_self = rd.PosObj() index = rd.PosInt() rd.Done() @@ -85,6 +86,9 @@ def __init__(self): def Call(self, rd): # type: (typed_args.Reader) -> value_t + + # TODO: also take an ExprFrag -> Expr + frag = rd.PosCommandFrag() frame = rd.PosFrame() rd.Done() diff --git a/core/shell.py b/core/shell.py index 27333fca2d..c6cdc1d035 100644 --- a/core/shell.py +++ b/core/shell.py @@ -585,6 +585,11 @@ def Main( io_props = {'stdin': value.Stdin} # type: Dict[str, value_t] io_obj = Obj(Obj(None, io_methods), io_props) + vm_methods = {} # type: Dict[str, value_t] + vm_methods['getFrame'] = value.BuiltinFunc(func_reflect.GetFrame(mem)) + vm_props = {} # type: Dict[str, value_t] + vm_obj = Obj(Obj(None, vm_methods), vm_props) + # Wire up circular dependencies. vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, prompt_ev, io_obj, tracer) @@ -869,6 +874,7 @@ def Main( _AddBuiltinFunc(mem, '_end', func_eggex.MatchFunc(func_eggex.E, None, mem)) _AddBuiltinFunc(mem, 'id', func_reflect.Id()) + # TODO: should this be parseCommandStr() vs. parseFile() for Hay? _AddBuiltinFunc(mem, 'parseCommand', func_reflect.ParseCommand(parse_ctx, mem, errfmt)) _AddBuiltinFunc(mem, 'parseExpr', @@ -879,12 +885,11 @@ def Main( _AddBuiltinFunc(mem, 'getVar', func_reflect.GetVar(mem)) _AddBuiltinFunc(mem, 'setVar', func_reflect.SetVar(mem)) - # TODO: implement - # and then parseCommand() and parseHay will not depend on mem; they will - # not bind a frame yet + # TODO: implement bindFrame() to turn CommandFrag -> Command + # Then parseCommand() and parseHay() will not depend on mem; they will not + # bind a frame yet # # what about newFrame() and globalFrame()? - _AddBuiltinFunc(mem, 'getFrame', func_reflect.GetFrame(mem)) _AddBuiltinFunc(mem, 'bindFrame', func_reflect.BindFrame()) _AddBuiltinFunc(mem, 'Object', func_misc.Object()) @@ -935,6 +940,7 @@ def Main( _AddBuiltinFunc(mem, '_opsp', func_misc.SparseOp()) mem.AddBuiltin('io', io_obj) + mem.AddBuiltin('vm', vm_obj) # Special case for testing mem.AddBuiltin('module-invoke', value.BuiltinProc(module_invoke)) diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 3b089913bc..484c3710b4 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -453,6 +453,22 @@ A Place is used as an "out param" by calling setValue(): ## Code Types +### Command + +An unevaluated command. You can create a `Command` with a "block expression" +([block-expr][]): + + var block = ^(echo $PWD; ls *.txt) + +The Command is bound to a stack frame. This frame will be pushed as an +"enclosed frame" when the command is evaluated. + +[block-expr]: chap-expr-lang.html#block-expr + +### CommandFrag + +A command that's not bound to a stack frame. + ### Expr An unevaluated expression. You can create an `Expr` with an expression literal @@ -460,16 +476,16 @@ An unevaluated expression. You can create an `Expr` with an expression literal var expr = ^[42 + a[i]] -[expr-literal]: chap-expr-lang.html#expr-lit +The Command is bound to a stack frame. This frame will be pushed as an +"enclosed frame" when the expression is evaluated. -### Command +[expr-literal]: chap-expr-lang.html#expr-lit -An unevaluated command. You can create a `Command` with a "block expression" -([block-expr][]): +### ExprFrag - var block = ^(echo $PWD; ls *.txt) +An expression command that's not bound to a stack frame. -[block-expr]: chap-expr-lang.html#block-expr +(TODO) ### BuiltinFunc @@ -485,6 +501,13 @@ The [thin-arrow][] and [fat-arrow][] create bound funcs: [thin-arrow]: chap-expr-lang.html#thin-arrow [fat-arrow]: chap-expr-lang.html#thin-arrow +### Frame + +A value that represents a stack frame. It can be bound to a `CommandFrag`, +producing a `Command`. + +Likewise, it can be found to a `ExprFrag`, producing an `Expr`. + ## Func User-defined functions. @@ -493,14 +516,6 @@ User-defined functions. User-defined procs. -## Module - -TODO: - -A module is a file with YSH code. - - - ## IO ### eval() @@ -510,7 +525,7 @@ Evaluate a command, and return `null`. var cmd = ^(echo hi) call io->eval(cmd) -It's like like the `eval` builtin, and meant to be used in pure functions. +It's similar to the `eval` builtin, and is meant to be used in pure functions. You can also bind: @@ -529,20 +544,26 @@ TODO: We should be able to bind positional args, env vars, and inspect the shell VM. Though this runs in the same VM, not a new one. - - --> ### evalToDict() -The `evalToDict()` method is like the `eval()` method, but it also returns a +The `evalToDict()` method is like the `eval()` method, but it returns a Dict of bindings. -TODO: +It pushes a new "enclosed frame", and executes the given code. + +Then it copies the frame's bindings into a Dict, and returns it. Only the +names that don't end with an underscore `_` are copied. + +Example: + + var x = 10 # captured + var cmd = ^(var a = 42; var hidden_ = 'h'; var b = x + 1) + + var d = io->evalToDict(cmd) -- Does it push a new frame? Or is this a new module? - - I think we have to change the lookup rules -- Move functions like `len()` to their own `__builtin__` module? + pp (d) # => {a: 42, b: 11} ### captureStdout() diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index ba121ef74b..ebc94bfdeb 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -54,14 +54,17 @@ error handling, and more. [Match] group() start() end() X groups() X groupDict() [Place] setValue() - [Code Types] Expr Command + [Code Types] Command CommandFrag + Expr ExprFrag BuiltinFunc BoundFunc + Frame X [Func] name() location() toJson() X [Proc] name() location() toJson() [IO] eval() evalToDict() captureStdout() promptVal() X time() X strftime() X glob() [Obj] __invoke__ X __call__ + [VM] X getFrame() ```

@@ -88,6 +91,7 @@ X [J8 Decode] J8.Bool() J8.Int() ... [Introspection] id() shvarGet() getVar() setVar() parseCommand() X parseExpr() evalExpr() + X bindFrame() [Hay Config] parseHay() evalHay() X [Hashing] sha1dc() sha256() ``` diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index faf3f97782..d9b5e8f51c 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -51,55 +51,6 @@ call io->eval(my_block) 1 ## END -#### io->eval(block) can read variables like eval '' - -# NO LONGER WORKS, but is this a feature rather than a bug? - -proc p2(code_str) { - var mylocal = 42 - eval $code_str -} - -p2 'echo mylocal=$mylocal' - -proc p (;;; block) { - var mylocal = 99 - call io->eval(block) -} - -p { - echo mylocal=$mylocal -} - - -## STDOUT: -mylocal=42 -mylocal=99 -## END - -#### eval should have a sandboxed mode - -proc p (;;; block) { - var this = 42 - - # like push-registers? Not sure - # We could use state.ctx_Temp ? There's also ctx_FuncCall etc. - # - # I think we want to provide full control over the stack. - push-frame { - call io->eval(block) - } -} - -p { - echo $this -} - -## status: 1 -## STDOUT: -TODO -## END - #### io->eval with argv bindings call io->eval(^(echo "$@"), pos_args=:| foo bar baz |) call io->eval(^(pp test_ (:| $1 $2 $3 |)), pos_args=:| foo bar baz |) @@ -661,11 +612,12 @@ var frag = ^(echo $i) proc my-cd (new_dir; ; ; block) { pushd $new_dir + var calling_frame = vm.getFrame(-2) + # could call this "unbound"? or unbind()? What about procs and funcs and # exprs? var frag = getCommandFrag(block) - var calling_frame = getFrame(-2) call io->evalInFrame(frag, calling_frame) popd @@ -684,3 +636,58 @@ x: i = 1, j = 3 x: i = 2, j = 4 ## END + +#### parseCommand(), io->evalInFrame(frag, frame) can behave like eval $mystr + +# NO LONGER WORKS, but is this a feature rather than a bug? + +proc p2(code_str) { + var mylocal = 42 + eval $code_str +} + +p2 'echo mylocal=$mylocal' + +proc p (;;; block) { + # To behave like eval $code_str, without variable capture: + # + # var frag = getCommandFrag(block) + # var this_frame = vm.getFrame(-1) + # call io->evalInFrame(frag, this_frame) + + var mylocal = 99 + call io->eval(block) +} + +p { + echo mylocal=$mylocal +} + + +## STDOUT: +mylocal=42 +mylocal=99 +## END + +#### eval should have a sandboxed mode + +proc p (;;; block) { + var this = 42 + + # like push-registers? Not sure + # We could use state.ctx_Temp ? There's also ctx_FuncCall etc. + # + # I think we want to provide full control over the stack. + push-frame { + call io->eval(block) + } +} + +p { + echo $this +} + +## status: 1 +## STDOUT: +TODO +## END From ed477d2c9b53c9ce2f72066e04e88f9b9aa47134 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 00:11:57 -0400 Subject: [PATCH 351/506] [doc/ref] Document use builtin We still need to implement a few things, like * --all-provided, and provide * --all-for-testing [doc] Update table of sigil pairs. More are now implemented! --- doc/ref/chap-builtin-cmd.md | 67 ++++++++++++++++++++++++------------- doc/style-guide.md | 3 ++ doc/syntax-feelings.md | 25 +++++++++----- 3 files changed, 62 insertions(+), 33 deletions(-) diff --git a/doc/ref/chap-builtin-cmd.md b/doc/ref/chap-builtin-cmd.md index d37cfdabae..15dfb6cb2e 100644 --- a/doc/ref/chap-builtin-cmd.md +++ b/doc/ref/chap-builtin-cmd.md @@ -353,45 +353,64 @@ Use it like this: ### use -Import code from other files, creating an `Obj` that acts like a namespace. +The `use` builtin evaluates a source file in a new `Frame`, and then creates an +`Obj` that is a namespace. - use my-dir/my-module.ysh + use my-dir/mymodule.ysh - echo $[my_module.my_integer] # the module Obj has attributes - my_module myproc # the module Obj is invokable + echo $[mymodule.my_integer] # the module Obj has attributes + mymodule my-proc # the module Obj is invokable -The evaluation of such files is cached, so it won't be re-evaluated if `use` is called again. +The evaluation of such files is cached, so it won't be re-evaluated if `use` is +called again. - + use my-dir/mymodule.ysh --pick my-proc other-proc - -The `--extern` flag make the invocation do nothing. It can be used be tools to -analyze what names are in the file. +--- + +The `--extern` flag means that `use` does nothing. These commands can be used +by tools to analyze names. use --extern grep sed awk +--- + +Notes: + +- To get a reference to `module-with-hyphens`, you may need to use + `getVar('module-with-hyphens')`. + - TODO: consider backtick syntax as well +- `use` must be used at the top level, not within a function. + - This behavior is unlike Python. + +Warnings: + +- `use` **copies** the module bindings into a new `Obj`. This means that if + you rebind `mymodule.my_integer`, it will **not** be visible to code in the + module. + - This behavior is unlike Python. +- `use` allows "circular imports". That is `A.ysh` can `use B.ysh`, and vice + versa. + - To eliminate confusion over uninitialized names, use **only** `const`, + `func`, and `proc` at the top level of `my-module.ysh`. Don't run + commands, use `setvar`, etc. + ## I/O ### ysh-read diff --git a/doc/style-guide.md b/doc/style-guide.md index 192fd0b36d..ea748d78df 100644 --- a/doc/style-guide.md +++ b/doc/style-guide.md @@ -52,6 +52,9 @@ Env vars use `CAP_WORDS`: my-script.ysh # runs with YSH + my-module.ysh # import with 'use' + mymodule.ysh # also OK + ## YSH Names Capital Letters are used for types: diff --git a/doc/syntax-feelings.md b/doc/syntax-feelings.md index c035ebc2f6..3ea8aecd38 100644 --- a/doc/syntax-feelings.md +++ b/doc/syntax-feelings.md @@ -303,7 +303,7 @@ Shell arithmetic is also discouraged in favor of YSH arithmetic: ## Appendix: Table of Sigil Pairs -This table is mainly for YSH language designers. Many constructs aren't +This table is mainly for YSH language designers. Some constructs aren't implemented, but we reserve space for them. The [Oils Reference](ref/index.html) is more complete. @@ -322,26 +322,32 @@ Reference](ref/index.html) is more complete. :|foo $bar| Array Literal Words expr $[42 + a[i]] Stringify Expr Expression cmd,expr - @[glob(x)] Array-ify Expr Expression cmd,expr not implemented - ^[42 + a[i]] Unevaluated Expr Expression expr not implemented + @[glob(x)] Array-ify Expr Expression cmd,expr + ^[42 + a[i]] Unevaluated Expr Expression expr - ^"$1 $2" Unevaluated Str DQ String expr not implemented + ^"$1 $2" value.Expr DQ String expr ${x %2d} Var Sub Formatting cmd,expr not implemented ${x|html} Var Sub Formatting cmd,expr not implemented - json (x) Typed Arg List Argument cmd + pp (x) Typed Arg List Argument cmd + Expressions + + pp [x] Lazy Arrg list Argument cmd Expressions $/d+/ Inline Eggex Eggex Expr cmd not implemented - r'' Raw String String expr cmd when shopt + $"x is $x" Interpolated DQ string cmd,expr usually "x is $x" + string $ is optional + + r'foo\bar' Raw String String expr cmd when shopt Literal parse_raw_string - j"" JSON8 String String cmd,expr not implemented - Literal + u'' b'' J8 Literals String cmd,expr,data - #'a' Char Literal UTF-8 char expr + j"" JSON8 String String data + Literal Discouraged / Deprecated @@ -362,6 +368,7 @@ Key to "where valid" column: - `cmd` means `lex_mode_e.ShCommand` - `expr` means `lex_mode_e.Expr` +- `data` means it's valid in J8 Notation Some unused sigil pairs: From 738ee7809843bffd4611684e1cf3db686e6484c6 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 02:14:59 -0400 Subject: [PATCH 352/506] [ysh] Fix bug where return [x] was allowed It must be return (x) Reported by Samuel on Zulip. --- osh/cmd_parse.py | 3 +++ test/ysh-parse-errors.sh | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/osh/cmd_parse.py b/osh/cmd_parse.py index a02d895f35..439488422e 100644 --- a/osh/cmd_parse.py +++ b/osh/cmd_parse.py @@ -1273,6 +1273,9 @@ def ParseSimpleCommand(self): if len(typed_args.named_args) != 0: p_die("Typed return doesn't take named arguments", typed_loc) + if typed_args.left.id != Id.Op_LParen: + # return [x] is not valid + p_die("Expected ( in typed return", typed_args.left) return command.Retval(kw_token, typed_args.pos_args[0]) # Except for return (x), we shouldn't have typed args diff --git a/test/ysh-parse-errors.sh b/test/ysh-parse-errors.sh index a969e162cb..fb39d7b4d8 100755 --- a/test/ysh-parse-errors.sh +++ b/test/ysh-parse-errors.sh @@ -44,6 +44,29 @@ test-return-args() { return (x, x) } ' + + # Bug regression + + if false; then + bin/ysh -c ' + func foo() { + return [42] + } + echo foo=$[foo()] + ' + fi + + _ysh-parse-error ' + func foo() { + return [42] + } + ' + + _ysh-parse-error ' + func foo() { + return [42 + a[i]] + } + ' } test-func-var-checker() { From 1d388653f30385c46c2131919980925eeee04995 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 02:44:21 -0400 Subject: [PATCH 353/506] [ysh] Multi-line strings passed to <<< don't have newline appended This is a better behavior, and is backward compatible. This has 3 lines, not 4: cat <<< ''' 1 2 3 ''' Ditto for """ $""" u''' etc. --- frontend/syntax.asdl | 1 + osh/cmd_eval.py | 31 +++++++++++++++++++----- osh/cmd_parse.py | 37 ++++++++++++++++++++++++---- spec/ysh-string.test.sh | 53 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 108 insertions(+), 14 deletions(-) diff --git a/frontend/syntax.asdl b/frontend/syntax.asdl index a0d5372499..c10284464a 100644 --- a/frontend/syntax.asdl +++ b/frontend/syntax.asdl @@ -262,6 +262,7 @@ module syntax redir_param = Word %CompoundWord + | HereWord(CompoundWord w, bool is_multiline) | HereDoc(word here_begin, # e.g. EOF or 'EOF' Token? here_end_tok, # Token consisting of the whole line # It's always filled in AFTER creation, but diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 3bc82ca452..90b1a43c26 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -175,7 +175,6 @@ def _HasManyStatuses(node): return True return False # otherwise allow 'if true && true; ...' - # - ShAssignment could be allowed, though its exit code will always be # 0 without command subs # - Naively, (non-singleton) pipelines could be allowed because pipefail. @@ -494,15 +493,35 @@ def _EvalRedirect(self, r): return result elif redir_type == redir_arg_type_e.Here: # here word - val = self.word_ev.EvalWordToString(arg_word) - assert val.tag() == value_e.Str, val - # NOTE: bash and mksh both add \n - result.arg = redirect_arg.HereDoc(val.s + '\n') - return result + # TODO: delete this + raise AssertionError() else: raise AssertionError('Unknown redirect op') + elif case(redir_param_e.HereWord): + arg = cast(redir_param.HereWord, UP_arg) + + val = self.word_ev.EvalWordToString(arg.w) + assert val.tag() == value_e.Str, val + + assert r.op.id == Id.Redir_TLess, r.op + #print(arg_word) + + s = val.s + if not arg.is_multiline: + # NOTE: bash and mksh both add \n for + # read <<< 'hi' + # + # YSH doesn't do this for multi-line strings: + # read <<< ''' + # read <<< u''' + # read <<< """ + s += '\n' + + result.arg = redirect_arg.HereDoc(s) + return result + elif case(redir_param_e.HereDoc): arg = cast(redir_param.HereDoc, UP_arg) w = CompoundWord( diff --git a/osh/cmd_parse.py b/osh/cmd_parse.py index 439488422e..d0ca915d7b 100644 --- a/osh/cmd_parse.py +++ b/osh/cmd_parse.py @@ -53,6 +53,8 @@ proc_sig_e, Proc, Func, + SingleQuoted, + DoubleQuoted, ) from _devbuild.gen.value_asdl import LiteralBlock from core import alloc @@ -64,7 +66,7 @@ from frontend import location from frontend import match from frontend import reader -from mycpp.mylib import log +from mycpp.mylib import log, tagswitch from osh import braces from osh import bool_parse from osh import word_ @@ -760,15 +762,40 @@ def ParseRedirect(self): self._SetNext() return r - arg_word = self.cur_word + # We should never get Empty, Token, etc. + assert self.cur_word.tag() == word_e.Compound, self.cur_word + arg_word = cast(CompoundWord, self.cur_word) + tilde = word_.TildeDetect(arg_word) if tilde: arg_word = tilde self._SetNext() - # We should never get Empty, Token, etc. - assert arg_word.tag() == word_e.Compound, arg_word - return Redir(op_tok, where, cast(CompoundWord, arg_word)) + # Special case for <<< 'hi' and <<< ''' multiline ''' + if op_tok.id == Id.Redir_TLess: + part0 = arg_word.parts[0] + + is_multiline = False + with tagswitch(part0) as case: + if case(word_part_e.SingleQuoted): + single = cast(SingleQuoted, part0) + if single.left.id in (Id.Left_TSingleQuote, + Id.Left_RTSingleQuote, + Id.Left_UTSingleQuote, + Id.Left_BTSingleQuote): + is_multiline = True + + elif case(word_part_e.DoubleQuoted): + double = cast(DoubleQuoted, part0) + if double.left.id in (Id.Left_TDoubleQuote, + Id.Left_DollarTDoubleQuote): + is_multiline = True + #log('is_multiline %r', is_multiline) + + param = redir_param.HereWord(arg_word, is_multiline) + return Redir(op_tok, where, param) + + return Redir(op_tok, where, arg_word) def _ParseRedirectList(self): # type: () -> List[Redir] diff --git a/spec/ysh-string.test.sh b/spec/ysh-string.test.sh index 33b8c85d6f..7996731ed5 100644 --- a/spec/ysh-string.test.sh +++ b/spec/ysh-string.test.sh @@ -235,10 +235,10 @@ echo $double ## END -#### C strings in %() array literals -shopt -s oil:upgrade +#### C strings in :| | array literals +shopt -s ysh:upgrade -var lines=%($'aa\tbb' $'cc\tdd') +var lines=:| $'aa\tbb' $'cc\tdd' | write @lines ## STDOUT: @@ -276,6 +276,52 @@ unset r\ ## END +#### Special rule for <<< ''' and <<< """ - no extra newline + +read --all <<< unquoted +pp test_ (_reply) + +read --all <<< 'single with newline' +pp test_ (_reply) + +read --all <<< "double with newline" +pp test_ (_reply) + +read --all <<< u'j8 with newline' +pp test_ (_reply) + +echo + +read --all <<< ''' +multi +single +''' +pp test_ (_reply) + +read --all <<< """ +multi +double +""" +pp test_ (_reply) + +read --all <<< u''' +multi +j8 +''' +pp test_ (_reply) + + +## STDOUT: +(Str) "unquoted\n" +(Str) "single with newline\n" +(Str) "double with newline\n" +(Str) "j8 with newline\n" + +(Str) "multi\nsingle\n" +(Str) "multi\ndouble\n" +(Str) "multi\nj8\n" +## END + #### $''' isn't a a multiline string (removed) shopt -s ysh:upgrade @@ -546,3 +592,4 @@ double """zz ## status: 2 ## stdout-json: "" + From 637f19b98d51021ee0d75e4491823b14c47272d3 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 02:53:25 -0400 Subject: [PATCH 354/506] [spec/ysh-string] Fix tests after HereWord change Also remove redir_arg_type_e.Here Fix translation. [stdlib] Use the new semantics in stream.ysh testdata! --- frontend/consts.py | 4 ++-- frontend/types.asdl | 2 +- osh/cmd_eval.py | 25 +++++++++++-------------- osh/cmd_parse.py | 17 ++++++++--------- spec/ysh-string.test.sh | 2 -- stdlib/ysh/stream.ysh | 5 +++-- 6 files changed, 25 insertions(+), 30 deletions(-) diff --git a/frontend/consts.py b/frontend/consts.py index bb9e1a4079..93b111f817 100644 --- a/frontend/consts.py +++ b/frontend/consts.py @@ -105,8 +105,8 @@ def BoolArgType(id_): # descriptor Id.Redir_GreatAnd: redir_arg_type_e.Desc, Id.Redir_LessAnd: redir_arg_type_e.Desc, - Id.Redir_TLess: redir_arg_type_e.Here, # here word - # note: here docs aren't included + + # Note: here docs aren't included } diff --git a/frontend/types.asdl b/frontend/types.asdl index 7bd6f11b9d..02f33e24f1 100644 --- a/frontend/types.asdl +++ b/frontend/types.asdl @@ -2,7 +2,7 @@ module types { bool_arg_type = Undefined | Path | Int | Str | Other - redir_arg_type = Path | Desc | Here + redir_arg_type = Path | Desc opt_group = StrictAll | YshUpgrade | YshAll generate [integers] diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 90b1a43c26..ceb93bb333 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -492,10 +492,6 @@ def _EvalRedirect(self, r): return result - elif redir_type == redir_arg_type_e.Here: # here word - # TODO: delete this - raise AssertionError() - else: raise AssertionError('Unknown redirect op') @@ -508,16 +504,17 @@ def _EvalRedirect(self, r): assert r.op.id == Id.Redir_TLess, r.op #print(arg_word) - s = val.s - if not arg.is_multiline: - # NOTE: bash and mksh both add \n for - # read <<< 'hi' - # - # YSH doesn't do this for multi-line strings: - # read <<< ''' - # read <<< u''' - # read <<< """ - s += '\n' + # NOTE: bash and mksh both add \n for + # read <<< 'hi' + # + # YSH doesn't do this for multi-line strings: + # read <<< ''' + # read <<< u''' + # read <<< """ + if arg.is_multiline: + s = val.s + else: + s = val.s + '\n' result.arg = redirect_arg.HereDoc(s) return result diff --git a/osh/cmd_parse.py b/osh/cmd_parse.py index d0ca915d7b..ae95e81da8 100644 --- a/osh/cmd_parse.py +++ b/osh/cmd_parse.py @@ -778,19 +778,18 @@ def ParseRedirect(self): is_multiline = False with tagswitch(part0) as case: if case(word_part_e.SingleQuoted): - single = cast(SingleQuoted, part0) - if single.left.id in (Id.Left_TSingleQuote, - Id.Left_RTSingleQuote, - Id.Left_UTSingleQuote, - Id.Left_BTSingleQuote): + sq = cast(SingleQuoted, part0) + if sq.left.id in (Id.Left_TSingleQuote, + Id.Left_RTSingleQuote, + Id.Left_UTSingleQuote, + Id.Left_BTSingleQuote): is_multiline = True elif case(word_part_e.DoubleQuoted): - double = cast(DoubleQuoted, part0) - if double.left.id in (Id.Left_TDoubleQuote, - Id.Left_DollarTDoubleQuote): + dq = cast(DoubleQuoted, part0) + if dq.left.id in (Id.Left_TDoubleQuote, + Id.Left_DollarTDoubleQuote): is_multiline = True - #log('is_multiline %r', is_multiline) param = redir_param.HereWord(arg_word, is_multiline) return Redir(op_tok, where, param) diff --git a/spec/ysh-string.test.sh b/spec/ysh-string.test.sh index 7996731ed5..7a641292e3 100644 --- a/spec/ysh-string.test.sh +++ b/spec/ysh-string.test.sh @@ -477,7 +477,6 @@ two = 2 "" three = 3 -- - three = 3 two = 2 "" one " @@ -539,7 +538,6 @@ tac <<< ''' ''' ## STDOUT: - \u{61} '' ' ' diff --git a/stdlib/ysh/stream.ysh b/stdlib/ysh/stream.ysh index a790822f0a..39e6800477 100644 --- a/stdlib/ysh/stream.ysh +++ b/stdlib/ysh/stream.ysh @@ -168,10 +168,11 @@ proc must-match (; pattern; block) { proc line-data { # note: trailing ''' issue, I should probably get rid of the last line - echo ''' + write --end '' -- ''' prefix 30 foo oils - /// 42 bar''' + /// 42 bar + ''' } const pat = / s+ / From f82b7a720b13e16d43c7b5087419a72385f330b9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 03:16:04 -0400 Subject: [PATCH 355/506] [doc/ref] Add here-str for OSH, and ysh-here-str for YSH Improve topics on here docs. --- doc/ref/chap-cmd-lang.md | 62 ++++++++++++++++++++++++++++++++++++++-- doc/ref/toc-osh.md | 3 +- doc/ref/toc-ysh.md | 15 +++++----- 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/doc/ref/chap-cmd-lang.md b/doc/ref/chap-cmd-lang.md index 823004b987..5b74adb1f3 100644 --- a/doc/ref/chap-cmd-lang.md +++ b/doc/ref/chap-cmd-lang.md @@ -315,6 +315,11 @@ In YSH, use the [fork][] builtin. ### redir-file +The operators `>` and `>>` redirect the `stdout` of a process to a disk file. +The `<` operator redirects `stdin` from a disk file. + +--- + Examples of redirecting the `stdout` of a command: echo foo > out.txt # overwrite out.txt @@ -362,22 +367,73 @@ There's no real difference. ### here-doc -TODO: unbalanced HTML if we use \<\ - +The string **plus a newline** is the `stdin` value, which is consistent with +GNU bash. + +### ysh-here-str + +You can also use YSH multi-line strings as "here strings". For example: + +Double-quoted: + + cat <<< """ + double + quoted = $x + """ + +Single-quoted: + + cat <<< ''' + price is + $3.99 + ''' + +J8-style with escapes: + + cat <<< u''' + j8 style string price is + mu = \u{3bc} + ''' + +In these cases, a trailing newline is **not** added. For example, the first +example is equivalent to: + + write --end '' -- """ + double + quoted = $x + """ ## Other Command diff --git a/doc/ref/toc-osh.md b/doc/ref/toc-osh.md index bab02fd800..25d6a925f8 100644 --- a/doc/ref/toc-osh.md +++ b/doc/ref/toc-osh.md @@ -110,7 +110,8 @@ X [Unsupported] enable [Concurrency] pipe | X pipe-amp |& ampersand & [Redirects] redir-file > >> >| < <> not impl: &> redir-desc >& <& - here-doc << <<- <<< + here-doc << <<- + here-str <<< [Other Command] dparen (( time X coproc X select ``` diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index ebc94bfdeb..40d94ef454 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -214,13 +214,14 @@ X [External Lang] BEGIN END when (awk) ```chapter-links-cmd-lang_33 - [YSH Simple] typed-arg json write (x) - lazy-expr-arg assert [42 === x] - block-arg cd /tmp { echo $PWD }; cd /tmp (; ; blockexpr) - [YSH Cond] ysh-case case (x) { *.py { echo 'python' } } - ysh-if if (x > 0) { echo } - [YSH Iter] ysh-for for i, item in (mylist) { echo } - ysh-while while (x > 0) { echo } + [Redirect] ysh-here-str read <<< ''' + [YSH Simple] typed-arg json write (x) + lazy-expr-arg assert [42 === x] + block-arg cd /tmp { echo $PWD }; cd /tmp (; ; blockexpr) + [YSH Cond] ysh-case case (x) { *.py { echo 'python' } } + ysh-if if (x > 0) { echo } + [YSH Iter] ysh-for for i, item in (mylist) { echo } + ysh-while while (x > 0) { echo } ```

From 63feb944af6cbc7497711dc5da44e639d3dadc04 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 03:54:39 -0400 Subject: [PATCH 356/506] [doctools/src-tree] Remove YSH file that doesn't highlight The here doc heuristic causes the whole job to fail. TODO: These should all be understood as YSH code. --- doctools/src-tree.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doctools/src-tree.sh b/doctools/src-tree.sh index af9fce039b..574377992f 100755 --- a/doctools/src-tree.sh +++ b/doctools/src-tree.sh @@ -44,7 +44,11 @@ _print-files() { # Remove binary file (probably should delete it altogether, but it's a nice # test of UTF-8) - git ls-files | egrep -v 'Python-2.7.13|^py-yajl|rsa_travis.enc' + # Remove spec/ysh-string.test.sh because it conatins YSH <<<, which messes up + # the shell here doc parser. + # TODO: ysh needs micro-syntax support + + git ls-files | egrep -v 'Python-2.7.13|^py-yajl|rsa_travis.enc|ysh-string.test.sh' return From c6fe460094460da92fff275e07f900a32e81f55c Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 12:23:47 -0400 Subject: [PATCH 357/506] [ysh parser] Fix op Token of expr.Slice now Preparing for adding Str -> Int conversions. --- spec/ysh-slice-range.test.sh | 45 +++++++++++++++++++++++++++++++++++- ysh/expr_eval.py | 10 ++++---- ysh/expr_to_ast.py | 7 +++++- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/spec/ysh-slice-range.test.sh b/spec/ysh-slice-range.test.sh index 523e1669c8..994b6b68e1 100644 --- a/spec/ysh-slice-range.test.sh +++ b/spec/ysh-slice-range.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 3 # Test a[1] @@ -55,6 +55,49 @@ out of bounds (List) ["1","2","3","4"] ## END +#### Range end points can be int-looking Strings + +pp test_ ('3' .. '6') + +var i = '5' + +pp test_ (i .. 7) +pp test_ (3 .. i) + +var i = '-5' + +pp test_ (i .. -3) +pp test_ (-3 .. i) + +# Not allowed +pp test_ ('a' .. 'z') + +## STDOUT: +## END + +#### Slice indices can be int-looking strings + +var a = list(1..10) + +pp test_ (a['3': '6']) + +var i = '5' + +pp test_ (a[i : 7]) +pp test_ (a[3 : i]) + +var i = '-5' + +pp test_ (a[i : -3]) +pp test_ (a[-3 : i]) + +# Not allowed +pp test_ (a['a' : 'z']) + +## STDOUT: +## END + + #### slice subscripts are adjusted like Python show-py() { diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index c4be1af6c9..7edce67d83 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -1220,14 +1220,12 @@ def _EvalExpr(self, node): if node.lower: msg = 'Slice begin should be Int' - i = val_ops.ToInt(self._EvalExpr(node.lower), msg, - loc.Missing) + i = val_ops.ToInt(self._EvalExpr(node.lower), msg, node.op) lower = IntBox(i) if node.upper: msg = 'Slice end should be Int' - i = val_ops.ToInt(self._EvalExpr(node.upper), msg, - loc.Missing) + i = val_ops.ToInt(self._EvalExpr(node.upper), msg, node.op) upper = IntBox(i) return value.Slice(lower, upper) @@ -1239,10 +1237,10 @@ def _EvalExpr(self, node): assert node.upper is not None msg = 'Range begin should be Int' - i = val_ops.ToInt(self._EvalExpr(node.lower), msg, loc.Missing) + i = val_ops.ToInt(self._EvalExpr(node.lower), msg, node.op) msg = 'Range end should be Int' - j = val_ops.ToInt(self._EvalExpr(node.upper), msg, loc.Missing) + j = val_ops.ToInt(self._EvalExpr(node.upper), msg, node.op) return value.Range(i, j) diff --git a/ysh/expr_to_ast.py b/ysh/expr_to_ast.py index de82476537..5615c4a476 100644 --- a/ysh/expr_to_ast.py +++ b/ysh/expr_to_ast.py @@ -489,9 +489,12 @@ def _Subscript(self, parent): if typ0 == grammar_nt.expr: if n == 3: # a[1:2] lower = self.Expr(parent.GetChild(0)) + op_tok = parent.GetChild(1).tok upper = self.Expr(parent.GetChild(2)) + elif n == 2: # a[1:] lower = self.Expr(parent.GetChild(0)) + op_tok = parent.GetChild(1).tok upper = None else: # a[1] return self.Expr(parent.GetChild(0)) @@ -499,11 +502,13 @@ def _Subscript(self, parent): assert typ0 == Id.Arith_Colon lower = None if n == 1: # a[:] + op_tok = parent.GetChild(0).tok upper = None else: # a[:3] + op_tok = parent.GetChild(0).tok upper = self.Expr(parent.GetChild(1)) - return expr.Slice(lower, parent.GetChild(0).tok, upper) + return expr.Slice(lower, op_tok, upper) def Expr(self, pnode): # type: (PNode) -> expr_t From 825d693e01c7406836fd75b216eadd7049160d12 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 12:37:27 -0400 Subject: [PATCH 358/506] [ysh semantics] Range and Slice params can be int-looking Strs This came up a few times, e.g. when writing benchmarks/ysh-for.sh --- benchmarks/ysh-for.sh | 34 ++++++++++++++++++++++++---------- spec/ysh-slice-range.test.sh | 29 +++++++++++++++++++++-------- ysh/expr_eval.py | 28 +++++++++++++++------------- 3 files changed, 60 insertions(+), 31 deletions(-) diff --git a/benchmarks/ysh-for.sh b/benchmarks/ysh-for.sh index 95f15f6fa6..945bff9ed6 100755 --- a/benchmarks/ysh-for.sh +++ b/benchmarks/ysh-for.sh @@ -16,9 +16,8 @@ sum() { echo " YSH for loop" time $YSH -c ' - var n = int($1) var sum = 0 - for i in (0 .. n) { + for i in (0 .. $1) { setvar sum += i } echo "i = $i" @@ -30,9 +29,8 @@ sum-closures() { echo " YSH closures" time $YSH -c ' - var n = int($1) var sum = 0 - for __hack__ in (0 .. n) { # trigger allocation + for __hack__ in (0 .. $1) { # trigger allocation setvar sum += __hack__ } # Does not leak! @@ -79,18 +77,34 @@ compare() { sum-py $n echo - sum-sh bash $n - echo - - sum-sh $OSH $n - echo - export OILS_GC_STATS sum $n echo sum-closures $n echo + + if true; then + # 3.9 seconds + sum-sh bash $n + echo + + # 3.7 seconds + sum-sh $OSH $n + echo + + # 1.2 seconds + sum-sh dash $n + echo + + # 2.3 seconds + sum-sh zsh $n + echo + + # 3.1 seconds + sum-sh mksh $n + echo + fi } "$@" diff --git a/spec/ysh-slice-range.test.sh b/spec/ysh-slice-range.test.sh index 994b6b68e1..fb810a9677 100644 --- a/spec/ysh-slice-range.test.sh +++ b/spec/ysh-slice-range.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 3 +## oils_failures_allowed: 1 # Test a[1] @@ -57,27 +57,34 @@ out of bounds #### Range end points can be int-looking Strings -pp test_ ('3' .. '6') +pp test_ (list('3' .. '6')) var i = '5' -pp test_ (i .. 7) -pp test_ (3 .. i) +pp test_ (list(i .. 7)) +pp test_ (list(3 .. i)) var i = '-5' -pp test_ (i .. -3) -pp test_ (-3 .. i) +pp test_ (list(i .. -3)) +pp test_ (list(-7 .. i)) # Not allowed pp test_ ('a' .. 'z') +## status: 3 ## STDOUT: +(List) [3,4,5] +(List) [5,6] +(List) [3,4] +(List) [-5,-4] +(List) [-7,-6] ## END #### Slice indices can be int-looking strings -var a = list(1..10) +var a = list(0..10) +#pp test_ (a) pp test_ (a['3': '6']) @@ -89,12 +96,18 @@ pp test_ (a[3 : i]) var i = '-5' pp test_ (a[i : -3]) -pp test_ (a[-3 : i]) +pp test_ (a[-7 : i]) # Not allowed pp test_ (a['a' : 'z']) +## status: 3 ## STDOUT: +(List) [3,4,5] +(List) [5,6] +(List) [3,4] +(List) [5,6] +(List) [3,4] ## END diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 7edce67d83..9c9cdd1e81 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -204,8 +204,6 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): Called by CommandEvaluator """ - # TODO: It might be nice to do auto d[x] += 1 too - UP_lval = lval with tagswitch(lval) as case: if case(y_lvalue_e.Local): # setvar x += 1 @@ -230,6 +228,7 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): with tagswitch(obj) as case: if case(value_e.List): obj = cast(value.List, UP_obj) + # TODO: could be int-looking Str index = val_ops.ToInt(lval.index, 'List index should be Int', loc.Missing) @@ -1219,14 +1218,16 @@ def _EvalExpr(self, node): upper = None # type: Optional[IntBox] if node.lower: - msg = 'Slice begin should be Int' - i = val_ops.ToInt(self._EvalExpr(node.lower), msg, node.op) - lower = IntBox(i) + i1 = _ConvertToInt(self._EvalExpr(node.lower), + 'Slice begin should be Int', node.op) + # TODO: don't truncate + lower = IntBox(mops.BigTruncate(i1)) if node.upper: - msg = 'Slice end should be Int' - i = val_ops.ToInt(self._EvalExpr(node.upper), msg, node.op) - upper = IntBox(i) + i1 = _ConvertToInt(self._EvalExpr(node.upper), + 'Slice end should be Int', node.op) + # TODO: don't truncate + upper = IntBox(mops.BigTruncate(i1)) return value.Slice(lower, upper) @@ -1236,13 +1237,14 @@ def _EvalExpr(self, node): assert node.lower is not None assert node.upper is not None - msg = 'Range begin should be Int' - i = val_ops.ToInt(self._EvalExpr(node.lower), msg, node.op) + i1 = _ConvertToInt(self._EvalExpr(node.lower), + 'Range begin should be Int', node.op) - msg = 'Range end should be Int' - j = val_ops.ToInt(self._EvalExpr(node.upper), msg, node.op) + i2 = _ConvertToInt(self._EvalExpr(node.upper), + 'Range end should be Int', node.op) - return value.Range(i, j) + # TODO: Don't truncate + return value.Range(mops.BigTruncate(i1), mops.BigTruncate(i2)) elif case(expr_e.Compare): node = cast(expr.Compare, UP_node) From b691f7d8a7cc2dbbdd85f9f8ed96b6e447f09982 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 12:55:39 -0400 Subject: [PATCH 359/506] [ysh semantics] List index can be int-looking Str All of these are supported: var i = mylist['3'] set mylist['3'] = 'foo' set mylist['3'] += 5 This is consistent with the rest of arithmetic, and Slice/Range. --- osh/cmd_eval.py | 8 +++--- spec/ysh-expr.test.sh | 14 +++++----- spec/ysh-list.test.sh | 40 ++++++++++++++++++++++++++++- ysh/expr_eval.py | 60 ++++++++++++++++++++++++------------------- 4 files changed, 84 insertions(+), 38 deletions(-) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index ceb93bb333..3b3feb8d63 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -762,10 +762,10 @@ def _DoMutation(self, node): with tagswitch(obj) as case: if case(value_e.List): obj = cast(value.List, UP_obj) - index = val_ops.ToInt(lval.index, - 'List index should be Int', - loc.Missing) - obj.items[index] = rval + index = expr_eval._ConvertToInt( + lval.index, 'List index should be Int', + loc.Missing) + obj.items[mops.BigTruncate(index)] = rval elif case(value_e.Dict): obj = cast(value.Dict, UP_obj) diff --git a/spec/ysh-expr.test.sh b/spec/ysh-expr.test.sh index e5f151476d..324ab8c7e0 100644 --- a/spec/ysh-expr.test.sh +++ b/spec/ysh-expr.test.sh @@ -8,9 +8,9 @@ echo x=${x:-default} y=${y:-default} x=hi y=default ## END -#### shell array %(a 'b c') +#### shell array :| a 'b c' | shopt -s parse_at -var x = %(a 'b c') +var x = :| a 'b c' | var empty = %() argv.py / @x @empty / @@ -161,7 +161,7 @@ gt=0 ## END #### Parse { var x = 42 } -shopt -s oil:upgrade +shopt -s ysh:upgrade g() { var x = 42 } var x = 1 @@ -239,7 +239,7 @@ a b c #### null / true / false -shopt -s oil:upgrade +shopt -s ysh:upgrade var n = null if (n) { echo yes @@ -510,7 +510,7 @@ array=3 comsub=6 ## END -#### obj->method() +#### obj=>method() - remove? var s = 'hi' # TODO: This does a bound method thing we probably don't want @@ -520,7 +520,7 @@ echo $s2 HI ## END -#### obj->method does NOT give you a bound method +#### s->upper does NOT work, should be s.upper() or => var s = 'hi' var method = s->upper echo $method @@ -576,7 +576,7 @@ Int Str 3 ## END #### s ~~ glob and s !~~ glob -shopt -s oil:all +shopt -s ysh:all if ('foo.py' ~~ '*.py') { echo yes diff --git a/spec/ysh-list.test.sh b/spec/ysh-list.test.sh index 1c6b88466e..17645b38ad 100644 --- a/spec/ysh-list.test.sh +++ b/spec/ysh-list.test.sh @@ -1,13 +1,51 @@ ## our_shell: ysh ## oils_failures_allowed: 0 -#### basic array +#### Basic List, a[42] a['42'] allowed + var x = :| 1 2 3 | write len=$[len(x)] + +pp test_ (x[1]) + +# Can be int-looking string +pp test_ (x['2']) + +# Not allowed +pp test_ (x['zz']) + +## status: 3 ## STDOUT: len=3 +(Str) "2" +(Str) "3" ## END +#### Mutate List entries, a[42] a['42'] allowed + +var a = :| 2 3 4 | + +setvar a[1] = 1 +pp test_ (a) + +setvar a['2'] += 5 +pp test_ (a) + +# Can be int-looking string +setvar a['2'] = 99 +pp test_ (a) + +# Not allowed +setvar a['zz'] = 101 + +## status: 3 +## STDOUT: +(List) ["2",1,"4"] +(List) ["2",1,9] +(List) ["2",1,99] +## END + + #### string array with command sub, varsub, etc. shopt -s ysh:all diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 9c9cdd1e81..43de726254 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -228,10 +228,11 @@ def EvalAugmented(self, lval, rhs_val, op, which_scopes): with tagswitch(obj) as case: if case(value_e.List): obj = cast(value.List, UP_obj) - # TODO: could be int-looking Str - index = val_ops.ToInt(lval.index, - 'List index should be Int', - loc.Missing) + i1 = _ConvertToInt(lval.index, + 'List index should be Int', + loc.Missing) + # TODO: don't truncate + index = mops.BigTruncate(i1) try: lhs_val_ = obj.items[index] except IndexError: @@ -321,7 +322,7 @@ def _EvalLeftLocalOrGlobal(self, lhs, which_scopes): obj = self._EvalLeftLocalOrGlobal(lhs.obj, which_scopes) index = self._EvalExpr(lhs.index) - return self._EvalSubscript(obj, index) + return self._EvalSubscript(obj, index, lhs.left) elif case(expr_e.Attribute): lhs = cast(Attribute, UP_lhs) @@ -874,8 +875,8 @@ def _EvalFuncCall(self, node): return self._CallFunc(to_call, rd) - def _EvalSubscript(self, obj, index): - # type: (value_t, value_t) -> value_t + def _EvalSubscript(self, obj, index, blame_loc): + # type: (value_t, value_t, loc_t) -> value_t UP_obj = obj UP_index = index @@ -899,45 +900,52 @@ def _EvalSubscript(self, obj, index): try: return value.Str(obj.s[i]) except IndexError: - # TODO: expr.Subscript has no error location - raise error.Expr('index out of range', loc.Missing) + raise error.Expr('index out of range', blame_loc) else: raise error.TypeErr(index, 'Str index expected Int or Slice', - loc.Missing) + blame_loc) elif case(value_e.List): obj = cast(value.List, UP_obj) + + big_i = mops.ZERO with tagswitch(index) as case2: if case2(value_e.Slice): index = cast(value.Slice, UP_index) - lower = index.lower.i if index.lower else 0 - upper = index.upper.i if index.upper else len( - obj.items) + lower = (index.lower.i if index.lower else 0) + upper = (index.upper.i + if index.upper else len(obj.items)) return value.List(obj.items[lower:upper]) elif case2(value_e.Int): index = cast(value.Int, UP_index) - i = mops.BigTruncate(index.i) - try: - return obj.items[i] - except IndexError: - # TODO: expr.Subscript has no error location - raise error.Expr('List index out of range: %d' % i, - loc.Missing) + big_i = index.i + + elif case2(value_e.Str): + index = cast(value.Str, UP_index) + big_i = _ConvertToInt(index, 'List index expected Int', + blame_loc) else: raise error.TypeErr( - index, 'List index expected Int or Slice', - loc.Missing) + index, 'List index expected Int, Str, or Slice', + blame_loc) + + i = mops.BigTruncate(big_i) # TODO: don't truncate + try: + return obj.items[i] + except IndexError: + raise error.Expr('List index out of range: %d' % i, + blame_loc) elif case(value_e.Dict): obj = cast(value.Dict, UP_obj) if index.tag() != value_e.Str: raise error.TypeErr(index, 'Dict index expected Str', - loc.Missing) + blame_loc) index = cast(value.Str, UP_index) try: @@ -945,10 +953,10 @@ def _EvalSubscript(self, obj, index): except KeyError: # TODO: expr.Subscript has no error location raise error.Expr('Dict entry not found: %r' % index.s, - loc.Missing) + blame_loc) raise error.TypeErr(obj, 'Subscript expected Str, List, or Dict', - loc.Missing) + blame_loc) def _ChainedLookup(self, obj, current, attr_name): # type: (Obj, Obj, str) -> Optional[value_t] @@ -1319,7 +1327,7 @@ def _EvalExpr(self, node): node = cast(Subscript, UP_node) obj = self._EvalExpr(node.obj) index = self._EvalExpr(node.index) - return self._EvalSubscript(obj, index) + return self._EvalSubscript(obj, index, node.left) elif case(expr_e.Attribute): # obj->method or mydict.key node = cast(Attribute, UP_node) From 0e302c180512cc7c1f7ed9232404c3d728744965 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 18:52:42 -0400 Subject: [PATCH 360/506] [ysh builtin func] get() has optional 3rd argument The default for the default value is null. ALso start 'proc Dict' in stdlib/ysh/def.ysh I ran into a bug - the value.Place validation doesn't take MODULES into account! There's a bug using Dict in a different module. --- builtin/method_dict.py | 4 +++- core/state.py | 6 +++-- doc/ref/chap-builtin-func.md | 17 +++++++++----- spec/ysh-dict.test.sh | 9 ++++++++ stdlib/TEST.sh | 2 ++ stdlib/ysh/def-test.ysh | 43 ++++++++++++++++++++++++++++++++++++ stdlib/ysh/def.ysh | 7 ++++++ 7 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 stdlib/ysh/def-test.ysh create mode 100644 stdlib/ysh/def.ysh diff --git a/builtin/method_dict.py b/builtin/method_dict.py index 747caf3b6d..be22d9f709 100644 --- a/builtin/method_dict.py +++ b/builtin/method_dict.py @@ -75,7 +75,7 @@ def Call(self, rd): obj = rd.PosValue() key = rd.PosStr() - default_value = rd.PosValue() + default_value = rd.OptionalValue() rd.Done() UP_obj = obj @@ -90,4 +90,6 @@ def Call(self, rd): raise error.TypeErr(obj, 'get() expected Dict or Obj', rd.BlamePos()) + if default_value is None: + default_value = value.Null return d.get(key, default_value) diff --git a/core/state.py b/core/state.py index 59917985c2..7dc942301d 100644 --- a/core/state.py +++ b/core/state.py @@ -1166,7 +1166,7 @@ def __init__(self, mem, name1): rear_frame = self.mem.var_stack[-1] self.front_frame = NewDict() # type: Dict[str, Cell] self.front_frame['__E__'] = Cell(False, False, False, - value.Frame(rear_frame)) + value.Frame(rear_frame)) mem.var_stack.append(self.front_frame) def __enter__(self): @@ -1211,7 +1211,7 @@ def __init__(self, mem, rear_frame, out_dict): # __E__ gets a lookup rule self.front_frame = NewDict() # type: Dict[str, Cell] self.front_frame['__E__'] = Cell(False, False, False, - value.Frame(rear_frame)) + value.Frame(rear_frame)) mem.var_stack.append(self.front_frame) @@ -1958,6 +1958,8 @@ def SetPlace(self, place, val, blame_loc): yval = cast(LeftName, UP_yval) # Check that the frame is still alive + + # TODO: This doesn't work with modules found = False for i in xrange(len(self.var_stack) - 1, -1, -1): frame = self.var_stack[i] diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index cdd9c12166..7a35e611ec 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -239,19 +239,26 @@ Similar to `keys()`, but returns the values of the dictionary. ### get() Return value for given key, falling back to the default value if the key -doesn't exist. Default is required. +doesn't exist. var book = { title: "Hitchhiker's Guide", published: 1979, } - var published = get(book, "published", null) + + var published = get(book, 'published', null) = published - # => (Int 1979) + # => (Int) 1979 + + var author = get(book, 'author', "???") + = author + # => (Str) "???" + +If not specified, the default value is `null`: - var author = get(book, "author", "???") + var author = get(book, 'author') = author - # => (Str "???") + # => (Null) null ## Float diff --git a/spec/ysh-dict.test.sh b/spec/ysh-dict.test.sh index 7d8ae519ea..1c96bdf486 100644 --- a/spec/ysh-dict.test.sh +++ b/spec/ysh-dict.test.sh @@ -107,5 +107,14 @@ pp test_ (get(d, 'key', 'default')) (Str) "default" ## END +#### get() has default null + +var d = {a: 42} + +pp test_ (get(d, 'b')) + +## STDOUT: +(Null) null +## END diff --git a/stdlib/TEST.sh b/stdlib/TEST.sh index 9e4a2764b5..f70af5a232 100755 --- a/stdlib/TEST.sh +++ b/stdlib/TEST.sh @@ -37,6 +37,8 @@ test-byo-protocol() { soil-run() { test-byo-protocol + devtools/byo.sh test $YSH stdlib/ysh/def-test.ysh + #return devtools/byo.sh test $YSH stdlib/ysh/args-test.ysh devtools/byo.sh test $YSH stdlib/ysh/list-test.ysh devtools/byo.sh test $YSH stdlib/ysh/math-test.ysh diff --git a/stdlib/ysh/def-test.ysh b/stdlib/ysh/def-test.ysh new file mode 100644 index 0000000000..9914add89e --- /dev/null +++ b/stdlib/ysh/def-test.ysh @@ -0,0 +1,43 @@ +use $LIB_YSH/def.ysh --pick Dict + +: ${LIB_OSH=stdlib/osh} +source $LIB_OSH/byo-server.sh + +proc test-dict { + var i = 0 + Dict (&d) { + a = 42 + b = i + 1 + } + pp test_ (d) +} + + +func Counter(start) { + ### constructor + var methods = Object(null, Counter_methods) + return (Object(methods, {i: start})) +} + +var Counter_methods = null + +# BUG: I think there is a problem with using Dict in a DIFFERENT module! + +if false { +Dict (&Counter_methods) { + #func inc(self, n) { + # setvar self.i += n + #} + i = 2 +} +} + +proc test-class-pattern { + #var c = Counter(5) + #call c.inc(5) + : +} + +if is-main { + byo-maybe-run +} diff --git a/stdlib/ysh/def.ysh b/stdlib/ysh/def.ysh new file mode 100644 index 0000000000..19ba6c710f --- /dev/null +++ b/stdlib/ysh/def.ysh @@ -0,0 +1,7 @@ +const __provide__ = :| Dict | + +proc Dict ( ; out; ; block) { + var d = io->evalToDict(block) + call out->setValue(d) +} + From 3c50c1c0bf9353be4a06a5a74668ee96790a38d9 Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 21:54:48 -0400 Subject: [PATCH 361/506] [cleanup] Reorganize value.asdl [doc] Publish types.html --- core/value.asdl | 65 +++++++++++++++++++++++----------------- doc/index.md | 1 + doc/published.md | 54 ++++++++++++++++++++------------- doc/style-guide.md | 8 ++++- doc/types.md | 74 +++++++++++++++++++++++++++++++++++++--------- 5 files changed, 139 insertions(+), 63 deletions(-) diff --git a/core/value.asdl b/core/value.asdl index 619e584708..495bf01296 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -73,13 +73,23 @@ module value # Commands, words, and expressions from syntax.asdl are evaluated to a VALUE. # value_t instances are stored in state.Mem(). value = - # Only used for val_ops.StdinIterator. (It would be nice if we could - # express iter_value.{Eof,Interrupted,Str,Int,...} in ASDL) + # + # Implementation details + # + + # Only used for io.stdin aka val_ops.StdinIterator. (It would be nice if + # we could express iter_value.{Eof,Interrupted,Str,Int,...} in ASDL) Interrupted | Stdin + # Can't be instantiated by users + # a[3:5] a[:10] a[3:] a[:] # both ends are optional + | Slice(IntBox? lower, IntBox? upper) - # Methods on state.Mem return value.Undef, but it's not visible in YSH. # + # OSH/Bash types + # + + # Methods on state.Mem return value.Undef, but it's not visible in YSH. # A var bound to Undef is different than no binding because of dynamic # scope. Undef can shadow values lower on the stack. | Undef @@ -88,8 +98,8 @@ module value # "holes" in the array are represented by None | BashArray(List[str] strs) - # TODO: Switch to this more efficient representation? - # max_index makes append-sparse workload faster, and normal append loops too + # TODO: Switch to this more efficient representation. max_index makes + # append-sparse workload faster, and normal append loops too | SparseArray(Dict[BigInt, str] d, BigInt max_index) | BashAssoc(Dict[str, str] d) @@ -107,9 +117,9 @@ module value # because they have attributes (functions), methods - not just methods | Obj %Obj - # CODE types - # unevaluated: Eggex, Expr, Template, Command/Block - # callable, in separate namespaces: Func, BoundFunc, Proc + # for i in (0 .. n) { echo $i } # both ends are required + # TODO: BigInt + | Range(int lower, int upper) # expr is spliced # / d+; ignorecase / -> '[[:digit:]]+' REG_ICASE @@ -124,16 +134,6 @@ module value # provide a nice interface. | Match %RegexMatch - # ^[42 + a[i]] - | Expr(expr e) - - # This is an UNBOUND command, like - # ^(echo 1; echo 2) and cd { echo 1; echo 2 } - | CommandFrag(command c) - - # Bound command - | Command(cmd_frag frag, Dict[str, Cell] captured_frame) - # A place has an additional stack frame where the value is evaluated. # The frame MUST be lower on the stack at the time of use. | Place(y_lvalue lval, Dict[str, Cell] frame) @@ -143,12 +143,16 @@ module value # TODO: ASDL should let us "collapse" this Dict directly into value_t | Frame(Dict[str, Cell] frame) + # + # Code units: BoundFunc, BuiltinFunc, Func, BuiltinProc, Proc + # + + # for obj.method and obj->mutatingMethod + | BoundFunc(value me, value func) # callable is vm._Callable. - # TODO: ASDL needs some kind of "extern" to declare vm._Callable and - # cmd_eval.CommandEvaluator. I think it would just generate a forward - # declaration. + # TODO: ASDL needs some kind of "extern" to declare vm._Callable, + # vm._Builtin. I think it would just generate a forward declaration. | BuiltinFunc(any callable) - | BoundFunc(value me, value func) | Func(str name, Func parsed, List[value] pos_defaults, Dict[str, value] named_defaults, @@ -168,12 +172,19 @@ module value # module is where "global" lookups happen Dict[str, Cell] module_frame) - # for i in (1:n) { echo $i } # both ends are required - | Range(int lower, int upper) + # + # Unevaluated CODE types: ExprFrag, Expr, CommandFrag, Command + # - # internal detail - can't be instantied by users - # a[3:5] a[:10] a[3:] a[:] # both ends are optional - | Slice(IntBox? lower, IntBox? upper) + # ^[42 + a[i]] + | Expr(expr e) + + # This is an UNBOUND command, like + # ^(echo 1; echo 2) and cd { echo 1; echo 2 } + | CommandFrag(command c) + + # Bound command + | Command(cmd_frag frag, Dict[str, Cell] captured_frame) # Other introspection # __builtins__ - Dict[str, value_t] - I would like to make this read-only diff --git a/doc/index.md b/doc/index.md index 5e9a48b63c..8eb9bfe3af 100644 --- a/doc/index.md +++ b/doc/index.md @@ -125,6 +125,7 @@ the wire, **not** the other way around. ## The Shared Oils Runtime +- [Types in the Oils Runtime](types.html) - [YSH Fixes Shell's Error Handling (`errexit`)](error-handling.html) - [Oils Error Catalog, With Hints](error-catalog.html) - [Tracing Execution](xtrace.html). YSH enhances shell's `set -x`. diff --git a/doc/published.md b/doc/published.md index 83770fdf07..70f18e848f 100644 --- a/doc/published.md +++ b/doc/published.md @@ -18,20 +18,22 @@ See [All Docs](index.html) for links to drafts. ## More -- OSH: - - [Known Differences Between OSH and Other Shells](known-differences.html) - | [Quirks](quirks.html) - | [Tracing Execution](xtrace.html) - | [Headless Mode](headless.html) - | [Shell Idioms](shell-idioms.html) -- YSH: - | [A Feel For YSH Syntax](syntax-feelings.html) - | [YSH Style Guide](style-guide.html) - | [What Breaks When You Upgrade to YSH](upgrade-breakage.html) - | [YSH Language FAQ](ysh-faq.html) - - Comparisons: [YSH vs. Shell](ysh-vs-shell.html) | [YSH Expressions vs. - Python](ysh-vs-python.html) - - Features: [Egg Expressions (YSH Regexes)](eggex.html) +OSH: + +- [Known Differences Between OSH and Other Shells](known-differences.html) | + [Quirks](quirks.html) | [Headless Mode](headless.html) | [Shell + Idioms](shell-idioms.html) + +YSH: + +- [A Feel For YSH Syntax](syntax-feelings.html) | [YSH Style + Guide](style-guide.html) | [What Breaks When You Upgrade to + YSH](upgrade-breakage.html) | [YSH Language FAQ](ysh-faq.html) + +- Comparisons: [YSH vs. Shell](ysh-vs-shell.html) | [YSH Expressions vs. + Python](ysh-vs-python.html) + +- Features: [Egg Expressions (YSH Regexes)](eggex.html) | [YSH Regex API](ysh-regex-api.html) | [Guide to YSH Error Handling](ysh-error.html) | [Guide to Procs and Funcs](proc-func.html) @@ -39,14 +41,24 @@ See [All Docs](index.html) for links to drafts. | [Simple Word Evaluation](simple-word-eval.html) | [Variable Declaration, Mutation, and Scope](variables.html) | [Hay - Custom Languages for Unix Systems](hay.html) -- Data Languages: - - [JSON](json.html) | [J8 Notation](j8-notation.html) | - [BYO Protocols](byo.html) -- Language Design: - - [Syntactic Concepts](syntactic-concepts.html) + +Data Languages: + +- [JSON](json.html) | [J8 Notation](j8-notation.html) | [BYO + Protocols](byo.html) + +Language Design: + +- [Syntactic Concepts](syntactic-concepts.html) | [Command vs. Expression Mode](command-vs-expression-mode.html) | [Language Influences](language-influences.html) - Notes: [Novelties in OSH and YSH](novelties.html) | [Warts](warts.html) -- Reference: - - [Oils Error Catalog, With Hints](error-catalog.html) + +The Shared Oils Runtime + +- [Types in the Oils Runtime](types.html) | [Tracing Execution](xtrace.html) + +Reference: + +- [Oils Error Catalog, With Hints](error-catalog.html) diff --git a/doc/style-guide.md b/doc/style-guide.md index ea748d78df..804b256194 100644 --- a/doc/style-guide.md +++ b/doc/style-guide.md @@ -85,7 +85,13 @@ Example: echo 'failed' } -## Related +## Appendix + +### Reserved Names + +As in Python, names like `__provide__` are reserved by the interpreter. + +### Related - [Shell Language Idioms](shell-idioms.html) - [A Feel For YSH Syntax](syntax-feelings.html) diff --git a/doc/types.md b/doc/types.md index 9bd5446e72..85182a23c2 100644 --- a/doc/types.md +++ b/doc/types.md @@ -1,29 +1,75 @@ --- -in_progress: yes default_highlighter: oils-sh --- -YSH Types - Atoms, Mutable Containers, Reflection, Objects +Types in the Oils Runtime - OSH and YSH =========== -- Atoms -- Mutable Containers -- Reflection -- Objects - - See [YSH Objects](objects.html) - - +This doc lists the type of values in the Oils runtime.
-## Atoms +## Seven Atoms + +These types are immutable: + +- `Null Str Int Float` +- `Range` +- `Eggex Match` + +Of these types, OSH only uses `Str`. That is, the string type is the only type +shared between OSH and YSH. + +## Four Mutable Containers + +For YSH: + +- `List Dict` + +For bash compatibility in OSH: + +- `BashArray BashAssoc` + +## `Obj` is for User-defined Types + +- `Obj` + +Objects allow **polymorphism**. See [YSH Objects](objects.html). + +## Five Units of Code + +- `BoundFunc` (for methods) +- `BuiltinFunc Func` +- `BuiltinProc Proc` + +(These types are immutable) + +## Six Types for Reflection + +- `CommandFrag Command`, `ExprFrag Expr` (TODO) +- `Place Frame` + +(These types are immutable) + +## Appendix + +### The JSON Data Model + +These types can be serialized to and from JSON: + +- `Null Str Int Float List Dict` + +### Implementation Details -## Mutable Containers +These types used internally: -Dict List +- `value.Undef` - used when looking up a variable +- `value.Interrupted` - for SIGINT +- `value.Slice` - for a[1:2] -## Reflection +### Related -## Objects +- [Types and Methods](ref/chap-type-method.html) in the [Oils + Reference](ref/index.html) From 2672fbd713e27e20306996df6a051d7cb33cadef Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 17 Oct 2024 22:55:17 -0400 Subject: [PATCH 362/506] [ysh/builtin] str() uses the Stringify() function, for consistency This adds Null Bool Eggex. Move the special message for List to the error catalog. [doc] Polish types.md --- builtin/func_misc.py | 27 ++++++------- builtin/method_str.py | 11 +++--- core/value.asdl | 18 +++++---- doc/error-catalog.md | 26 ++++++++++++ doc/types.md | 81 +++++++++++++++++++++++++++++--------- osh/word_eval.py | 4 +- spec/ysh-convert.test.sh | 22 +++++++++-- test/ysh-runtime-errors.sh | 20 +++++++++- ysh/expr_eval.py | 7 ++-- ysh/val_ops.py | 35 ++++++++++------ 10 files changed, 181 insertions(+), 70 deletions(-) diff --git a/builtin/func_misc.py b/builtin/func_misc.py index 134c7db5ba..9cc62a04ae 100644 --- a/builtin/func_misc.py +++ b/builtin/func_misc.py @@ -135,6 +135,11 @@ def Call(self, rd): val = rd.PosValue() rd.Done() + # TODO: assert it's not Undef, Interrupted, Slice + # Then return an Obj type + # + # It would be nice if they were immutable, if we didn't have to create + # 23-24 dicts and 23-24 Obj on startup? return value.Str(ui.ValType(val)) @@ -154,7 +159,7 @@ def Call(self, rd): strs = [] # type: List[str] for i, el in enumerate(li): - strs.append(val_ops.Stringify(el, rd.LeftParenToken())) + strs.append(val_ops.Stringify(el, rd.LeftParenToken(), 'join() ')) return value.Str(delim.join(strs)) @@ -287,23 +292,13 @@ def Call(self, rd): val = rd.PosValue() rd.Done() - # TODO: Should we call Stringify here? That would handle Eggex. - - UP_val = val with tagswitch(val) as case: - if case(value_e.Int): - val = cast(value.Int, UP_val) - return value.Str(mops.ToStr(val.i)) - - elif case(value_e.Float): - val = cast(value.Float, UP_val) - return value.Str(str(val.f)) - - elif case(value_e.Str): + # Avoid extra allocation + if case(value_e.Str): return val - - raise error.TypeErr(val, 'str() expected Str, Int, or Float', - rd.BlamePos()) + else: + s = val_ops.Stringify(val, rd.LeftParenToken(), 'str() ') + return value.Str(s) class List_(vm._Callable): diff --git a/builtin/method_str.py b/builtin/method_str.py index d37e33d20b..9ac2a779ac 100644 --- a/builtin/method_str.py +++ b/builtin/method_str.py @@ -407,8 +407,7 @@ def Call(self, rd): if eggex_val: if '\0' in string: raise error.Structured( - 3, - "cannot replace by eggex on a string with NUL bytes", + 3, "cannot replace by eggex on a string with NUL bytes", rd.LeftParenToken()) ere = regex_translate.AsPosixEre(eggex_val) @@ -449,7 +448,7 @@ def Call(self, rd): # strings. Furthermore, they can only be used in string # contexts # eg. "$[1]" != "$1". - val_str = val_ops.Stringify(val, rd.LeftParenToken()) + val_str = val_ops.Stringify(val, rd.LeftParenToken(), '') if group == 0: arg0 = val_str else: @@ -472,8 +471,7 @@ def Call(self, rd): end = indices[1] if pos == end: raise error.Structured( - 3, - "eggex should never match the empty string", + 3, "eggex should never match the empty string", rd.LeftParenToken()) parts.append(string[pos:start]) # Unmatched substring @@ -523,7 +521,8 @@ def Call(self, rd): string_sep = string_sep_.s else: - raise error.TypeErr(sep, 'expected separator to be Eggex or Str', + raise error.TypeErr(sep, + 'expected separator to be Eggex or Str', rd.LeftParenToken()) count = mops.BigTruncate(rd.NamedInt("count", -1)) diff --git a/core/value.asdl b/core/value.asdl index 495bf01296..621ce4fc6a 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -89,9 +89,9 @@ module value # OSH/Bash types # - # Methods on state.Mem return value.Undef, but it's not visible in YSH. - # A var bound to Undef is different than no binding because of dynamic - # scope. Undef can shadow values lower on the stack. + # Methods on state::Mem return value.Undef, but it's not visible in YSH. + # Note: A var bound to Undef is different than no binding because of + # dynamic scope. Undef can shadow values lower on the stack. | Undef | Str(str s) @@ -104,8 +104,8 @@ module value | BashAssoc(Dict[str, str] d) - # DATA model for YSH follows JSON. Note: YSH doesn't have 'undefined' and - # 'null' like JavaScript, just 'null'. + # The DATA model for YSH follows JSON. Note: YSH doesn't have 'undefined' + # and 'null' like JavaScript, just 'null'. | Null | Bool(bool b) | Int(BigInt i) @@ -113,8 +113,12 @@ module value | List(List[value] items) | Dict(Dict[str, value] d) - # for polymorphism - should replace value.{IO,Module} too - # because they have attributes (functions), methods - not just methods + # Possible types + # value.Htm8 - a string that can be queried, with lazily materialized "views" + # value.Tsv8 - ditto + # value.Json8 - some kind of jq or JSONPath query language + + # Objects are for for polymorphism | Obj %Obj # for i in (0 .. n) { echo $i } # both ends are required diff --git a/doc/error-catalog.md b/doc/error-catalog.md index a3546c1dc5..ea55adaa39 100644 --- a/doc/error-catalog.md +++ b/doc/error-catalog.md @@ -330,6 +330,32 @@ Floating point numbers shouldn't be tested for equality. Alternatives: = abs(42.0 - x) < 0.1 = floatEquals(42.0, x) +### OILS-ERR-203 + + + +``` + var mylist = [1,2,3]; write $[mylist] + ^~ +[ -c flag ]:1: fatal: Expr sub got a List, which can't be stringified (OILS-ERR-203) +``` + +- Did you mean to use `@mylist` instead of `$mylist`? +- Did you mean to use `@[myfunc()]` instead of `$[myfunc()]`? +- Did you mean `$[join(mylist)]`? + +Or: + +- Do you have an element that can't be stringified in a list, like `['good', + {bad: true}]`? + + + + + ## Appendix ### Kinds of Errors from Oils diff --git a/doc/types.md b/doc/types.md index 85182a23c2..c29e713960 100644 --- a/doc/types.md +++ b/doc/types.md @@ -5,52 +5,85 @@ default_highlighter: oils-sh Types in the Oils Runtime - OSH and YSH =========== -This doc lists the type of values in the Oils runtime. +Here are all types of values in the Oils runtime, organized for understanding.
-## Seven Atoms +## Eight Atoms -These types are immutable: +Values of these types are immutable: -- `Null Str Int Float` -- `Range` -- `Eggex Match` +- `Null`, `Str Int Float` - data types +- `Range` - iteration over `3 .. 5` +- `Eggex Match` - pattern matching -Of these types, OSH only uses `Str`. That is, the string type is the only type -shared between OSH and YSH. +A type with one value: -## Four Mutable Containers +- `Stdin` - used for buffered line I/O in the YSH `for` loop -For YSH: + -- `List Dict` +The `Str` type is the only type shared between OSH and YSH. -For bash compatibility in OSH: + + +## Five Mutable Types + +YSH containers: + +- `List Dict` - arbitrarily recursive + +A special YSH type for "out params": + +- `Place` - created by `&myvar`, and mutated by `call place->setValue(42)` + +Containers for bash compatibility in OSH: + +- `BashArray BashAssoc` - flat ## `Obj` is for User-defined Types -- `Obj` +- `Obj` - has a prototype chain Objects allow **polymorphism**. See [YSH Objects](objects.html). +Modules and types are represented by `Obj` instances of a certain shape, not by +primitive types. + +1. Modules are `Obj` with attributes, and an `__invoke__` method. +1. Types are `Obj` with a `__str__` method, and are often compared for + identity. + +In general, Objects are mutable. Do not mutate modules or types! + ## Five Units of Code +Values of these types are immutable: + - `BoundFunc` (for methods) - `BuiltinFunc Func` - `BuiltinProc Proc` -(These types are immutable) +## Five Types for Reflection -## Six Types for Reflection +Values of these types are immutable: - `CommandFrag Command`, `ExprFrag Expr` (TODO) -- `Place Frame` -(These types are immutable) +A handle to a stack frame: + +- `Frame` - implicitly mutable, by `setvar`, etc. ## Appendix @@ -60,6 +93,17 @@ These types can be serialized to and from JSON: - `Null Str Int Float List Dict` +### Why Isn't Everything an Object? + +In YSH, the `Obj` type is used for **polymorphism** and reflection. + +Polymorphism is when you hide **different** kinds of data behind the **same** +interface. + +But most shell scripts deal with **concrete** textual data, which may be +JSON-like or TSV-like. The data is **not** hidden or encapsulated, and +shouldn't be. + ### Implementation Details These types used internally: @@ -73,3 +117,4 @@ These types used internally: - [Types and Methods](ref/chap-type-method.html) in the [Oils Reference](ref/index.html) + diff --git a/osh/word_eval.py b/osh/word_eval.py index 971678e0cf..0bce3bb9fc 100644 --- a/osh/word_eval.py +++ b/osh/word_eval.py @@ -234,10 +234,10 @@ def _ValueToPartValue(val, quoted, part_loc): return part_value.Array(val.d.values()) # Cases added for YSH - # value_e.List is also here - we use val_ops.stringify()s err message + # value_e.List is also here - we use val_ops.Stringify()s err message elif case(value_e.Null, value_e.Bool, value_e.Int, value_e.Float, value_e.Eggex, value_e.List): - s = val_ops.Stringify(val, loc.Missing) + s = val_ops.Stringify(val, loc.Missing, 'Word eval ') return Piece(s, quoted, not quoted) else: diff --git a/spec/ysh-convert.test.sh b/spec/ysh-convert.test.sh index f07051114d..9e102c9f88 100644 --- a/spec/ysh-convert.test.sh +++ b/spec/ysh-convert.test.sh @@ -90,13 +90,29 @@ inf ## END #### str() conversion -echo "$[str(1234)]" -echo "$[str(1.234)]" -echo "$[str('foo')]" +echo $[str(1234)] +echo $[str(1.234)] +echo $[str('foo')] + +echo + +# Added with Stringify() + +echo $[str(true)] +echo $[str(null)] +echo $[str(/d+/)] + +echo $[str([1,2,3])] + +## status: 3 ## STDOUT: 1234 1.234 foo + +true +null +[[:digit:]]+ ## END #### dict() converts from BashAssoc to Dict diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 583371c6a1..5cd8e34609 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -69,8 +69,6 @@ test-ysh-word-eval() { # this should be consistent _ysh-expr-error 'source $LIB_YSH/math.ysh; write -- @[identity([{key: "val"}])]' - _ysh-expr-error 'const x = [1, 2]; echo $x' - _ysh-should-run 'var x = [1, 2]; write @x' # errors in items @@ -84,6 +82,24 @@ test-ysh-word-eval() { _ysh-expr-error 'var x = /d+/; write @[x]' } +# Continuation of above +test-cannot-stringify-list() { + # List can't be stringified + _ysh-expr-error 'var mylist = [1,2,3]; write $mylist' + _ysh-expr-error 'var mylist = [1,2,3]; write $[mylist]' + + _ysh-should-run '= str(/d+/)' + + _ysh-expr-error '= str([1,2])' + _ysh-expr-error '= str({})' + + # Not sure if I like this join() behavior + _ysh-should-run '= join([true, null])' + + # Bad error + _ysh-expr-error '= join([[1,2], null])' +} + test-ysh-expr-eval() { _ysh-expr-error 'echo $[42 / 0 ]' diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 43de726254..e512bf9ad0 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -392,13 +392,12 @@ def EvalExprSub(self, part): with switch(part.left.id) as case: if case(Id.Left_DollarBracket): # $[join(x)] - s = val_ops.Stringify(val, loc.WordPart(part)) + s = val_ops.Stringify(val, loc.WordPart(part), 'Expr sub ') return Piece(s, False, False) elif case(Id.Lit_AtLBracket): # @[split(x)] - strs = val_ops.ToShellArray(val, - loc.WordPart(part), - prefix='Expr splice ') + strs = val_ops.ToShellArray(val, loc.WordPart(part), + 'Expr splice ') return part_value.Array(strs) else: diff --git a/ysh/val_ops.py b/ysh/val_ops.py index 7c1d9ad670..c697b1d5db 100644 --- a/ysh/val_ops.py +++ b/ysh/val_ops.py @@ -84,14 +84,23 @@ def ToCommandFrag(val, msg, blame_loc): raise error.TypeErr(val, msg, blame_loc) -def Stringify(val, blame_loc, prefix=''): +def Stringify(val, blame_loc, op_desc): # type: (value_t, loc_t, str) -> str """ - Used by + Args: + op_desc: could be empty string '' + or 'Expr Sub ' or 'Expr Splice ', with trailing space + + Used by: + + $[x] Expr Sub - stringify operator + @[x] Expr splice - each element is stringified + @x Splice value - $[x] stringify operator - @[x] expression splice - each element is stringified - @x splice value + str() Builtin function + join() Each element is stringified, e.g. join([1,2]) + Not sure I like join([null, true]), but it's consistent + Str.replace() ^"x = $x" after eggex conversion function """ if blame_loc is None: blame_loc = loc.Missing @@ -126,14 +135,16 @@ def Stringify(val, blame_loc, prefix=''): val = cast(value.Eggex, UP_val) s = regex_translate.AsPosixEre(val) # lazily converts to ERE - elif case(value_e.List): - raise error.TypeErrVerbose( - "%sgot a List, which can't be stringified. Perhaps use @ instead of $, or use join()" - % prefix, blame_loc) - else: + if val.tag() == value_e.List: + # Special error message for using the wrong sigil, or maybe join + raise error.TypeErrVerbose( + "%sgot a List, which can't be stringified (OILS-ERR-203)" % + op_desc, blame_loc) + raise error.TypeErr( - val, "%sexpected Null, Bool, Int, Float, Eggex" % prefix, + val, + "%sexpected one of (Null Bool Int Float Str Eggex)" % op_desc, blame_loc) return s @@ -158,7 +169,7 @@ def ToShellArray(val, blame_loc, prefix=''): # Note: it would be nice to add the index to the error message # prefix, WITHOUT allocating a string for every item for item in val.items: - strs.append(Stringify(item, blame_loc, prefix=prefix)) + strs.append(Stringify(item, blame_loc, prefix)) # I thought about getting rid of this to keep OSH and YSH separate, # but: From 60322ece24308823eea5171442dbfbcb6b01b067 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 00:38:19 -0400 Subject: [PATCH 363/506] [translation] Fix build mycpp at least detects this bad case, but it makes the code uglier --- ysh/val_ops.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ysh/val_ops.py b/ysh/val_ops.py index c697b1d5db..e65bdb6ad6 100644 --- a/ysh/val_ops.py +++ b/ysh/val_ops.py @@ -125,10 +125,6 @@ def Stringify(val, blame_loc, op_desc): elif case(value_e.Float): val = cast(value.Float, UP_val) - # TODO: what precision does this have? - # The default could be like awk or Python, and then we also allow - # ${myfloat %.3f} and more. - # Python 3 seems to give a few more digits than Python 2 for str(1.0/3) s = str(val.f) elif case(value_e.Eggex): @@ -136,6 +132,8 @@ def Stringify(val, blame_loc, op_desc): s = regex_translate.AsPosixEre(val) # lazily converts to ERE else: + pass # mycpp workaround + if val.tag() == value_e.List: # Special error message for using the wrong sigil, or maybe join raise error.TypeErrVerbose( From 272e89690dfa22e9070fb25dfe14819fcbcf8df0 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 02:44:09 -0400 Subject: [PATCH 364/506] [builtin/pp] Remove special case for pp [x] assert [42 === x] has a rationale But pp [x] shouldn't evaluate. It just prints To evaluate, you write pp (x), which is the normal rule. --- builtin/io_ysh.py | 31 ++++++++++--------------------- spec/ysh-builtin-meta.test.sh | 6 +++--- test/ysh-runtime-errors.sh | 3 +-- 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/builtin/io_ysh.py b/builtin/io_ysh.py index f20be00c51..f657c83d8f 100644 --- a/builtin/io_ysh.py +++ b/builtin/io_ysh.py @@ -7,7 +7,7 @@ from _devbuild.gen import arg_types from _devbuild.gen.runtime_asdl import cmd_value from _devbuild.gen.syntax_asdl import command_e, BraceGroup -from _devbuild.gen.value_asdl import value, value_e, value_t +from _devbuild.gen.value_asdl import value, value_e from asdl import format as fmt from core import error from core.error import e_usage @@ -19,7 +19,7 @@ from frontend import match from frontend import typed_args from mycpp import mylib -from mycpp.mylib import tagswitch, log, iteritems +from mycpp.mylib import log, iteritems from typing import TYPE_CHECKING, cast if TYPE_CHECKING: @@ -67,27 +67,17 @@ def _PrettyPrint(self, cmd_val): blame_tok = rd.LeftParenToken() - # It might be nice to add a string too, like - # pp 'my annotation' (actual) - # But the var name should meaningful in most cases - - UP_val = val - result = None # type: value_t - with tagswitch(val) as case: - if case(value_e.Expr): # Destructured assert [true === f()] - val = cast(value.Expr, UP_val) - - # In this case, we could get the unevaluated code string and - # print it. Although quoting the line seems enough. - result = self.expr_ev.EvalExpr(val.e, blame_tok) - else: - result = val - # Show it with location + # It looks like + # pp (42) + # ^ + # [ stdin ]:5: (Int) 42 + # We could also print with ! or -^- + self.stdout_.write('\n') excerpt, prefix = ui.CodeExcerptAndPrefix(blame_tok) self.stdout_.write(excerpt) - ui.PrettyPrintValue(prefix, result, self.stdout_) + ui.PrettyPrintValue(prefix, val, self.stdout_) return 0 @@ -100,8 +90,7 @@ def Run(self, cmd_val): action, action_loc = arg_r.Peek2() # Special cases - # pp (x) quotes its code location - # pp [x] also evaluates + # pp (x) quotes its code location, can also be pp [x] if action is None: return self._PrettyPrint(cmd_val) diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index 9f0d58d23b..768a870dbe 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -201,7 +201,7 @@ pp (42) shopt --set ysh:upgrade -pp [42] +pp [42] | sed 's/0x[a-f0-9]\+/[replaced]/' ## STDOUT: @@ -209,9 +209,9 @@ pp [42] ^ [ stdin ]:1: (Int) 42 - pp [42] + pp [42] | sed 's/0x[a-f0-9]\+/[replaced]/' ^ -[ stdin ]:5: (Int) 42 +[ stdin ]:5: ## END #### pp test_ supports BashArray, BashAssoc diff --git a/test/ysh-runtime-errors.sh b/test/ysh-runtime-errors.sh index 5cd8e34609..2bbfec3148 100755 --- a/test/ysh-runtime-errors.sh +++ b/test/ysh-runtime-errors.sh @@ -996,10 +996,9 @@ test-assert() { test-pp() { _ysh-expr-error 'pp (42/0)' - _ysh-expr-error 'pp [42/0]' # Multiple lines - _ysh-expr-error 'pp [42 + _ysh-should-run 'pp [42 /0]' _ysh-expr-error 'pp [5, 6]' From 9db992cc33fb026030f7df8f83a68d4ca35a6adf Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 12:32:46 -0400 Subject: [PATCH 365/506] [ysh builtin] Add basic Obj types: Bool Int Float Str For the flag parser. This can be made more general with __str__ on the Obj, and so forth. --- core/shell.py | 22 +++++++++++++++++++++- spec/ysh-builtin-meta.test.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/core/shell.py b/core/shell.py index c6cdc1d035..52f08971c4 100644 --- a/core/shell.py +++ b/core/shell.py @@ -10,7 +10,7 @@ from _devbuild.gen.option_asdl import option_i, builtin_i from _devbuild.gen.syntax_asdl import (loc, source, source_t, IntParamBox, debug_frame, debug_frame_t) -from _devbuild.gen.value_asdl import (value, value_e, value_t, Obj) +from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj) from core import alloc from core import comp_ui from core import dev @@ -590,6 +590,26 @@ def Main( vm_props = {} # type: Dict[str, value_t] vm_obj = Obj(Obj(None, vm_methods), vm_props) + # Add basic type objects for flag parser + # flag -v --verbose (Bool, help='foo') + # + # TODO: + # - can add __str__ method + # - Add other types like Dict, CommandFlag + # - Dict should have __invoke__ + # - List() Dict() Obj() can do shallow copy with __call__ + # - Bool() Int() Float() Str() List() Dict() conversions + # - type(x) should return these Obj, or perhaps typeObj(x) + + type_obj_methods = Obj(None, {}) + for tag in (value_e.Bool, value_e.Int, value_e.Float, value_e.Str): + type_name = value_str(tag, dot=False) + #log('%s %s' , type_name, tag) + type_obj = Obj(type_obj_methods, {'name': value.Str(type_name)}) + mem.AddBuiltin(type_name, type_obj) + + vm_obj = Obj(Obj(None, vm_methods), vm_props) + # Wire up circular dependencies. vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, prompt_ev, io_obj, tracer) diff --git a/spec/ysh-builtin-meta.test.sh b/spec/ysh-builtin-meta.test.sh index 768a870dbe..8d9a2f29b9 100644 --- a/spec/ysh-builtin-meta.test.sh +++ b/spec/ysh-builtin-meta.test.sh @@ -1,5 +1,32 @@ ## oils_failures_allowed: 1 +#### Builtin types + +pp test_ (Bool) +pp test_ (Int) +pp test_ (Float) +pp test_ (Str) +echo + +var b = Bool + +pp test_ (b is Bool) + +# Objects don't have equality, only identity +#pp test_ (b === Bool) + +pp test_ (id(b) === id(Bool)) + +## STDOUT: +(Obj) ("name":"Bool") --> () +(Obj) ("name":"Int") --> () +(Obj) ("name":"Float") --> () +(Obj) ("name":"Str") --> () + +(Bool) true +(Bool) true +## END + #### runproc shopt --set parse_proc parse_at @@ -270,3 +297,4 @@ pp value (repeat([123], 40)) | cat 123, 123, 123, 123, 123, 123, 123, 123, 123, 123 ] ## END + From b5c200fc1a9d79766f9672cb8590cf34ad475c61 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 12:50:27 -0400 Subject: [PATCH 366/506] [doc/ref] Add help topis, TODOs - io.stdin - vm.getFrame() - TODO: List->clear(), Dict->clear() Cleanup: remove duplicate statement --- core/shell.py | 2 -- doc/ref/chap-type-method.md | 46 +++++++++++++++++++++++++++++++++++++ doc/ref/toc-ysh.md | 11 +++++---- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/core/shell.py b/core/shell.py index 52f08971c4..8036523011 100644 --- a/core/shell.py +++ b/core/shell.py @@ -608,8 +608,6 @@ def Main( type_obj = Obj(type_obj_methods, {'name': value.Str(type_name)}) mem.AddBuiltin(type_name, type_obj) - vm_obj = Obj(Obj(None, vm_methods), vm_props) - # Wire up circular dependencies. vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex, prompt_ev, io_obj, tracer) diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 484c3710b4..272c760f36 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -347,6 +347,15 @@ Reverses a list in place. call fruits->reverse() echo @fruits # => pear banana apple +### clear() + +TODO: + +Remove all entries from the List: + + call mylist->clear() + + ## Dict A Dict contains an ordered sequence of key-value pairs. Given the key, the @@ -374,8 +383,20 @@ Ensures that the given key does not exist in the dictionary. ### inc() +TODO + ### accum() +TODO + +### clear() + +TODO: + +Remove all entries from the Dict: + + call mydict->clear() + ## Range A `Range` is a pair of two numbers, like `42 .. 45`. @@ -518,6 +539,17 @@ User-defined procs. ## IO +### stdin + +Returns the singleton `stdin` value, which you can iterate over: + + for line in (io.stdin) { + echo $line + } + +This is buffered line-based I/O, as opposed to the unbuffered I/O of the `read` +builtin. + ### eval() Evaluate a command, and return `null`. @@ -638,3 +670,17 @@ Then invoke it like a proc: TODO +### `__str__` + +TODO + +## VM + +### getFrame() + +TODO + + var frame = vm.getFrame(-1) # local frame + var frame = vm.getFrame(0) # global frame + + var frame = vm.getFrame(-2) # calling frame, for my-cd { echo } diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 40d94ef454..310a96fc02 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -46,9 +46,9 @@ error handling, and more. startsWith() endsWith() upper() lower() search() leftMatch() - [List] List/append() pop() extend() indexOf() - X insert() X remove() reverse() - [Dict] erase() X inc() X accum() + [List] List/append() pop() extend() indexOf() + X insert() X remove() reverse() X clear() + [Dict] erase() X inc() X accum() X clear() [Range] [Eggex] [Match] group() start() end() @@ -60,10 +60,11 @@ error handling, and more. Frame X [Func] name() location() toJson() X [Proc] name() location() toJson() - [IO] eval() evalToDict() captureStdout() + [IO] stdin eval() evalToDict() + captureStdout() promptVal() X time() X strftime() X glob() - [Obj] __invoke__ X __call__ + [Obj] __invoke__ X __call__ X __str__ [VM] X getFrame() ``` From 64a3f2fa2a437418c68cac1b0ee46f88816bc76c Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 12:55:25 -0400 Subject: [PATCH 367/506] [translation] Fix build --- core/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/shell.py b/core/shell.py index 8036523011..36d1ca1c52 100644 --- a/core/shell.py +++ b/core/shell.py @@ -602,7 +602,7 @@ def Main( # - type(x) should return these Obj, or perhaps typeObj(x) type_obj_methods = Obj(None, {}) - for tag in (value_e.Bool, value_e.Int, value_e.Float, value_e.Str): + for tag in [value_e.Bool, value_e.Int, value_e.Float, value_e.Str]: type_name = value_str(tag, dot=False) #log('%s %s' , type_name, tag) type_obj = Obj(type_obj_methods, {'name': value.Str(type_name)}) From 1e9bfbe71283f9200c570fa9c98b1c9b4e4eda04 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 12:57:01 -0400 Subject: [PATCH 368/506] [builtin/pp] Show object cycles with (...) Not {...}, which for the Dict type --- data_lang/j8.py | 4 ++-- spec/ysh-printing.test.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data_lang/j8.py b/data_lang/j8.py index 19f64a11f8..1d00538f56 100644 --- a/data_lang/j8.py +++ b/data_lang/j8.py @@ -139,7 +139,7 @@ def Utf8Encode(code): return ''.join(tmp) -SHOW_CYCLES = 1 << 1 # show as [...] or {...} I think, with object ID +SHOW_CYCLES = 1 << 1 # show as [...] or {...} or (...), with object ID SHOW_NON_DATA = 1 << 2 # non-data objects like Eggex can be LOSSY_JSON = 1 << 3 # JSON may lose data about strings INF_NAN_ARE_NULL = 1 << 4 # for JSON @@ -583,7 +583,7 @@ def Print(self, val, level=0): if self.visiting.get(heap_id, False): if self.options & SHOW_CYCLES: - self.buf.write('{...}') + self.buf.write('(...)') return else: # node.js prints which key closes the cycle diff --git a/spec/ysh-printing.test.sh b/spec/ysh-printing.test.sh index c859838563..cc866baa4b 100644 --- a/spec/ysh-printing.test.sh +++ b/spec/ysh-printing.test.sh @@ -342,9 +342,9 @@ pp test_ (two) ## STDOUT: (Obj) ("z":99) --> ("__foo__":null) -(Obj) ("z":99,"cycle":{...}) --> ("__foo__":null) +(Obj) ("z":99,"cycle":(...)) --> ("__foo__":null) -(List) [("z":99,"cycle":{...}) --> ("__foo__":null),("z":99,"cycle":{...}) --> ("__foo__":null)] +(List) [("z":99,"cycle":(...)) --> ("__foo__":null),("z":99,"cycle":(...)) --> ("__foo__":null)] ## END From e3b1994f4bfcc55ede0abf3564e2e70af76c9ec8 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 13:00:02 -0400 Subject: [PATCH 369/506] [spec/ysh-regex-api] Test case for lexical scope in Str.replace() for value.Expr capture - Change => to . --- spec/ysh-regex-api.test.sh | 207 +++++++++++++++++++++---------------- 1 file changed, 118 insertions(+), 89 deletions(-) diff --git a/spec/ysh-regex-api.test.sh b/spec/ysh-regex-api.test.sh index 0a0df973ec..43321daf36 100644 --- a/spec/ysh-regex-api.test.sh +++ b/spec/ysh-regex-api.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 0 +## oils_failures_allowed: 1 #### s ~ regex and s !~ regex shopt -s ysh:upgrade @@ -192,18 +192,18 @@ start=-1 end=-1 var s = '= Hi5- Bye6-' -var m = s => search(/ '-' ; i /) -echo "g0 $[m => start(0)] $[m => end(0)] $[m => group(0)]" -echo "g1 $[m => start(1)] $[m => end(1)] $[m => group(1)]" -echo "g2 $[m => start(2)] $[m => end(2)] $[m => group(2)]" +var m = s.search(/ '-' ; i /) +echo "g0 $[m.start(0)] $[m.end(0)] $[m.group(0)]" +echo "g1 $[m.start(1)] $[m.end(1)] $[m.group(1)]" +echo "g2 $[m.start(2)] $[m.end(2)] $[m.group(2)]" echo --- -var pos = m => end(0) # search from end position -var m = s => search(/ '-' ; i /, pos=pos) -echo "g0 $[m => start(0)] $[m => end(0)] $[m => group(0)]" -echo "g1 $[m => start(1)] $[m => end(1)] $[m => group(1)]" -echo "g2 $[m => start(2)] $[m => end(2)] $[m => group(2)]" +var pos = m.end(0) # search from end position +var m = s.search(/ '-' ; i /, pos=pos) +echo "g0 $[m.start(0)] $[m.end(0)] $[m.group(0)]" +echo "g1 $[m.start(1)] $[m.end(1)] $[m.group(1)]" +echo "g2 $[m.start(2)] $[m.end(2)] $[m.group(2)]" ## STDOUT: g0 2 6 Hi5- @@ -229,12 +229,12 @@ for pat in ([anchored, free]) { var pos = 0 while (true) { - var m = s => search(pat, pos=pos) + var m = s.search(pat, pos=pos) if (not m) { break } - echo $[m => group(0)] - setvar pos = m => end(0) + echo $[m.group(0)] + setvar pos = m.end(0) } } @@ -252,16 +252,16 @@ pat=([[:digit:]]+)- var s = '= hi5- bye6-' -var m = s => search('([[:alpha:]]+)([[:digit:]]+)-') -echo "g0 $[m => start(0)] $[m => end(0)] $[m => group(0)]" -echo "g1 $[m => start(1)] $[m => end(1)] $[m => group(1)]" -echo "g2 $[m => start(2)] $[m => end(2)] $[m => group(2)]" +var m = s.search('([[:alpha:]]+)([[:digit:]]+)-') +echo "g0 $[m.start(0)] $[m.end(0)] $[m.group(0)]" +echo "g1 $[m.start(1)] $[m.end(1)] $[m.group(1)]" +echo "g2 $[m.start(2)] $[m.end(2)] $[m.group(2)]" echo --- -var m = s[2:] => leftMatch('([[:alpha:]]+)([[:digit:]]+)-') -echo "g0 $[m => start(0)] $[m => end(0)] $[m => group(0)]" -echo "g1 $[m => start(1)] $[m => end(1)] $[m => group(1)]" -echo "g2 $[m => start(2)] $[m => end(2)] $[m => group(2)]" +var m = s[2:].leftMatch('([[:alpha:]]+)([[:digit:]]+)-') +echo "g0 $[m.start(0)] $[m.end(0)] $[m.group(0)]" +echo "g1 $[m.start(1)] $[m.end(1)] $[m.group(1)]" +echo "g2 $[m.start(2)] $[m.end(2)] $[m.group(2)]" ## STDOUT: g0 2 6 hi5- @@ -273,7 +273,7 @@ g1 0 2 hi g2 2 3 5 ## END -#### Str=>leftMatch() can implement lexer pattern +#### Str.leftMatch() can implement lexer pattern shopt -s ysh:upgrade @@ -286,17 +286,17 @@ proc show-tokens (s) { while (true) { echo "pos=$pos" - var m = s=>leftMatch(lexer, pos=pos) + var m = s.leftMatch(lexer, pos=pos) if (not m) { break } # TODO: add groups() - #var groups = [m => group(1), m => group(2), m => group(3)] - echo "$[m => group(1)]/$[m => group(2)]/$[m => group(3)]/" + #var groups = [m.group(1), m.group(2), m.group(3)] + echo "$[m.group(1)]/$[m.group(2)]/$[m.group(3)]/" echo - setvar pos = m => end(0) + setvar pos = m.end(0) } } @@ -325,18 +325,18 @@ null/ab/null/ pos=2 ## END -#### Named captures with m => group() +#### Named captures with m.group() shopt -s ysh:all var s = 'zz 2020-08-20' var pat = / '-' / -var m = s => search(pat) -argv.py $[m => group('year')] $[m => group('month')] -echo $[m => start('year')] $[m => end('year')] -echo $[m => start('month')] $[m => end('month')] +var m = s.search(pat) +argv.py $[m.group('year')] $[m.group('month')] +echo $[m.start('year')] $[m.end('year')] +echo $[m.start('month')] $[m.end('month')] -argv.py $[m => group('oops')] +argv.py $[m.group('oops')] echo 'error' ## status: 3 @@ -391,25 +391,25 @@ var pat = / 'when: ' ( | ) / #echo $pat proc show-groups (; m) { - echo 0 $[m => group(0)] - echo 1 $[m => group(1)] # this is everything except when - echo 2 $[m => group(2)] + echo 0 $[m.group(0)] + echo 1 $[m.group(1)] # this is everything except when + echo 2 $[m.group(2)] echo - echo $[m => group('two')] - echo $[m => group('year')] $[m => group('month')] - echo $[m => group('hour')] $[m => group('minute')] $[m => group('secs')] + echo $[m.group('two')] + echo $[m.group('year')] $[m.group('month')] + echo $[m.group('hour')] $[m.group('minute')] $[m.group('secs')] } -var m = 'when: 2023-10' => leftMatch(pat) +var m = 'when: 2023-10'.leftMatch(pat) show-groups (m) -var m = 'when: 23:30' => leftMatch(pat) +var m = 'when: 23:30'.leftMatch(pat) echo --- show-groups (m) -var m = 'when: 23:30:59' => leftMatch(pat) +var m = 'when: 23:30:59'.leftMatch(pat) echo --- show-groups (m) @@ -452,9 +452,9 @@ if (s ~ pat) { echo $[type(g1)] $[type(g2)] } -var m = s => search(pat) +var m = s.search(pat) if (m) { - echo $[m => group(1) => type()] $[m => group(2) => type()] + echo $[m.group(1) => type()] $[m.group(2) => type()] } ## STDOUT: @@ -480,10 +480,10 @@ if (s ~ pat) { echo $[type(g1)] $[type(g2)] } -var m = s => search(pat) +var m = s.search(pat) if (m) { - echo $[m => group('right')] - echo $[m => group('left') => type()] $[m => group('right') => type()] + echo $[m.group('right')] + echo $[m.group('left') => type()] $[m.group('right') => type()] } ## STDOUT: @@ -656,16 +656,16 @@ sq char class ## END -#### Str => replace(Str, Str) +#### Str.replace(Str, Str) shopt --set ysh:all var mystr = 'abca' -write $[mystr => replace('a', 'A')] # Two matches -write $[mystr => replace('b', 'B')] # One match -write $[mystr => replace('x', 'y')] # No matches +write $[mystr.replace('a', 'A')] # Two matches +write $[mystr.replace('b', 'B')] # One match +write $[mystr.replace('x', 'y')] # No matches -write $[mystr => replace('abc', '')] # Empty substitution -write $[mystr => replace('', 'new')] # Empty substring +write $[mystr.replace('abc', '')] # Empty substitution +write $[mystr.replace('', 'new')] # Empty substring ## STDOUT: AbcA aBca @@ -674,43 +674,43 @@ a newanewbnewcnewanew ## END -#### Str => replace(Eggex, Str) +#### Str.replace(Eggex, Str) shopt --set ysh:all var mystr = 'mangled----kebab--case' -write $[mystr => replace(/ '-'+ /, '-')] +write $[mystr.replace(/ '-'+ /, '-')] setvar mystr = 'smaller-to-bigger' -write $[mystr => replace(/ '-'+ /, '---')] +write $[mystr.replace(/ '-'+ /, '---')] ## STDOUT: mangled-kebab-case smaller---to---bigger ## END -#### Str => replace(Eggex, Expr) +#### Str.replace(Eggex, Expr) shopt --set ysh:all var mystr = 'name: Bob' -write $[mystr => replace(/ 'name: ' /, ^"Hello $1")] -write $[mystr => replace(/ 'name: ' /, ^"Hello $1 (extracted from '$0')")] +write $[mystr.replace(/ 'name: ' /, ^"Hello $1")] +write $[mystr.replace(/ 'name: ' /, ^"Hello $1 (extracted from '$0')")] ## STDOUT: Hello Bob Hello Bob (extracted from 'name: Bob') ## END -#### Str => replace(*, Expr), $0 +#### Str.replace(*, Expr), $0 shopt --set ysh:all # Functionality var mystr = 'class Foo: # this class is called Foo' -write $[mystr => replace("Foo", ^"$0Bar")] -write $[mystr => replace(/ 'Foo' /, ^"$0Bar")] +write $[mystr.replace("Foo", ^"$0Bar")] +write $[mystr.replace(/ 'Foo' /, ^"$0Bar")] # Edge-cases var dollar0 = "$0" func f() { return ("$0") } -write $["foo" => replace("o", "$0") === "f$dollar0$dollar0"] -write $["foo" => replace("o", ^[f()]) === "f$dollar0$dollar0"] +write $["foo".replace("o", "$0") === "f$dollar0$dollar0"] +write $["foo".replace("o", ^[f()]) === "f$dollar0$dollar0"] write $[f() === "$dollar0"] ## STDOUT: class FooBar: # this class is called FooBar @@ -720,34 +720,34 @@ true true ## END -#### Str => replace(Eggex, Expr), scopes +#### Str.replace(Eggex, Expr), scopes shopt --set ysh:all var mystr = '123' var anotherVar = 'surprise!' -write $[mystr => replace(/ /, ^"Hello $1 ($anotherVar)")] +write $[mystr.replace(/ /, ^"Hello $1 ($anotherVar)")] var globalName = '456' -write $[mystr => replace(/ /, ^"Hello $globalName")] +write $[mystr.replace(/ /, ^"Hello $globalName")] -write $[mystr => replace(/ /, ^"Hello $localName, $globalName")] +write $[mystr.replace(/ /, ^"Hello $localName, $globalName")] ## STDOUT: Hello 123 (surprise!) Hello 123 Hello 123, 456 ## END -#### Str => replace(Eggex, *, count) +#### Str.replace(Eggex, *, count) shopt --set ysh:all var mystr = '1abc2abc3abc' for count in (-2..4) { - write $[mystr => replace('abc', "-", count=count)] - write $[mystr => replace('abc', ^"-", count=count)] - write $[mystr => replace(/ [a-z]+ /, "-", count=count)] - write $[mystr => replace(/ [a-z]+ /, "-", count=count)] + write $[mystr.replace('abc', "-", count=count)] + write $[mystr.replace('abc', ^"-", count=count)] + write $[mystr.replace(/ [a-z]+ /, "-", count=count)] + write $[mystr.replace(/ [a-z]+ /, "-", count=count)] } ## STDOUT: 1-2-3- @@ -776,12 +776,12 @@ for count in (-2..4) { 1-2-3- ## END -#### Str => replace(Str, Str), empty new/old strings +#### Str.replace(Str, Str), empty new/old strings var mystr = 'abca' -write $[mystr => replace('abc', '')] # Empty substitution -write $[mystr => replace('', 'new')] # Empty substring -write $[mystr => replace('', 'new', count=1)] # Empty substring, count != -1 -write $[mystr => replace('', 'new', count=10)] # Empty substring, count too large +write $[mystr.replace('abc', '')] # Empty substitution +write $[mystr.replace('', 'new')] # Empty substring +write $[mystr.replace('', 'new', count=1)] # Empty substring, count != -1 +write $[mystr.replace('', 'new', count=10)] # Empty substring, count too large ## STDOUT: a newanewbnewcnewanew @@ -789,22 +789,22 @@ newabca newanewbnewcnewanew ## END -#### Str => replace(Eggex, Lazy), convert_func +#### Str.replace(Eggex, Lazy), convert_func shopt --set ysh:all var mystr = '123' -write $[mystr => replace(/ /, ^"$[n + 1]")] +write $[mystr.replace(/ /, ^"$[n + 1]")] # values automatically get stringified -write $[mystr => replace(/ /, ^"$1")] +write $[mystr.replace(/ /, ^"$1")] func not_str(inp) { return ({ "value": inp }) } # should fail to stringify $1 -try { call mystr => replace(/ /, ^"$1") } +try { call mystr.replace(/ /, ^"$1") } write status=$_status ## STDOUT: 124 @@ -812,13 +812,13 @@ write status=$_status status=3 ## END -#### Str => replace(Eggex, *), eflags +#### Str.replace(Eggex, *), eflags shopt --set ysh:all var mystr = $'1-2-3\n4-5' -write $[mystr => replace(/ d+ /, ^"[$0]")] -write $[mystr => replace(/ ^ d+ /, ^"[$0]")] -write $[mystr => replace(/ ^ d+ ; reg_newline /, ^"[$0]")] +write $[mystr.replace(/ d+ /, ^"[$0]")] +write $[mystr.replace(/ ^ d+ /, ^"[$0]")] +write $[mystr.replace(/ ^ d+ ; reg_newline /, ^"[$0]")] ## STDOUT: [1]-[2]-[3] [4]-[5] @@ -828,20 +828,49 @@ write $[mystr => replace(/ ^ d+ ; reg_newline /, ^"[$0]")] [4]-5 ## END -#### Str => replace(Eggex, *), guard against infinite loop +#### Str.replace(Eggex, *), guard against infinite loop shopt --set ysh:all var mystr = 'foo bar baz' -write $[mystr => replace(/ space* /, ' ')] +write $[mystr.replace(/ space* /, ' ')] ## status: 3 ## STDOUT: ## END -#### Str => replace(Eggex, *), str cannot contain NUL bytes +#### Str.replace(Eggex, *), str cannot contain NUL bytes shopt --set ysh:all var mystr = b'foo bar baz\y00' -write $[mystr => replace(/ space+ /, ' ')] +write $[mystr.replace(/ space+ /, ' ')] ## status: 3 ## STDOUT: ## END + +#### Str.replace() lexical scope +shopt --set ysh:upgrade + +var pat = / 's' / + +proc p { + var x = 'x' + + #var template = ^"[$x $0 $x]" + var template = ^"[$x $0 $1 $x]" + pp test_ (template) + + var s = 'mystr' + + var new = s.replace(pat, template) + + func myreplace(s, template) { + return (s.replace(pat, template)) + } + + echo $new + echo $[myreplace(s, template)] +} + +p + +## STDOUT: +## END From 05f364576c2591448e4017fac9272623194c2454 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 13:53:18 -0400 Subject: [PATCH 370/506] [test/spec] Failing test cases for value.Expr closures Added new file spec/ysh-closures Also change the value.Expr schema. --- core/value.asdl | 8 ++- spec/ysh-builtin-eval.test.sh | 32 ------------ spec/ysh-closures.test.sh | 91 +++++++++++++++++++++++++++++++++++ spec/ysh-object.test.sh | 29 ----------- spec/ysh-regex-api.test.sh | 64 +++++++++++++++++++++--- test/spec.sh | 4 ++ ysh/expr_eval.py | 2 +- ysh/func_proc.py | 5 +- 8 files changed, 161 insertions(+), 74 deletions(-) create mode 100644 spec/ysh-closures.test.sh diff --git a/core/value.asdl b/core/value.asdl index 621ce4fc6a..4f9e532b4c 100644 --- a/core/value.asdl +++ b/core/value.asdl @@ -180,8 +180,12 @@ module value # Unevaluated CODE types: ExprFrag, Expr, CommandFrag, Command # - # ^[42 + a[i]] - | Expr(expr e) + # This can be the output of parseExpr()? + #| ExprFrag(expr e) + + # var x = ^[42 + a[i]] + # my-ls | where [size > 10] + | Expr(expr e, Dict[str, Cell] captured_frame) # This is an UNBOUND command, like # ^(echo 1; echo 2) and cd { echo 1; echo 2 } diff --git a/spec/ysh-builtin-eval.test.sh b/spec/ysh-builtin-eval.test.sh index d9b5e8f51c..ea0e9c7cdc 100644 --- a/spec/ysh-builtin-eval.test.sh +++ b/spec/ysh-builtin-eval.test.sh @@ -573,38 +573,6 @@ inner=z inner2=z ## END -#### Block Closures in a Loop ! - -proc task (; tasks; ; b) { - call tasks->append(b) -} - -func makeTasks() { - var tasks = [] - var x = 'x' - for __hack__ in (0 .. 3) { - var i = __hack__ - var j = i + 2 - task (tasks) { echo "$x: i = $i, j = $j" } - } - return (tasks) -} - -var blocks = makeTasks() -#= blocks - -for b in (blocks) { - call io->eval(b) -} - -## STDOUT: -x: i = 0, j = 2 -x: i = 1, j = 3 -x: i = 2, j = 4 -## END - - - #### io->evalInFrame() can express try, cd builtins var frag = ^(echo $i) diff --git a/spec/ysh-closures.test.sh b/spec/ysh-closures.test.sh new file mode 100644 index 0000000000..cfb8768091 --- /dev/null +++ b/spec/ysh-closures.test.sh @@ -0,0 +1,91 @@ +## oils_failures_allowed: 1 + +#### Expr Closures in a Loop ! +shopt --set ysh:upgrade + +proc task (; tasks, expr) { + call tasks->append(expr) +} + +func makeTasks() { + var tasks = [] + var x = 'x' + for __hack__ in (0 .. 3) { + var i = __hack__ + var j = i + 2 + task (tasks, ^["$x: i = $i, j = $j"]) + } + return (tasks) +} + +var exprs = makeTasks() +#= blocks + +for ex in (exprs) { + var s = evalExpr(ex) + echo $s +} + +## STDOUT: +## END + +#### Block Closures in a Loop ! +shopt --set ysh:upgrade + +proc task (; tasks; ; b) { + call tasks->append(b) +} + +func makeTasks() { + var tasks = [] + var x = 'x' + for __hack__ in (0 .. 3) { + var i = __hack__ + var j = i + 2 + task (tasks) { echo "$x: i = $i, j = $j" } + } + return (tasks) +} + +var blocks = makeTasks() +#= blocks + +for b in (blocks) { + call io->eval(b) +} + +## STDOUT: +x: i = 0, j = 2 +x: i = 1, j = 3 +x: i = 2, j = 4 +## END + + +#### Explicit __invoke__ for "objects in a loop", not closures in a loop +shopt --set ysh:upgrade + +var procs = [] +for i in (0 .. 3) { + proc __invoke__ (; self) { + echo "i = $[self.i]" + } + var methods = Object(null, {__invoke__}) + var obj = Object(methods, {i}) + call procs->append(obj) +} + +for p in (procs) { + p +} + +# TODO: sugar +# proc p (; self) capture {i} { +# echo "i = $[self.i]" +# } +# call procs->append(p) + +## STDOUT: +i = 0 +i = 1 +i = 2 +## END diff --git a/spec/ysh-object.test.sh b/spec/ysh-object.test.sh index f8f12e0713..af5aa62e4b 100644 --- a/spec/ysh-object.test.sh +++ b/spec/ysh-object.test.sh @@ -230,32 +230,3 @@ pp test_ (instance) ## STDOUT: (Obj) ("foo":1,"bar":2,"x":3) --> ("foo":42,"bar":[1,2]) --> ("foo":"zz") ## END - - -#### Closures in a loop idiom - -var procs = [] -for i in (0 .. 3) { - proc __invoke__ (; self) { - echo "i = $[self.i]" - } - var methods = Object(null, {__invoke__}) - var obj = Object(methods, {i}) - call procs->append(obj) -} - -for p in (procs) { - p -} - -# TODO: sugar -# proc p (; self) capture {i} { -# echo "i = $[self.i]" -# } -# call procs->append(p) - -## STDOUT: -i = 0 -i = 1 -i = 2 -## END diff --git a/spec/ysh-regex-api.test.sh b/spec/ysh-regex-api.test.sh index 43321daf36..48e847f7d2 100644 --- a/spec/ysh-regex-api.test.sh +++ b/spec/ysh-regex-api.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 2 #### s ~ regex and s !~ regex shopt -s ysh:upgrade @@ -846,28 +846,76 @@ write $[mystr.replace(/ space+ /, ' ')] ## STDOUT: ## END -#### Str.replace() lexical scope +#### Str.replace() at top level shopt --set ysh:upgrade +var s = 'mystr' var pat = / 's' / +var template = ^"[$x $0 $1 $x]" +pp test_ (template) + +var x = 'x' + +var new = s.replace(pat, template) +echo 'replace ' $new + +func myreplace(s, template) { + return (s.replace(pat, template)) +} + +echo myreplace $[myreplace(s, template)] + +## STDOUT: + +replace my[x st t x]r +myreplace my[x st t x]r +## END + +#### Str.replace() lexical scope with ^"" +shopt --set ysh:upgrade + +var s = 'mystr' +var pat = / 's' / +var template = ^"[$x $0 $1 $x]" +pp test_ (template) proc p { var x = 'x' - #var template = ^"[$x $0 $x]" - var template = ^"[$x $0 $1 $x]" - pp test_ (template) + var new = s.replace(pat, template) + echo 'replace ' $new + + func myreplace(s, template) { + return (s.replace(pat, template)) + } - var s = 'mystr' + echo myreplace $[myreplace(s, template)] +} + +p + +## STDOUT: +## END + +#### Str.replace() lexical scope with ^[] +shopt --set ysh:upgrade + +var s = 'mystr' +var pat = / 's' / +var template = ^['[' ++ x ++ ' ' ++ $0 ++ ' ' ++ $1 ++ ' ' ++ x ++ ']'] +pp test_ (template) + +proc p { + var x = 'x' var new = s.replace(pat, template) + echo 'replace ' $new func myreplace(s, template) { return (s.replace(pat, template)) } - echo $new - echo $[myreplace(s, template)] + echo myreplace $[myreplace(s, template)] } p diff --git a/test/spec.sh b/test/spec.sh index e5a877c01f..c0aa1d4d79 100755 --- a/test/spec.sh +++ b/test/spec.sh @@ -838,6 +838,10 @@ ysh-object() { run-file ysh-object "$@" } +ysh-closures() { + run-file ysh-closures "$@" +} + ysh-func() { run-file ysh-func "$@" } diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index e512bf9ad0..845bb02d4a 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -1311,7 +1311,7 @@ def _EvalExpr(self, node): elif case(expr_e.Literal): # ^[1 + 2] node = cast(expr.Literal, UP_node) - return value.Expr(node.inner) + return value.Expr(node.inner, self.mem.CurrentFrame()) elif case(expr_e.Lambda): # |x| x+1 syntax is reserved # TODO: Location information for |, or func diff --git a/ysh/func_proc.py b/ysh/func_proc.py index 84478a9b59..96af74a7b4 100644 --- a/ysh/func_proc.py +++ b/ysh/func_proc.py @@ -231,7 +231,7 @@ def EvalTypedArgsToProc( # Defer evaluation by wrapping in value.Expr for exp in ty.pos_args: - proc_args.pos_args.append(value.Expr(exp)) + proc_args.pos_args.append(value.Expr(exp, current_frame)) # TODO: ...spread is illegal n1 = ty.named_args @@ -239,7 +239,8 @@ def EvalTypedArgsToProc( proc_args.named_args = NewDict() for named_arg in n1: name = lexer.TokenVal(named_arg.name) - proc_args.named_args[name] = value.Expr(named_arg.value) + proc_args.named_args[name] = value.Expr( + named_arg.value, current_frame) # TODO: ...spread is illegal else: # json write (x) From bb81474e9acffb6255dc7fc74c0c2e6713a85f3b Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 14:17:16 -0400 Subject: [PATCH 371/506] [ysh breaking] Move evalExpr() to io->evalExpr() Updated doc/ref, which has examples of expressions that have effects. --- builtin/func_reflect.py | 1 + core/shell.py | 3 ++- doc/ref/chap-builtin-func.md | 11 ----------- doc/ref/chap-type-method.md | 15 +++++++++++++++ doc/ref/toc-ysh.md | 4 ++-- spec/ysh-closures.test.sh | 2 +- spec/ysh-expr.test.sh | 16 ++++++++-------- 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index e160aaca08..92fa5a3cd6 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -215,6 +215,7 @@ def __init__(self, expr_ev): def Call(self, rd): # type: (typed_args.Reader) -> value_t + unused_self = rd.PosObj() lazy = rd.PosExpr() rd.Done() diff --git a/core/shell.py b/core/shell.py index 36d1ca1c52..86daaa7e28 100644 --- a/core/shell.py +++ b/core/shell.py @@ -572,6 +572,8 @@ def Main( method_io.Eval(mem, cmd_ev, method_io.EVAL_DICT)) io_methods['M/evalInFrame'] = value.BuiltinFunc( method_io.EvalInFrame(mem, cmd_ev)) + io_methods['M/evalExpr'] = value.BuiltinFunc( + func_reflect.EvalExpr(expr_ev)) # Identical to command sub io_methods['captureStdout'] = value.BuiltinFunc( @@ -897,7 +899,6 @@ def Main( func_reflect.ParseCommand(parse_ctx, mem, errfmt)) _AddBuiltinFunc(mem, 'parseExpr', func_reflect.ParseExpr(parse_ctx, errfmt)) - _AddBuiltinFunc(mem, 'evalExpr', func_reflect.EvalExpr(expr_ev)) _AddBuiltinFunc(mem, 'shvarGet', func_reflect.Shvar_get(mem)) _AddBuiltinFunc(mem, 'getVar', func_reflect.GetVar(mem)) diff --git a/doc/ref/chap-builtin-func.md b/doc/ref/chap-builtin-func.md index 7a35e611ec..2dd87b0b9e 100644 --- a/doc/ref/chap-builtin-func.md +++ b/doc/ref/chap-builtin-func.md @@ -476,17 +476,6 @@ Given a code string, parse it as an expression. Returns a `value.Expr` instance, or raises an error. -### `evalExpr()` - -Given a an expression quotation, evaluate it and return its value: - - $ var expr = ^[1 + 2] - - $ = evalExpr(expr) - 3 - - - ## Hay Config ### parseHay() diff --git a/doc/ref/chap-type-method.md b/doc/ref/chap-type-method.md index 272c760f36..8147aac535 100644 --- a/doc/ref/chap-type-method.md +++ b/doc/ref/chap-type-method.md @@ -614,6 +614,21 @@ with `try`. var s = _io->captureStdout(c) } +### evalExpr() + +Given an `Expr` value, evaluate it and return its value: + + $ var i = 42 + $ var expr = ^[i + 1] + + $ = io->evalExpr(expr) + 43 + +Examples of expressions that have effects: + +- `^[ myplace->setValue(42) ]` - memory operation +- `^[ $(echo 42 > hi) ]` - I/O operation + ### promptVal() An API the wraps the `$PS1` language. For example, to simulate `PS1='\w\$ '`: diff --git a/doc/ref/toc-ysh.md b/doc/ref/toc-ysh.md index 310a96fc02..2ed476eb92 100644 --- a/doc/ref/toc-ysh.md +++ b/doc/ref/toc-ysh.md @@ -62,6 +62,7 @@ X [Func] name() location() toJson() X [Proc] name() location() toJson() [IO] stdin eval() evalToDict() captureStdout() + evalExpr() promptVal() X time() X strftime() X glob() [Obj] __invoke__ X __call__ X __str__ @@ -91,8 +92,7 @@ X [J8 Decode] J8.Bool() J8.Int() ... [Pattern] _group() _start() _end() [Introspection] id() shvarGet() getVar() setVar() - parseCommand() X parseExpr() evalExpr() - X bindFrame() + parseCommand() X parseExpr() X bindFrame() [Hay Config] parseHay() evalHay() X [Hashing] sha1dc() sha256() ``` diff --git a/spec/ysh-closures.test.sh b/spec/ysh-closures.test.sh index cfb8768091..3b992bb882 100644 --- a/spec/ysh-closures.test.sh +++ b/spec/ysh-closures.test.sh @@ -22,7 +22,7 @@ var exprs = makeTasks() #= blocks for ex in (exprs) { - var s = evalExpr(ex) + var s = io->evalExpr(ex) echo $s } diff --git a/spec/ysh-expr.test.sh b/spec/ysh-expr.test.sh index 324ab8c7e0..9294a199d9 100644 --- a/spec/ysh-expr.test.sh +++ b/spec/ysh-expr.test.sh @@ -639,17 +639,17 @@ echo $x var e = ^[1 + 2] echo type=$[type(e)] -echo $[evalExpr(e)] +echo $[io->evalExpr(e)] var e = ^[2 < 1] -echo $[evalExpr(e)] +echo $[io->evalExpr(e)] var x = 42 var e = ^[42 === x and true] -echo $[evalExpr(e)] +echo $[io->evalExpr(e)] var mylist = ^[3, 4] -pp test_ (evalExpr(mylist)) +pp test_ (io->evalExpr(mylist)) ## STDOUT: type=Expr @@ -662,7 +662,7 @@ true #### No list comprehension in ^[] var mylist = ^[x for x in y] -pp test_ (evalExpr(mylist)) +pp test_ (io->evalExpr(mylist)) ## status: 2 ## STDOUT: @@ -671,7 +671,7 @@ pp test_ (evalExpr(mylist)) #### expression literals, evaluation failure var e = ^[1 / 0] -call evalExpr(e) +call io->evalExpr(e) ## status: 3 ## STDOUT: ## END @@ -681,7 +681,7 @@ var x = 0 var e = ^[x] setvar x = 1 -echo result=$[evalExpr(e)] +echo result=$[io->evalExpr(e)] ## STDOUT: result=1 ## END @@ -691,7 +691,7 @@ var x = 0 var e = ^"x is $x" setvar x = 1 -echo result=$[evalExpr(e)] +echo result=$[io->evalExpr(e)] ## STDOUT: result=x is 1 ## END From c3a5633be0649c8f104558fd3d736a3a048461fa Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 14:37:26 -0400 Subject: [PATCH 372/506] [ysh semantics] value.Expr closures in a loop works Str.replace() still needs some work. But I removed the mechanism of putting $0 in the var_stack, and then looking it up there. Instead, we mutate mem.dollar0. This is consistent with the way we handle other vars, and fixes some quirks. --- Though in the future, it may be nice to deprecate the mem.argv_stack and mem.dollar0, and put ALL of $0 $1 $2 in the var_stack. Right now that is a little tricky, and it's an orthogonal change. --- builtin/func_reflect.py | 2 +- builtin/method_str.py | 2 +- core/state.py | 30 ++++++++++++++++++++++-------- frontend/typed_args.py | 9 ++++----- spec/ysh-closures.test.sh | 7 +++++-- spec/ysh-regex-api.test.sh | 23 ++++++++++++++++------- ysh/expr_eval.py | 12 ++++++++++++ 7 files changed, 61 insertions(+), 24 deletions(-) diff --git a/builtin/func_reflect.py b/builtin/func_reflect.py index 92fa5a3cd6..47cfb20f46 100644 --- a/builtin/func_reflect.py +++ b/builtin/func_reflect.py @@ -219,6 +219,6 @@ def Call(self, rd): lazy = rd.PosExpr() rd.Done() - result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken()) + result = self.expr_ev.EvalExprClosure(lazy, rd.LeftParenToken()) return result diff --git a/builtin/method_str.py b/builtin/method_str.py index 9ac2a779ac..489a16cdee 100644 --- a/builtin/method_str.py +++ b/builtin/method_str.py @@ -328,7 +328,7 @@ def __init__(self, mem, expr_ev): def EvalSubstExpr(self, expr, blame_loc): # type: (value.Expr, loc_t) -> str - res = self.expr_ev.EvalExpr(expr.e, blame_loc) + res = self.expr_ev.EvalExprClosure(expr, blame_loc) if res.tag() == value_e.Str: return cast(value.Str, res).s diff --git a/core/state.py b/core/state.py index 7dc942301d..4a122d3df5 100644 --- a/core/state.py +++ b/core/state.py @@ -758,6 +758,10 @@ def Dump(self): def GetArgNum(self, arg_num): # type: (int) -> value_t + + # $0 is handled elsewhere + assert 1 <= arg_num, arg_num + index = self.num_shifted + arg_num - 1 if index >= len(self.argv): return value.Undef @@ -1332,9 +1336,12 @@ def __init__( # $0 needs to have lexical scoping. So we store it with other locals. # As "0" cannot be parsed as an lvalue, we can safely store dollar0 there. if dollar0 is not None: - assert mem.GetValue("0", scope_e.LocalOnly).tag() == value_e.Undef - self.dollar0_lval = LeftName("0", loc.Missing) - mem.SetLocalName(self.dollar0_lval, value.Str(dollar0)) + #assert mem.GetValue("0", scope_e.LocalOnly).tag() == value_e.Undef + #self.dollar0_lval = LeftName("0", loc.Missing) + #mem.SetLocalName(self.dollar0_lval, value.Str(dollar0)) + + self.restore_dollar0 = self.mem.dollar0 + self.mem.dollar0 = dollar0 if pos_args is not None: mem.argv_stack.append(_ArgFrame(pos_args)) @@ -1356,7 +1363,8 @@ def __exit__(self, type, value_, traceback): self.mem.argv_stack.pop() if self.dollar0 is not None: - self.mem.SetLocalName(self.dollar0_lval, value.Undef) + #self.mem.SetLocalName(self.dollar0_lval, value.Undef) + self.mem.dollar0 = self.restore_dollar0 # Note: _Push and _Pop are separate methods because the C++ translation # doesn't like when they are inline in __init__ and __exit__. @@ -1777,10 +1785,16 @@ def GetArg0(self): def GetArgNum(self, arg_num): # type: (int) -> value_t if arg_num == 0: - # $0 may be overriden, eg. by Str => replace() - vars = self.var_stack[-1] - if "0" in vars and vars["0"].val.tag() != value_e.Undef: - return vars["0"].val + # Disabled + if 0: + # Problem: Doesn't obey enclosing frame? + # Yeah it needs FrameLookup + cell, _ = _FrameLookup(self.var_stack[-1], '0') + if cell is not None: + val = cell.val + if val.tag() != value_e.Undef: + return val + return value.Str(self.dollar0) return self.argv_stack[-1].GetArgNum(arg_num) diff --git a/frontend/typed_args.py b/frontend/typed_args.py index 4ef3ee65c7..e5fd0405d8 100644 --- a/frontend/typed_args.py +++ b/frontend/typed_args.py @@ -2,8 +2,7 @@ from __future__ import print_function from _devbuild.gen.runtime_asdl import cmd_value, ProcArgs, Cell -from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, command_t, expr_t, - Token) +from _devbuild.gen.syntax_asdl import (loc, loc_t, ArgList, command_t, Token) from _devbuild.gen.value_asdl import (value, value_e, value_t, RegexMatch, Obj, cmd_frag, cmd_frag_e, cmd_frag_str, LiteralBlock) @@ -353,9 +352,9 @@ def _ToEggex(self, val): self.BlamePos()) def _ToExpr(self, val): - # type: (value_t) -> expr_t + # type: (value_t) -> value.Expr if val.tag() == value_e.Expr: - return cast(value.Expr, val).e + return cast(value.Expr, val) raise error.TypeErr(val, 'Arg %d should be a Expr' % self.pos_consumed, self.BlamePos()) @@ -503,7 +502,7 @@ def PosCommand(self): return self._ToCommand(val) def PosExpr(self): - # type: () -> expr_t + # type: () -> value.Expr val = self.PosValue() return self._ToExpr(val) diff --git a/spec/ysh-closures.test.sh b/spec/ysh-closures.test.sh index 3b992bb882..04adb4da1f 100644 --- a/spec/ysh-closures.test.sh +++ b/spec/ysh-closures.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 1 +## oils_failures_allowed: 0 #### Expr Closures in a Loop ! shopt --set ysh:upgrade @@ -13,7 +13,7 @@ func makeTasks() { for __hack__ in (0 .. 3) { var i = __hack__ var j = i + 2 - task (tasks, ^["$x: i = $i, j = $j"]) + task (tasks, ^"$x: i = $i, j = $j") } return (tasks) } @@ -27,6 +27,9 @@ for ex in (exprs) { } ## STDOUT: +x: i = 0, j = 2 +x: i = 1, j = 3 +x: i = 2, j = 4 ## END #### Block Closures in a Loop ! diff --git a/spec/ysh-regex-api.test.sh b/spec/ysh-regex-api.test.sh index 48e847f7d2..9ffd9950f2 100644 --- a/spec/ysh-regex-api.test.sh +++ b/spec/ysh-regex-api.test.sh @@ -708,16 +708,25 @@ write $[mystr.replace(/ 'Foo' /, ^"$0Bar")] # Edge-cases var dollar0 = "$0" -func f() { return ("$0") } -write $["foo".replace("o", "$0") === "f$dollar0$dollar0"] -write $["foo".replace("o", ^[f()]) === "f$dollar0$dollar0"] -write $[f() === "$dollar0"] +#echo dollar0=$dollar0 +#echo "0 = $0" + +var expected = "f($dollar0)($dollar0)" +#echo "expected = $expected" + +# Eager replacement +assert [expected === "foo".replace("o", "($0)")] + +assert ['f(o)(o)' === "foo".replace("o", ^"($0)")] + +func f() { return ( "<$0>" ) } +assert ["<$dollar0>" === f()] + +assert ['f' === "foo".replace("o", ^[f()])] + ## STDOUT: class FooBar: # this class is called FooBar class FooBar: # this class is called FooBar -true -true -true ## END #### Str.replace(Eggex, Expr), scopes diff --git a/ysh/expr_eval.py b/ysh/expr_eval.py index 845bb02d4a..bf4f4c9219 100644 --- a/ysh/expr_eval.py +++ b/ysh/expr_eval.py @@ -369,6 +369,18 @@ def _EvalLhsExpr(self, lhs, which_scopes): else: raise AssertionError() + def EvalExprClosure(self, expr_val, blame_loc): + # type: (value.Expr, loc_t) -> value_t + """ + Used by user-facing APIs that take value.Expr closures: + + var i = 42 + var x = io->evalExpr(^[i + 1]) + var x = s.replace(pat, ^"- $0 $i -") + """ + with state.ctx_EnclosedFrame(self.mem, expr_val.captured_frame, None): + return self.EvalExpr(expr_val.e, blame_loc) + def EvalExpr(self, node, blame_loc): # type: (expr_t, loc_t) -> value_t """Public API for _EvalExpr to ensure command_sub_errexit""" From 71bcde368a1780968dd6d60453f0bd1946fff07a Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 15:47:02 -0400 Subject: [PATCH 373/506] [spec/ysh-closures] Simple test cases Remove unused EvalCommandClosure() function All the builtins use EvalCommandFrag(). Though io.captureStdout() should probably use EvalCommandClosure(), because we don't need variables to persist. --- osh/cmd_eval.py | 11 ++++++----- spec/ysh-closures.test.sh | 40 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/osh/cmd_eval.py b/osh/cmd_eval.py index 3b3feb8d63..a23d21aa6c 100644 --- a/osh/cmd_eval.py +++ b/osh/cmd_eval.py @@ -2117,11 +2117,12 @@ def EvalCommandFrag(self, frag): """ return self._Execute(frag) # can raise FatalRuntimeError, etc. - def EvalCommand(self, cmd): - # type: (value.Command) -> int - frag = typed_args.GetCommandFrag(cmd) - with state.ctx_EnclosedFrame(self.mem, cmd.captured_frame, None): - return self.EvalCommandFrag(frag) + if 0: + def EvalCommandClosure(self, cmd): + # type: (value.Command) -> int + frag = typed_args.GetCommandFrag(cmd) + with state.ctx_EnclosedFrame(self.mem, cmd.captured_frame, None): + return self.EvalCommandFrag(frag) def RunTrapsOnExit(self, mut_status): # type: (IntParamBox) -> None diff --git a/spec/ysh-closures.test.sh b/spec/ysh-closures.test.sh index 04adb4da1f..2ddb2f751f 100644 --- a/spec/ysh-closures.test.sh +++ b/spec/ysh-closures.test.sh @@ -1,5 +1,45 @@ ## oils_failures_allowed: 0 +#### Simple Expr Closure +shopt --set ysh:upgrade + +proc my-expr (; expr) { + echo $[io->evalExpr(expr)] +} + +proc p { + var i = 42 + my-expr [i + 1] +} + +p + +## STDOUT: +43 +## END + +#### Simple Block Closure +shopt --set ysh:upgrade + +shopt --set ysh:upgrade + +proc my-expr (; ; ; block) { + call io->eval(block) +} + +proc p { + var i = 42 + my-expr { + echo $[i + 1] + } +} + +p + +## STDOUT: +43 +## END + #### Expr Closures in a Loop ! shopt --set ysh:upgrade From 32476ddc152dd6821e42b889ccb9bf5889687b81 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 15:58:14 -0400 Subject: [PATCH 374/506] [spec/ysh-regex-api] Fix test cases for Str.replace value.Expr as a closure works! --- spec/ysh-method-io.test.sh | 13 +++++++++---- spec/ysh-regex-api.test.sh | 16 +++++++++++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/spec/ysh-method-io.test.sh b/spec/ysh-method-io.test.sh index 75fe81fc7d..0df79d64aa 100644 --- a/spec/ysh-method-io.test.sh +++ b/spec/ysh-method-io.test.sh @@ -3,13 +3,18 @@ #### captureStdout() is like $() -var c = ^(echo one; echo two) +proc p { + var captured = 'captured' + var cmd = ^(echo one; echo $captured) + + var stdout = io.captureStdout(cmd) + pp test_ (stdout) +} -var y = io.captureStdout(c) -pp test_ (y) +p ## STDOUT: -(Str) "one\ntwo" +(Str) "one\ncaptured" ## END #### captureStdout() failure diff --git a/spec/ysh-regex-api.test.sh b/spec/ysh-regex-api.test.sh index 9ffd9950f2..25a0c81d7f 100644 --- a/spec/ysh-regex-api.test.sh +++ b/spec/ysh-regex-api.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 2 +## oils_failures_allowed: 0 #### s ~ regex and s !~ regex shopt -s ysh:upgrade @@ -885,11 +885,11 @@ shopt --set ysh:upgrade var s = 'mystr' var pat = / 's' / -var template = ^"[$x $0 $1 $x]" -pp test_ (template) proc p { var x = 'x' + var template = ^"[$x $0 $1 $x]" + pp test_ (template) var new = s.replace(pat, template) echo 'replace ' $new @@ -904,6 +904,9 @@ proc p { p ## STDOUT: + +replace my[x st t x]r +myreplace my[x st t x]r ## END #### Str.replace() lexical scope with ^[] @@ -911,11 +914,11 @@ shopt --set ysh:upgrade var s = 'mystr' var pat = / 's' / -var template = ^['[' ++ x ++ ' ' ++ $0 ++ ' ' ++ $1 ++ ' ' ++ x ++ ']'] -pp test_ (template) proc p { var x = 'x' + var template = ^['[' ++ x ++ ' ' ++ $0 ++ ' ' ++ $1 ++ ' ' ++ x ++ ']'] + pp test_ (template) var new = s.replace(pat, template) echo 'replace ' $new @@ -930,4 +933,7 @@ proc p { p ## STDOUT: + +replace my[x st t x]r +myreplace my[x st t x]r ## END From 84136bb43dfa506e396dc2774787d9c8e45c64ca Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 19:02:42 -0400 Subject: [PATCH 375/506] [ysh] Move YSH constants from main module Frame -> __builtins__ module We'll also move env vars out of the main module frame. --- core/shell.py | 3 ++- core/state.py | 18 ++++++++---------- spec/ysh-namespaces.test.sh | 5 +++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/shell.py b/core/shell.py index 86daaa7e28..912ad99b6f 100644 --- a/core/shell.py +++ b/core/shell.py @@ -359,7 +359,8 @@ def Main( attrs.shopt_changes) version_str = pyutil.GetVersion(loader) - state.InitMem(mem, environ, version_str) + state.InitBuiltins(mem, environ, version_str) + state.InitDefaultVars(mem) # TODO: consider turning on no_copy_env in YSH if exec_opts.no_copy_env(): diff --git a/core/state.py b/core/state.py index 4a122d3df5..562fa9cab4 100644 --- a/core/state.py +++ b/core/state.py @@ -850,7 +850,7 @@ def _AddCallToken(d, token): d['call_line'] = value.Str(token.line.content) -def _InitDefaults(mem): +def InitDefaultVars(mem): # type: (Mem) -> None # Default value; user may unset it. @@ -931,20 +931,20 @@ def InitVarsFromEnv(mem, environ): SetGlobalString(mem, 'PATH', '/bin:/usr/bin') -def InitMem(mem, environ, version_str): +def InitBuiltins(mem, environ, version_str): # type: (Mem, Dict[str, str], str) -> None """Initialize memory with shell defaults. Other interpreters could have different builtin variables. """ # TODO: REMOVE this legacy. ble.sh checks it! - SetGlobalString(mem, 'OIL_VERSION', version_str) + mem.builtins['OIL_VERSION'] = value.Str(version_str) - SetGlobalString(mem, 'OILS_VERSION', version_str) + mem.builtins['OILS_VERSION'] = value.Str(version_str) # The source builtin understands '///' to mean "relative to embedded stdlib" - SetGlobalString(mem, 'LIB_OSH', '///osh') - SetGlobalString(mem, 'LIB_YSH', '///ysh') + mem.builtins['LIB_OSH'] = value.Str('///osh') + mem.builtins['LIB_YSH'] = value.Str('///ysh') # - C spells it NAN # - JavaScript spells it NaN @@ -953,10 +953,8 @@ def InitMem(mem, environ, version_str): # - libc prints the strings 'nan' and 'inf' # - Python 3 prints the strings 'nan' and 'inf' # - JavaScript prints 'NaN' and 'Infinity', which is more stylized - SetGlobalValue(mem, 'NAN', value.Float(pyutil.nan())) - SetGlobalValue(mem, 'INFINITY', value.Float(pyutil.infinity())) - - _InitDefaults(mem) + mem.builtins['NAN'] = value.Float(pyutil.nan()) + mem.builtins['INFINITY'] = value.Float(pyutil.infinity()) def InitInteractive(mem, lang): diff --git a/spec/ysh-namespaces.test.sh b/spec/ysh-namespaces.test.sh index 9a9e6249f3..efde8a2226 100644 --- a/spec/ysh-namespaces.test.sh +++ b/spec/ysh-namespaces.test.sh @@ -26,6 +26,8 @@ pp test_ (_pipeline_status) #### global frame doesn't contain env vars +#pp frame_vars_ + try { pp frame_vars_ | grep -o TMP } @@ -36,8 +38,6 @@ pp test_ (_pipeline_status) (List) [0,1] ## END - - #### __builtins__ module var b = len(propView(__builtins__)) @@ -53,3 +53,4 @@ assert [2 === len] ## STDOUT: ## END + From 04c50063b2ce9c56ad385b72c2819df868e51739 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 19:25:10 -0400 Subject: [PATCH 376/506] [test/unit] Fix build [test/spec] Add test for initialized PATH PWD PS4 SHELLOPTS --- core/completion_test.py | 2 +- core/process_test.py | 3 ++- core/shell.py | 7 +++++ core/test_lib.py | 8 +++--- osh/arith_parse_test.py | 3 ++- spec/vars-special.test.sh | 55 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 71 insertions(+), 7 deletions(-) diff --git a/core/completion_test.py b/core/completion_test.py index 95a6cf28ae..74c649be31 100755 --- a/core/completion_test.py +++ b/core/completion_test.py @@ -57,7 +57,7 @@ def _MakeRootCompleter(parse_ctx=None, comp_lookup=None): parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None) mem.exec_opts = exec_opts - state.InitMem(mem, {}, '0.1') + state.InitDefaultVars(mem) mutable_opts.Init() if not parse_ctx: diff --git a/core/process_test.py b/core/process_test.py index 2ce7a31a00..c51c8c73ab 100755 --- a/core/process_test.py +++ b/core/process_test.py @@ -55,7 +55,8 @@ def setUp(self): parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None) mem.exec_opts = exec_opts - state.InitMem(mem, {}, '0.1') + #state.InitMem(mem, {}, '0.1') + state.InitDefaultVars(mem) self.job_control = process.JobControl() self.job_list = process.JobList() diff --git a/core/shell.py b/core/shell.py index 912ad99b6f..38b8034635 100644 --- a/core/shell.py +++ b/core/shell.py @@ -363,7 +363,14 @@ def Main( state.InitDefaultVars(mem) # TODO: consider turning on no_copy_env in YSH + # + # But we also need a way for $PATH to be set, because + # - PATH, PWD, SHELLOPTS could be special cases + # - and then we need to copy them into new modules, like PS4? + # - they are also exported? + if exec_opts.no_copy_env(): + #if 1: # Don't consult the environment mem.SetPwd(state.GetWorkingDir()) else: diff --git a/core/test_lib.py b/core/test_lib.py index ffebf5cbdb..e7bb5cf445 100644 --- a/core/test_lib.py +++ b/core/test_lib.py @@ -169,7 +169,7 @@ def InitWordEvaluator(exec_opts=None): if exec_opts is None: parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None) mem.exec_opts = exec_opts # circular dep - state.InitMem(mem, {}, '0.1') + state.InitDefaultVars(mem) mutable_opts.Init() else: mutable_opts = None @@ -204,7 +204,8 @@ def InitCommandEvaluator(parse_ctx=None, exec_opts = optview.Exec(opt0_array, opt_stacks) mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None) mem.exec_opts = exec_opts - state.InitMem(mem, {}, '0.1') + #state.InitMem(mem, {}, '0.1') + state.InitDefaultVars(mem) mutable_opts.Init() # No 'readline' in the tests. @@ -323,7 +324,8 @@ def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None): parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None) mem.exec_opts = exec_opts - state.InitMem(mem, {}, '0.1') + #state.InitMem(mem, {}, '0.1') + state.InitDefaultVars(mem) mutable_opts.Init() line_reader, _ = InitLexer(code_str, arena) diff --git a/osh/arith_parse_test.py b/osh/arith_parse_test.py index de39080783..6ca4e0f0a2 100755 --- a/osh/arith_parse_test.py +++ b/osh/arith_parse_test.py @@ -37,7 +37,8 @@ def ParseAndEval(code_str): mem = state.Mem('', [], arena, []) parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None) mem.exec_opts = exec_opts - state.InitMem(mem, {}, '0.1') + #state.InitMem(mem, {}, '0.1') + state.InitDefaultVars(mem) splitter = split.SplitContext(mem) errfmt = ui.ErrorFormatter() diff --git a/spec/vars-special.test.sh b/spec/vars-special.test.sh index 2f0b76a9ec..39303ce6ef 100644 --- a/spec/vars-special.test.sh +++ b/spec/vars-special.test.sh @@ -66,7 +66,6 @@ yes yes ## END - #### $HOME is NOT set case $SH in *zsh) echo 'zsh sets HOME'; exit ;; esac @@ -90,6 +89,59 @@ status=1 zsh sets HOME ## END +#### Some vars are set, even without startup file, or env: PATH, PWD + +flags='' +case $SH in + dash) exit ;; + bash*) + flags='--noprofile --norc --rcfile /devnull' + ;; + osh) + flags='--rcfile /devnull' + ;; +esac + +sh_path=$(which $SH) + +case $sh_path in + */bin/osh) + # Hack for running with Python2 + export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/vendor" + sh_prefix="$(which python2) $REPO_ROOT/bin/oils_for_unix.py osh" + ;; + *) + sh_prefix=$sh_path + ;; +esac + +#echo PATH=$PATH + + +# mksh has typeset, not declare +# bash exports PWD, but not PATH PS4 + +/usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p PATH PWD PS4' >&2 +echo status=$? + +/usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p SHELLOPTS' >&2 +echo status=$? + +# hm bash doesn't set $HOME + +## STDOUT: +status=0 +status=0 +## END + +## OK zsh STDOUT: +status=0 +status=1 +## END + +## N-I dash STDOUT: +## END + #### $1 .. $9 are scoped, while $0 is not fun() { @@ -643,3 +695,4 @@ seconds=0 ## N-I dash STDOUT: seconds= ## END + From 60762ae16ac5a45e6fbbe5a46ea7849bc6e8be42 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 19:33:52 -0400 Subject: [PATCH 377/506] [refactor] Simplify shopt --set no_copy_env logic Some initialization happens before environment variables, and some after. --- core/shell.py | 26 +++++--------------------- core/state.py | 27 ++++++++++++++++----------- osh/arith_parse_gen.py | 1 - spec/vars-special.test.sh | 31 ++++++++++++++++++++++++------- 4 files changed, 45 insertions(+), 40 deletions(-) diff --git a/core/shell.py b/core/shell.py index 38b8034635..9c36b49127 100644 --- a/core/shell.py +++ b/core/shell.py @@ -362,27 +362,11 @@ def Main( state.InitBuiltins(mem, environ, version_str) state.InitDefaultVars(mem) - # TODO: consider turning on no_copy_env in YSH - # - # But we also need a way for $PATH to be set, because - # - PATH, PWD, SHELLOPTS could be special cases - # - and then we need to copy them into new modules, like PS4? - # - they are also exported? - - if exec_opts.no_copy_env(): - #if 1: - # Don't consult the environment - mem.SetPwd(state.GetWorkingDir()) - else: - state.InitVarsFromEnv(mem, environ) - - # MUTABLE GLOBAL that's SEPARATE from $PWD. Used by the 'pwd' builtin, but - # it can't be modified by users. - val = mem.GetValue('PWD') - # should be true since it's exported - assert val.tag() == value_e.Str, val - pwd = cast(value.Str, val).s - mem.SetPwd(pwd) + if not exec_opts.no_copy_env(): + state.CopyVarsFromEnv(mem, environ) + + # PATH PWD SHELLOPTS, etc. must be set after CopyVarsFromEnv() + state.InitVarsAfterEnv(mem) if attrs.show_options: # special case: sh -o mutable_opts.ShowOptions([]) diff --git a/core/state.py b/core/state.py index 562fa9cab4..165c135b23 100644 --- a/core/state.py +++ b/core/state.py @@ -885,7 +885,7 @@ def InitDefaultVars(mem): # set_home_var (); -def InitVarsFromEnv(mem, environ): +def CopyVarsFromEnv(mem, environ): # type: (Mem, Dict[str, str]) -> None # This is the way dash and bash work -- at startup, they turn everything in @@ -897,11 +897,11 @@ def InitVarsFromEnv(mem, environ): scope_e.GlobalOnly, flags=SetExport) - # If it's not in the environment, initialize it. This makes it easier to - # update later in MutableOpts. - # TODO: IFS, etc. should follow this pattern. Maybe need a SysCall - # interface? self.syscall.getcwd() etc. +def InitVarsAfterEnv(mem): + # type: (Mem) -> None + + # If SHELLOPTS PWD PATH are not in environ, then initialize them. val = mem.GetValue('SHELLOPTS') if val.tag() == value_e.Undef: @@ -912,22 +912,27 @@ def InitVarsFromEnv(mem, environ): scope_e.GlobalOnly, flags=SetReadOnly) - # Usually we inherit PWD from the parent shell. When it's not set, we may - # compute it. val = mem.GetValue('PWD') if val.tag() == value_e.Undef: SetGlobalString(mem, 'PWD', GetWorkingDir()) - # Now mark it exported, no matter what. This is one of few variables - # EXPORTED. bash and dash both do it. (e.g. env -i -- dash -c env) + # Mark it exported, no matter what. This is one of few variables EXPORTED. + # bash and dash both do it. (e.g. env -i -- dash -c env) mem.SetNamed(location.LName('PWD'), None, scope_e.GlobalOnly, flags=SetExport) + # MUTABLE GLOBAL that's SEPARATE from $PWD. Used by the 'pwd' builtin, but + # it can't be modified by users. + val = mem.GetValue('PWD') + assert val.tag() == value_e.Str, val + pwd = cast(value.Str, val).s + mem.SetPwd(pwd) + val = mem.GetValue('PATH') if val.tag() == value_e.Undef: - # Setting PATH to these two dirs match what zsh and mksh do. bash and dash - # add {,/usr/,/usr/local}/{bin,sbin} + # Setting PATH to these two dirs match what zsh and mksh do. bash and + # dash add {,/usr/,/usr/local}/{bin,sbin} SetGlobalString(mem, 'PATH', '/bin:/usr/bin') diff --git a/osh/arith_parse_gen.py b/osh/arith_parse_gen.py index debfadb2d2..0b18a22771 100755 --- a/osh/arith_parse_gen.py +++ b/osh/arith_parse_gen.py @@ -1,5 +1,4 @@ #!/usr/bin/env python2 -"""Arith_parse_gen.py.""" from __future__ import print_function import collections diff --git a/spec/vars-special.test.sh b/spec/vars-special.test.sh index 39303ce6ef..4885bdd99c 100644 --- a/spec/vars-special.test.sh +++ b/spec/vars-special.test.sh @@ -122,21 +122,38 @@ esac # bash exports PWD, but not PATH PS4 /usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p PATH PWD PS4' >&2 -echo status=$? +echo path pwd ps4 $? /usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p SHELLOPTS' >&2 -echo status=$? +echo shellopts $? + +# bash doesn't set HOME, mksh and zsh do +/usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p HOME' >&2 +echo home $? -# hm bash doesn't set $HOME +# bash doesn't set PS1, mksh and zsh do +/usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p PS1' >&2 +echo ps1 $? ## STDOUT: -status=0 -status=0 +path pwd ps4 0 +shellopts 0 +home 1 +ps1 1 +## END + +## OK mksh STDOUT: +path pwd ps4 0 +shellopts 0 +home 0 +ps1 0 ## END ## OK zsh STDOUT: -status=0 -status=1 +path pwd ps4 0 +shellopts 1 +home 0 +ps1 0 ## END ## N-I dash STDOUT: From 7fa54ce1ab481f861771ddf72cf0f37835d26624 Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 19:59:04 -0400 Subject: [PATCH 378/506] [spec/vars-special] Tests for read-only and overridable vars UID EUID PPID can't be overriden, and OPTIND is not meant to be either. Re-organize initialization too. --- core/state.py | 29 ++++++++------- spec/vars-special.test.sh | 75 +++++++++++++++++++++++++++++++++------ 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/core/state.py b/core/state.py index 165c135b23..4eae86a25b 100644 --- a/core/state.py +++ b/core/state.py @@ -853,16 +853,22 @@ def _AddCallToken(d, token): def InitDefaultVars(mem): # type: (Mem) -> None + # These 3 are special, can't be changed + SetGlobalString(mem, 'UID', str(posix.getuid())) + SetGlobalString(mem, 'EUID', str(posix.geteuid())) + SetGlobalString(mem, 'PPID', str(posix.getppid())) + + # For getopts builtin - meant to be read, not changed + SetGlobalString(mem, 'OPTIND', '1') + + # These can be changed. Could go AFTER environment, e.g. in + # InitVarsAfterEnv(). + # Default value; user may unset it. # $ echo -n "$IFS" | python -c 'import sys;print repr(sys.stdin.read())' # ' \t\n' SetGlobalString(mem, 'IFS', split.DEFAULT_IFS) - # NOTE: Should we put these in a var_frame for Oil? - SetGlobalString(mem, 'UID', str(posix.getuid())) - SetGlobalString(mem, 'EUID', str(posix.geteuid())) - SetGlobalString(mem, 'PPID', str(posix.getppid())) - SetGlobalString(mem, 'HOSTNAME', libc.gethostname()) # In bash, this looks like 'linux-gnu', 'linux-musl', etc. Scripts test @@ -870,9 +876,6 @@ def InitDefaultVars(mem): # 'musl'. We don't have that info, so just make it 'linux'. SetGlobalString(mem, 'OSTYPE', pyos.OsType()) - # For getopts builtin - SetGlobalString(mem, 'OPTIND', '1') - # When xtrace_rich is off, this is just like '+ ', the shell default SetGlobalString(mem, 'PS4', '${SHX_indent}${SHX_punct}${SHX_pid_str} ') @@ -906,7 +909,7 @@ def InitVarsAfterEnv(mem): val = mem.GetValue('SHELLOPTS') if val.tag() == value_e.Undef: SetGlobalString(mem, 'SHELLOPTS', '') - # Now make it readonly + # It's readonly, even if it's not set mem.SetNamed(location.LName('SHELLOPTS'), None, scope_e.GlobalOnly, @@ -915,15 +918,15 @@ def InitVarsAfterEnv(mem): val = mem.GetValue('PWD') if val.tag() == value_e.Undef: SetGlobalString(mem, 'PWD', GetWorkingDir()) - # Mark it exported, no matter what. This is one of few variables EXPORTED. - # bash and dash both do it. (e.g. env -i -- dash -c env) + # It's exported, even if it's not set. bash and dash both do this: + # env -i -- dash -c env mem.SetNamed(location.LName('PWD'), None, scope_e.GlobalOnly, flags=SetExport) - # MUTABLE GLOBAL that's SEPARATE from $PWD. Used by the 'pwd' builtin, but - # it can't be modified by users. + # Set a MUTABLE GLOBAL that's SEPARATE from $PWD. It's used by the 'pwd' + # builtin, and it can't be modified by users. val = mem.GetValue('PWD') assert val.tag() == value_e.Str, val pwd = cast(value.Str, val).s diff --git a/spec/vars-special.test.sh b/spec/vars-special.test.sh index 4885bdd99c..2afd05494f 100644 --- a/spec/vars-special.test.sh +++ b/spec/vars-special.test.sh @@ -1,4 +1,4 @@ -## oils_failures_allowed: 2 +## oils_failures_allowed: 3 ## compare_shells: dash bash-4.4 mksh zsh @@ -128,37 +128,90 @@ echo path pwd ps4 $? echo shellopts $? # bash doesn't set HOME, mksh and zsh do -/usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p HOME' >&2 -echo home $? +/usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p HOME PS4' >&2 +echo home ps1 $? # bash doesn't set PS1, mksh and zsh do -/usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p PS1' >&2 -echo ps1 $? +/usr/bin/env -i PYTHONPATH=$PYTHONPATH $sh_prefix $flags -c 'typeset -p IFS' >&2 +echo ifs $? ## STDOUT: path pwd ps4 0 shellopts 0 -home 1 -ps1 1 +home ps1 1 +ifs 0 ## END ## OK mksh STDOUT: path pwd ps4 0 shellopts 0 -home 0 -ps1 0 +home ps1 0 +ifs 0 ## END ## OK zsh STDOUT: path pwd ps4 0 shellopts 1 -home 0 -ps1 0 +home ps1 0 +ifs 0 ## END ## N-I dash STDOUT: ## END +#### UID EUID PPID can't be changed + +# bash makes these 3 read-only +{ + UID=xx $SH -c 'echo uid=$UID' + + EUID=xx $SH -c 'echo euid=$EUID' + + PPID=xx $SH -c 'echo ppid=$PPID' + +} > out.txt + +# bash shows that vars are readonly +# zsh shows other errors +# cat out.txt +#echo + +grep '=xx' out.txt +echo status=$? + +## STDOUT: +status=1 +## END +## BUG dash/mksh STDOUT: +uid=xx +euid=xx +status=0 +## END + +#### HOSTNAME OSTYPE can be changed +case $SH in zsh) exit ;; esac + +#$SH -c 'echo hostname=$HOSTNAME' + +HOSTNAME=x $SH -c 'echo hostname=$HOSTNAME' +OSTYPE=x $SH -c 'echo ostype=$OSTYPE' +echo + +#PS4=x $SH -c 'echo ps4=$PS4' + +# OPTIND is special +#OPTIND=xx $SH -c 'echo optind=$OPTIND' + + +## STDOUT: +hostname=x +ostype=x + +## END + +## BUG zsh STDOUT: +## END + #### $1 .. $9 are scoped, while $0 is not fun() { From 28bdba578f174863473208d687183923358b112d Mon Sep 17 00:00:00 2001 From: Andy C Date: Fri, 18 Oct 2024 23:03:57 -0400 Subject: [PATCH 379/506] [rename] "Oil" -> Oils or YSH In comments and so forth --- asdl/format.py | 2 +- asdl/front_end.py | 7 +++---- builtin/readline_osh.py | 2 +- builtin/trap_osh.py | 2 +- core/alloc_test.py | 16 ++++++++-------- core/completion.py | 2 +- core/error.py | 2 +- core/process.py | 2 +- core/pyutil.py | 5 +---- core/runtime.asdl | 15 +++++++++++---- core/shell.py | 2 +- core/state.py | 18 ++++++++---------- core/state_test.py | 2 +- doctools/cmark.py | 2 +- frontend/consts.py | 4 ++-- osh/glob_test.py | 4 ++-- ysh/expr_eval.py | 2 +- ysh/expr_parse_test.py | 4 ++-- ysh/regex_translate.py | 2 +- 19 files changed, 48 insertions(+), 47 deletions(-) diff --git a/asdl/format.py b/asdl/format.py index bb6f915e2b..e17fc96a76 100644 --- a/asdl/format.py +++ b/asdl/format.py @@ -143,7 +143,7 @@ def FileHeader(self): self.f.write(""" - oil AST + Oils AST