Fix destructuring alternation

Attempting to use the existing FORK_OPT opcode resulted in difficulty
knowing when to pop an error message off the stack and when not to. This
commit makes DESTRUCTURE_ALT a real opcode that is identical to
FORK_OPT, except for never pushing the error message onto the stack when
continuing from an error backtrack.

Some small changes were necessary to the DUP/POP behavior surrounding
destructuring to accomodate this.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,29 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+<!--
+READ THIS FIRST!
+
+If you have a usage question, please ask us on either Stack Overflow (https://stackoverflow.com/questions/tagged/jq) or in the #jq channel (http://irc.lc/freenode/%23jq/) on Freenode (https://webchat.freenode.net/).
+
+-->
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Provide a minimal test case to reproduce the behavior.
+If the input is large, either attach it as a file, or [create a gist](https://gist.github.com) and link to it here.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Environment (please complete the following information):**
+ - OS and Version: [e.g. macOS, Windows, Linux (please specify distro)]
+ - jq version [e.g. 1.5]
+
+**Additional context**
+Add any other context about the problem here.

.travis.yml

@@ -1,93 +1,201 @@
-os:
-    - linux
-    - osx
-
 sudo: false
 
 language: c
 
-compiler:
-    - gcc
-    - clang
+stages:
+    -   name: test
 
-matrix:
+    -   name: build
+        # Don't run build stage for pull requests to save time and resources.
+        if: type != pull_request
+
+
+jobs:
     include:
-        - compiler: gcc
-          env: COVERAGE="--disable-valgrind --enable-gcov"
-          os: linux
+        # Build with gcc and run tests on Ubuntu.
+        -   &test-ubuntu
+            stage: test
+            os: linux
+            compiler: gcc
 
-addons:
-    apt:
-        packages:
-            - valgrind
-            - bison
-            - automake
+            addons:
+                apt:
+                    packages:
+                        - valgrind
+                        - bison
+                        - automake
 
-before_install:
-    - echo "$TRAVIS_OS_NAME"
-    - uname -s
-    - brew update               || true;
-      brew install flex         || true;
-      brew install bison        || true;
-    - rvm install ruby-1.9.3-p551
-    - rvm use 1.9.3
-    - rm src/{lexer,parser}.{c,h}
-    - sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac
+            before_install:
+                - uname -s
+                - rvm install ruby-1.9.3-p551
+                - rvm use 1.9.3
+                - rm src/{lexer,parser}.{c,h}
+                - sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac
 
-install:
-    - bundle install --gemfile=docs/Gemfile
-    - wget http://ftp.debian.org/debian/pool/main/b/bison/bison_3.0.2.dfsg-2_amd64.deb
-    - ar p bison_3.0.2.dfsg-2_amd64.deb data.tar.xz | tar xJ
-    - if [ -n "$COVERAGE" ]; then pip install --user cpp-coveralls; fi
+            install:
+                - bundle install --gemfile=docs/Gemfile
+                - wget http://ftp.debian.org/debian/pool/main/b/bison/bison_3.0.2.dfsg-2_amd64.deb
+                - ar p bison_3.0.2.dfsg-2_amd64.deb data.tar.xz | tar xJ
+                - if [ -n "$COVERAGE" ]; then pip install --user cpp-coveralls; fi
 
-before_script:
-    # If this is OS X we'll get bison from brew, else we'll get bison
-    # from the .deb unpacked above in the install section.
-    - PATH=/usr/local/opt/bison/bin:$PWD/usr/bin:$PATH
-    - echo SHELL=$SHELL
-    - echo PATH=$PATH
-    - which bison
-    - bison --version
-    - autoreconf -if
-    - ./configure --with-oniguruma=builtin YACC="$(which bison) -y" $COVERAGE
+            before_script:
+                # If this is OS X we'll get bison from brew, else we'll get bison
+                # from the .deb unpacked above in the install section.
+                - PATH=/usr/local/opt/bison/bin:$PWD/usr/bin:$PATH
+                - echo SHELL=$SHELL
+                - echo PATH=$PATH
+                - which bison
+                - bison --version
+                - autoreconf -if
+                - ./configure --with-oniguruma=builtin YACC="$(which bison) -y" $COVERAGE
 
-script:
-    # When using the bison from Debian we need to tell that bison where
-    # to find its data.  Yay non-relocatable code.  Not.
-    - echo PATH=$PATH
-    - which bison
-    - make BISON_PKGDATADIR=$PWD/usr/share/bison src/parser.c || make src/parser.c
-    # Make dist!
-    #
-    # Make it first to fail the build early, before we test with
-    # valgrind.
-    - make dist
-    # Build and test the dist (without valgrind)
-    - |
-        (
-          tar xvf jq-`scripts/version`.tar.gz &&
-          cd jq-`scripts/version` &&
-          pwd &&
-          ./configure --disable-valgrind --with-oniguruma=builtin YACC="$(which bison) -y" $COVERAGE &&
-          make BISON_PKGDATADIR=$PWD/usr/share/bison src/parser.c || make src/parser.c &&
-          make -j4 &&
-          make check -j4 || true
-        )
-    # Build and test the HEAD
-    - make -j4
-    - make check -j4
+            script:
+                # When using the bison from Debian we need to tell that bison where
+                # to find its data.  Yay non-relocatable code.  Not.
+                - echo PATH=$PATH
+                - which bison
+                - make BISON_PKGDATADIR=$PWD/usr/share/bison src/parser.c || make src/parser.c
+                # Make dist!
+                #
+                # Make it first to fail the build early, before we test with
+                # valgrind.
+                - make dist
+                # Build and test the dist (without valgrind)
+                - |
+                    (
+                    tar xvf jq-`scripts/version`.tar.gz &&
+                    cd jq-`scripts/version` &&
+                    pwd &&
+                    ./configure --disable-valgrind --with-oniguruma=builtin YACC="$(which bison) -y" $COVERAGE &&
+                    make BISON_PKGDATADIR=$PWD/usr/share/bison src/parser.c || make src/parser.c &&
+                    make -j4 &&
+                    make check -j4 || true
+                    )
+                # Build and test the HEAD
+                - make -j4
+                - make check -j4
 
-after_script:
-    - |
-        if [ -n "$COVERAGE" ]; then
-            rm -rf src/.libs # don't care about coverage for libjq
-            coveralls --gcov-options '\-lp' \
-                -e src/lexer.c -e src/parser.c -e src/jv_dtoa.c
-        fi
+            after_failure:
+                - cat test-suite.log
+                - cat tests/*.log
+
+
+        # Build with clang and run tests on Ubuntu.
+        -   <<: *test-ubuntu
+            compiler: clang
+
+
+        # Build with gcc and run tests with gcov on Ubuntu.
+        -   <<: *test-ubuntu
+            env: COVERAGE="--disable-valgrind --enable-gcov"
+
+            after_script:
+                - rm -rf src/.libs # don't care about coverage for libjq
+                - coveralls --gcov-options '\-lp'
+                    -e src/lexer.c -e src/parser.c -e src/jv_dtoa.c
+
+
+        # Build with gcc and run tests on macOS.
+        -   &test-osx
+            <<: *test-ubuntu
+            os: osx
+
+            before_install:
+                - uname -s
+                - brew update
+                - brew install flex
+                - brew install bison
+                - rvm install ruby-1.9.3-p551
+                - rvm use 1.9.3
+                - rm src/{lexer,parser}.{c,h}
+                - sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac
+
+            install:
+                - bundle install --gemfile=docs/Gemfile
+                - if [ -n "$COVERAGE" ]; then pip install --user cpp-coveralls; fi
+
+
+        # Build with clang and run tests on macOS.
+        -   <<: *test-osx
+            compiler: clang
+
+
+        # Build with gcc and run tests on Alpine Linux v3.7 (inside chroot).
+        # Note: Alpine uses musl libc.
+        -   &test-alpine
+            stage: test
+            os: linux
+            language: minimal
+            compiler: gcc
+            sudo: true
+
+            before_install:
+                - "wget 'https://raw.githubusercontent.com/alpinelinux/alpine-chroot-install/v0.7.0/alpine-chroot-install' \
+                    && echo '090d323d887ef3a2fd4e752428553f22a52b87bb  alpine-chroot-install' | sha1sum -c || travis_terminate 1"
+                - alpine() { /alpine/enter-chroot -u "$USER" "$@"; }
+
+            install:
+                - sudo sh alpine-chroot-install -b v3.7 -a "$ARCH"
+                    -p 'build-base automake autoconf bison libtool oniguruma-dev'
+
+            before_script:
+                - autoreconf -if
+
+            script:
+                - alpine ./configure --disable-docs
+                - alpine make
+                - alpine make check
+
+
+        # Build release binary statically linked with musl libc on Alpine Linux
+        # (inside chroot). If building a tagged commit, then deploy release
+        # tarball to GitHub Releases.
+        -   &build-alpine
+            <<: *test-alpine
+            stage: build
+            env: ARCH=x86_64
+
+            script:
+                - alpine ./configure --disable-docs --enable-all-static
+                    CFLAGS='-Os -static -no-pie' CXXFLAGS='-Os -static -no-pie'
+                - alpine make
+                - alpine strip jq
+
+                - jq -V
+                - ls -lah jq
+                - file jq
+                # Ensure that the built executable is really statically linked.
+                - file jq | grep -Fw 'statically linked'
+
+            before_deploy:
+                - PKGNAME="jq-$TRAVIS_TAG-$ARCH-linux"
+                - mkdir $PKGNAME && mv jq $PKGNAME/
+                - tar -czf $PKGNAME.tar.gz $PKGNAME/
+                - sha256sum $PKGNAME.tar.gz > $PKGNAME.tar.gz.sha256
+
+            deploy:
+                provider: releases
+                api_key:
+                    secure:  # TODO: put encrypted GitHub token here!
+                file: jq-$TRAVIS_TAG-*.tar.gz*
+                file_glob: true
+                skip_cleanup: true
+                on:
+                    tags: true
+
+        # Build binaries for other architectures using QEMU user-mode emulation.
+        -   <<: *build-alpine
+            env: ARCH=x86
+
+        -   <<: *build-alpine
+            env: ARCH=aarch64
+
+        -   <<: *build-alpine
+            env: ARCH=armhf
+
+        -   <<: *build-alpine
+            env: ARCH=ppc64le
 
-after_failure:
-    - cat test-suite.log
-    - cat tests/*.log
 
 notifications:
     email: false

AUTHORS

@@ -19,6 +19,7 @@ cdnbacon             <mike@busbud.com>
 Charles Merriam      <charles.merriam@gmail.com>
 Colin von Heuring    <colin@janrain.com>
 Damian Gryski        <damian@gryski.com>
+David Fetter         <david@fetter.org>               - added --rawfile
 David Haguenauer     <ml@kurokatta.org>
 David R. MacIver     <david@drmaciver.com>            - bug fixes
 David Tolnay         <dtolnay@gmail.com>              - destructuring, build improvements

compile-ios.sh

@@ -10,10 +10,10 @@ NJOBS="-j`sysctl -n hw.ncpu || echo 1`"
 
 
 # Get oniguruma
-rm -rf $PWD/build/ios onig-5.9.5
-echo "Downloading oniguruma 5.9.5"
+rm -rf $PWD/build/ios oniguruma-5.9.6
+echo "Downloading oniguruma 5.9.6"
 curl -L https://github.com/kkos/oniguruma/archive/v5.9.6.tar.gz | tar xz
-cd oniguruma-5.9.5
+cd oniguruma-5.9.6
 
 # So, we need to remake the configure scripts so that the arm64 architecture
 # exists in config.sub.  In order to keep autoreconf from failing, create
@@ -50,7 +50,7 @@ for arch in i386 x86_64 armv7 armv7s arm64; do
 
 
 	# Build oniguruma for this architecture
-	cd onig-5.9.5
+	cd oniguruma-5.9.6
 	CC=$CC CFLAGS=$CFLAGS LDFLAGS=$LDFLAGS ./configure --disable-shared --enable-static --host=$HOST --prefix=$ORIG_PWD/build/ios/$arch
 	STATUS=$?
 	if [ $STATUS -ne 0 ]

docs/content/2.download/default.yml

@@ -66,8 +66,12 @@ body:
 
       ### FreeBSD
 
-         * Use [FreshPorts](https://www.freshports.org/textproc/jq/) to install
-           jq 1.4 with `pkg install jq`.
+	 * `pkg install jq` as root installs a pre-built
+	   [binary package](https://www.freebsd.org/doc/en_US.ISO8859-1/books/handbook/pkgng-intro.html).
+	 * `make -C /usr/ports/textproc/jq install clean` as root installs the
+	   [jq](https://www.freshports.org/textproc/jq/)
+	   [port](https://www.freebsd.org/doc/en_US.ISO8859-1/books/handbook/ports-using.html)
+	   from source.
 
       ### Solaris
 

docs/content/3.manual/manual.yml

@@ -236,10 +236,17 @@ sections:
 
         This option reads all the JSON texts in the named file and binds
         an array of the parsed JSON values to the given global variable.
-        If you run jq with `--argfile foo bar`, then `$foo` is available
+        If you run jq with `--slurpfile foo bar`, then `$foo` is available
         in the program and has an array whose elements correspond to the
         texts in the file named `bar`.
 
+      * `--rawfile variable-name filename`:
+
+        This option reads in the named file and binds its contents to the given
+        global variable.  If you run jq with `--rawfile foo bar`, then `$foo` is
+        available in the program and has a string whose contents are to the texs
+        in the file named `bar`.
+
       * `--argfile variable-name filename`:
 
         Do not use.  Use `--slurpfile` instead.
@@ -2596,18 +2603,10 @@ sections:
                   (2 as $item | . + $item) |
                   (1 as $item | . + $item)
 
-          If the reduction update expression outputs `empty` (that is,
-          no values), then the reduction state is left as-is.  For
-          example, `reduce range(4) as $n ({}; if .==2 then empty else
-          .[$n|tostring] |= $n)` will produce `{"0":0,"1":1,"3":3}`.
-
         examples:
           - program: 'reduce .[] as $item (0; . + $item)'
             input: '[10,2,5,3]'
             output: ['20']
-          - program: 'reduce .[] as $n ([]; if $n%2==0 then empty else . + [$n] end)'
-            input: '[0,1,2,3,4,5]'
-            output: ['[1,3,5]']
 
       - title: "`isempty(exp)`"
         body: |
@@ -2936,7 +2935,7 @@ sections:
       can be represented in JSON.
 
       All the assignment operators in jq have path expressions on the
-      left-hand side (LHS).  The right-hand side (RHS) procides values
+      left-hand side (LHS).  The right-hand side (RHS) provides values
       to set to the paths named by the LHS path expressions.
 
       Values in jq are always immutable.  Internally, assignment works

jq.1.prebuilt

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "JQ" "1" "May 2017" "" ""
+.TH "JQ" "1" "December 2017" "" ""
 .
 .SH "NAME"
 \fBjq\fR \- Command\-line JSON processor
@@ -2100,7 +2100,7 @@ The \fBstrftime(fmt)\fR builtin formats a time (GMT) with the given format\. The
 The format strings for \fBstrptime\fR and \fBstrftime\fR are described in typical C library documentation\. The format string for ISO 8601 datetime is \fB"%Y\-%m\-%dT%H:%M:%SZ"\fR\.
 .
 .P
-jq may not support some or all of this date functionality on some systems\.
+jq may not support some or all of this date functionality on some systems\. In particular, the \fB%u\fR and \fB%j\fR specifiers for \fBstrptime(fmt)\fR are not supported on macOS\.
 .
 .IP "" 4
 .
@@ -2378,7 +2378,7 @@ break $out
 .P
 because no label \fB$out\fR is visible\.
 .
-.SS "Error Suppresion / Optional Operator: ?"
+.SS "Error Suppression / Optional Operator: ?"
 The \fB?\fR operator, used as \fBEXP?\fR, is shorthand for \fBtry EXP\fR\.
 .
 .IP "" 4
@@ -2861,25 +2861,10 @@ For each result that \fB\.[]\fR produces, \fB\. + $item\fR is run to accumulate
 0 | (3 as $item | \. + $item) |
     (2 as $item | \. + $item) |
     (1 as $item | \. + $item)
-.
-.fi
-.
-.IP "" 0
-.
-.P
-If the reduction update expression outputs \fBempty\fR (that is, no values), then the reduction state is left as\-is\. For example, \fBreduce range(4) as $n ({}; if \.==2 then empty else \.[$n|tostring] |= $n)\fR will produce \fB{"0":0,"1":1,"3":3}\fR\.
-.
-.IP "" 4
-.
-.nf
 
 jq \'reduce \.[] as $item (0; \. + $item)\'
    [10,2,5,3]
 => 20
-
-jq \'reduce \.[] as $n ([]; if $n%2==0 then empty else \. + [$n] end)\'
-   [0,1,2,3,4,5]
-=> [1,3,5]
 .
 .fi
 .

src/builtin.jq

@@ -167,7 +167,10 @@ def until(cond; next):
      def _until:
          if cond then . else (next|_until) end;
      _until;
-def limit($n; exp): if $n < 0 then exp else label $out | foreach exp as $item ([$n, null]; if .[0] < 1 then break $out else [.[0] -1, $item] end; .[1]) end;
+def limit($n; exp):
+  if $n < 0 then exp
+  else label $out | foreach exp as $item ($n; .-1; $item, if . <= 0 then break $out else empty end)
+  end;
 def isempty(g): 0 == ((label $go | g | (1, break $go)) // 0);
 def first(g): label $out | g | ., break $out;
 def last(g): reduce g as $item (null; $item);

src/compile.c

@@ -759,7 +759,7 @@ static void block_get_unbound_vars(block b, jv *vars) {
 
 /* Build wrappers around destructuring matchers so that we can chain them
  * when we have errors.  The approach is as follows:
- * FORK_OPT NEXT_MATCHER (unless last matcher)
+ * DESTRUCTURE_ALT NEXT_MATCHER (unless last matcher)
  * existing_matcher_block
  * JUMP BODY
  */
@@ -767,20 +767,18 @@ static block bind_alternation_matchers(block matchers, block body) {
   block preamble = {0};
   block altmatchers = {0};
   block mb = {0};
+  block final_matcher = matchers;
 
   // Pass through the matchers to find all destructured names.
-  while (matchers.first && matchers.first->op == DESTRUCTURE_ALT) {
-    block_append(&altmatchers, inst_block(block_take(&matchers)));
+  while (final_matcher.first && final_matcher.first->op == DESTRUCTURE_ALT) {
+    block_append(&altmatchers, inst_block(block_take(&final_matcher)));
   }
 
   // We don't have any alternations here, so we can use the simplest case.
   if (altmatchers.first == NULL) {
-    return bind_matcher(matchers, body);
+    return bind_matcher(final_matcher, body);
   }
 
-  // The final matcher needs to strip the error from the previous FORK_OPT
-  block final_matcher = BLOCK(gen_op_simple(POP), gen_op_simple(DUP), matchers);
-
   // Collect var names
   jv all_vars = jv_object();
   block_get_unbound_vars(altmatchers, &all_vars);
@@ -800,16 +798,11 @@ static block bind_alternation_matchers(block matchers, block body) {
   for (inst *i = altmatchers.first; i; i = i->next) {
     block submatcher = i->subfn;
 
-    // Get rid of the error from the previous matcher
-    if (mb.first != NULL) {
-      submatcher = BLOCK(gen_op_simple(POP), gen_op_simple(DUP), submatcher);
-    }
-
     // If we're successful, jump to the end of the matchers
     submatcher = BLOCK(submatcher, gen_op_target(JUMP, final_matcher));
 
-    // FORK_OPT to the end of this submatcher so we can skip to the next one on error
-    mb = BLOCK(mb, gen_op_target(FORK_OPT, submatcher), submatcher);
+    // DESTRUCTURE_ALT to the end of this submatcher so we can skip to the next one on error
+    mb = BLOCK(mb, gen_op_target(DESTRUCTURE_ALT, submatcher), submatcher);
 
     // We're done with this inst and we don't want it anymore
     // But we can't let it free the submatcher block.
@@ -823,32 +816,12 @@ static block bind_alternation_matchers(block matchers, block body) {
 
 block gen_reduce(block source, block matcher, block init, block body) {
   block res_var = gen_op_var_fresh(STOREV, "reduce");
-  block update_var = gen_op_bound(STOREV, res_var);
-  block jmp = gen_op_targetlater(JUMP);
-  if (body.last == NULL) {
-    inst_set_target(jmp, jmp);
-  } else {
-    inst_set_target(jmp, body);
-  }
   block loop = BLOCK(gen_op_simple(DUPN),
                      source,
                      bind_alternation_matchers(matcher,
                                   BLOCK(gen_op_bound(LOADVN, res_var),
-                                        /*
-                                         * We fork to the body, jump to
-                                         * the STOREV.  This means that
-                                         * if body produces no results
-                                         * (i.e., it just does empty)
-                                         * then we keep the current
-                                         * reduction state as-is.
-                                         *
-                                         * To break out of a
-                                         * reduction... use break.
-                                         */
-                                        gen_op_target(FORK, jmp),
-                                        jmp,
                                         body,
-                                        update_var)),
+                                        gen_op_bound(STOREV, res_var))),
                      gen_op_simple(BACKTRACK));
   return BLOCK(gen_op_simple(DUP),
                init,
@@ -1023,12 +996,13 @@ block gen_destructure(block var, block matchers, block body) {
   if (body.first && body.first->op == TOP)
     top = inst_block(block_take(&body));
 
-  if (matchers.first && matchers.first->op == DESTRUCTURE_ALT && !block_is_noop(var)) {
+  if (matchers.first && matchers.first->op == DESTRUCTURE_ALT) {
     block_append(&var, gen_op_simple(DUP));
-    block_append(&matchers, gen_op_simple(POP));
+  } else {
+    top = BLOCK(top, gen_op_simple(DUP));
   }
 
-  return BLOCK(top, gen_op_simple(DUP), gen_subexp(var), gen_op_simple(POP), bind_alternation_matchers(matchers, body));
+  return BLOCK(top, gen_subexp(var), gen_op_simple(POP), bind_alternation_matchers(matchers, body));
 }
 
 // Like gen_var_binding(), but bind `break`'s wildcard unbound variable

src/execute.c

@@ -799,21 +799,27 @@ jv jq_next(jq_state *jq) {
     }
 
     case FORK_OPT:
+    case DESTRUCTURE_ALT:
     case FORK: {
       stack_save(jq, pc - 1, stack_get_pos(jq));
       pc++; // skip offset this time
       break;
     }
 
-    case ON_BACKTRACK(FORK_OPT): {
+    case ON_BACKTRACK(FORK_OPT):
+    case ON_BACKTRACK(DESTRUCTURE_ALT): {
       if (jv_is_valid(jq->error)) {
         // `try EXP ...` backtracked here (no value, `empty`), so we backtrack more
         jv_free(stack_pop(jq));
         goto do_backtrack;
       }
       // `try EXP ...` exception caught in EXP
-      jv_free(stack_pop(jq)); // free the input
-      stack_push(jq, jv_invalid_get_msg(jq->error));  // push the error's message
+      // DESTRUCTURE_ALT doesn't want the error message on the stack,
+      // as we would just want to throw it away anyway.
+      if (opcode != ON_BACKTRACK(DESTRUCTURE_ALT)) {
+        jv_free(stack_pop(jq)); // free the input
+        stack_push(jq, jv_invalid_get_msg(jq->error));  // push the error's message
+      }
       jq->error = jv_null();
       uint16_t offset = *pc++;
       pc += offset;

src/jv_file.c

@@ -1,16 +1,33 @@
+#include <sys/stat.h>
 
 #include <errno.h>
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 #include "jv.h"
 #include "jv_unicode.h"
 
 jv jv_load_file(const char* filename, int raw) {
-  FILE* file = fopen(filename, "r");
+  struct stat sb;
+  int fd = open(filename, O_RDONLY);
+  if (fd == -1) {
+    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
+                                             filename,
+                                             strerror(errno)));
+  }
+  if (fstat(fd, &sb) == -1 || S_ISDIR(sb.st_mode)) {
+    close(fd);
+    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
+                                             filename,
+                                             "It's a directory"));
+  }
+  FILE* file = fdopen(fd, "r");
   struct jv_parser* parser = NULL;
   jv data;
   if (!file) {
+    close(fd);
     return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
                                              filename,
                                              strerror(errno)));

src/linker.c

@@ -279,6 +279,8 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block
       if (nerrors == 0) {
         // Bind the library to the program
         bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, as_str);
+        if (is_data)
+          bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, NULL);
       }
     }
     jv_free(as);

src/main.c

@@ -83,6 +83,7 @@ static void usage(int code, int keep_it_short) {
       "  --arg a v        set variable $a to value <v>;\n"
       "  --argjson a v    set variable $a to JSON value <v>;\n"
       "  --slurpfile a f  set variable $a to an array of JSON texts read from <f>;\n"
+      "  --rawfile a f    set variable $a to a string consisting of the contents of <f>;\n"
       "  --args           remaining arguments are string arguments, not files;\n"
       "  --jsonargs       remaining arguments are JSON arguments, not files;\n"
       "  --               terminates argument processing;\n\n"
@@ -166,7 +167,11 @@ static int process(jq_state *jq, jv value, int flags, int dumpopts) {
   jv result;
   while (jv_is_valid(result = jq_next(jq))) {
     if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
-      fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
+      if (options & ASCII_OUTPUT) {
+        jv_dumpf(result, stdout, JV_PRINT_ASCII);
+      } else {
+        fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
+      }
       ret = 0;
       jv_free(result);
     } else {
@@ -443,10 +448,14 @@ int main(int argc, char* argv[]) {
         continue;
       }
       if (isoption(argv[i], 0, "argfile", &short_opts) ||
+          isoption(argv[i], 0, "rawfile", &short_opts) ||
           isoption(argv[i], 0, "slurpfile", &short_opts)) {
+        int raw = isoption(argv[i], 0, "rawfile", &short_opts);
         const char *which;
         if (isoption(argv[i], 0, "argfile", &short_opts))
           which = "argfile";
+        else if (raw)
+          which = "rawfile";
         else
           which = "slurpfile";
         if (i >= argc - 2) {
@@ -454,7 +463,7 @@ int main(int argc, char* argv[]) {
           die();
         }
         if (!jv_object_has(jv_copy(program_arguments), jv_string(argv[i+1]))) {
-          jv data = jv_load_file(argv[i+2], 0);
+          jv data = jv_load_file(argv[i+2], raw);
           if (!jv_is_valid(data)) {
             data = jv_invalid_get_msg(data);
             fprintf(stderr, "%s: Bad JSON in --%s %s %s: %s\n", progname, which,

src/opcode_list.h

@@ -44,5 +44,5 @@ OP(DEPS, CONSTANT, 0, 0)
 OP(MODULEMETA, CONSTANT, 0, 0)
 OP(GENLABEL, NONE, 0, 1)
 
-OP(DESTRUCTURE_ALT, NONE, 0, 0)
+OP(DESTRUCTURE_ALT, BRANCH, 0, 0)
 OP(STOREVN, VARIABLE, 1, 0)

src/util.c

@@ -410,6 +410,7 @@ jv jq_util_input_get_current_line(jq_state* jq) {
 // When slurping, it returns just one value
 jv jq_util_input_next_input(jq_util_input_state *state) {
   int is_last = 0;
+  int has_more = 0;
   jv value = jv_invalid(); // need more input
   do {
     if (state->parser == NULL) {
@@ -441,6 +442,10 @@ jv jq_util_input_next_input(jq_util_input_state *state) {
       }
       value = jv_parser_next(state->parser);
       if (jv_is_valid(state->slurped)) {
+        // When slurping an input that doesn't have a trailing newline,
+        // we might have more than one value on the same line, so let's check
+        // to see if we have more data to parse.
+        has_more = jv_parser_remaining(state->parser);
         if (jv_is_valid(value)) {
           state->slurped = jv_array_append(state->slurped, value);
           value = jv_invalid();
@@ -450,7 +455,7 @@ jv jq_util_input_next_input(jq_util_input_state *state) {
         return value;
       }
     }
-  } while (!is_last);
+  } while (!is_last || has_more);
 
   if (jv_is_valid(state->slurped)) {
     value = state->slurped;

tests/jq.test

@@ -705,11 +705,6 @@ reduce [[1,2,10], [3,4,10]][] as [$i,$j] (0; . + $i * $j)
 null
 14
 
-# Test fix for #1313 (reduce should handle empty updates)
-reduce range(5) as $n ([]; select($n%2 == 1) | . + [$n])
-null
-[1,3]
-
 # This, while useless, should still compile.
 reduce . as $n (.; .)
 null
@@ -731,6 +726,107 @@ null
 [4, 5, null, null, 7, null]
 [null, null, null, null, null, "foo"]
 
+# Destructuring DUP/POP issues
+.[] | . as {a:$a} ?// {a:$a} ?// {a:$a} | $a
+[[3],[4],[5],6]
+# Runtime error: "jq: Cannot index array with string \"c\""
+
+.[] as {a:$a} ?// {a:$a} ?// {a:$a} | $a
+[[3],[4],[5],6]
+# Runtime error: "jq: Cannot index array with string \"c\""
+
+[[3],[4],[5],6][] | . as {a:$a} ?// {a:$a} ?// {a:$a} | $a
+null
+# Runtime error: "jq: Cannot index array with string \"c\""
+
+[[3],[4],[5],6] | .[] as {a:$a} ?// {a:$a} ?// {a:$a} | $a
+null
+# Runtime error: "jq: Cannot index array with string \"c\""
+
+.[] | . as {a:$a} ?// {a:$a} ?// $a | $a
+[[3],[4],[5],6]
+[3]
+[4]
+[5]
+6
+
+.[] as {a:$a} ?// {a:$a} ?// $a | $a
+[[3],[4],[5],6]
+[3]
+[4]
+[5]
+6
+
+[[3],[4],[5],6][] | . as {a:$a} ?// {a:$a} ?// $a | $a
+null
+[3]
+[4]
+[5]
+6
+
+[[3],[4],[5],6] | .[] as {a:$a} ?// {a:$a} ?// $a | $a
+null
+[3]
+[4]
+[5]
+6
+
+.[] | . as {a:$a} ?// $a ?// {a:$a} | $a
+[[3],[4],[5],6]
+[3]
+[4]
+[5]
+6
+
+.[] as {a:$a} ?// $a ?// {a:$a} | $a
+[[3],[4],[5],6]
+[3]
+[4]
+[5]
+6
+
+[[3],[4],[5],6][] | . as {a:$a} ?// $a ?// {a:$a} | $a
+null
+[3]
+[4]
+[5]
+6
+
+[[3],[4],[5],6] | .[] as {a:$a} ?// $a ?// {a:$a} | $a
+null
+[3]
+[4]
+[5]
+6
+
+.[] | . as $a ?// {a:$a} ?// {a:$a} | $a
+[[3],[4],[5],6]
+[3]
+[4]
+[5]
+6
+
+.[] as $a ?// {a:$a} ?// {a:$a} | $a
+[[3],[4],[5],6]
+[3]
+[4]
+[5]
+6
+
+[[3],[4],[5],6][] | . as $a ?// {a:$a} ?// {a:$a} | $a
+null
+[3]
+[4]
+[5]
+6
+
+[[3],[4],[5],6] | .[] as $a ?// {a:$a} ?// {a:$a} | $a
+null
+[3]
+[4]
+[5]
+6
+
 . as $dot|any($dot[];not)
 [1,2,3,4,true,false,1,2,3,4,5]
 true

tests/setup

@@ -35,7 +35,7 @@ clean () {
     fi
 }
 trap clean EXIT
-d=`mktemp -d -t jqXXXXXX || true`
+d=$(mktemp -d -t jqXXXXXX || true)
 if [ -z "$d" ]; then
     echo "Your OS does not support mktemp(1) -d" 1>&2
     exit 1

tests/shtest

@@ -29,45 +29,45 @@ $VALGRIND $Q $JQ -Rne '[inputs] == ["a\u0000b", "c\u0000d", "e"]' $d/input
 ## move all of these into tests/all.test
 
 # String constant folding (addition only)
-nref=`$VALGRIND $Q $JQ -n --debug-dump-disasm '"foo"' | wc -l`
+nref=$($VALGRIND $Q $JQ -n --debug-dump-disasm '"foo"' | wc -l)
 
 # Numeric constant folding (not all ops yet)
-n=`$VALGRIND $Q $JQ -n --debug-dump-disasm '1+1' | wc -l`
+n=$($VALGRIND $Q $JQ -n --debug-dump-disasm '1+1' | wc -l)
 if [ $n -ne $nref ]; then
     echo "Constant expression folding for strings didn't work"
     exit 1
 fi
-n=`$VALGRIND $Q $JQ -n --debug-dump-disasm '1-1' | wc -l`
+n=$($VALGRIND $Q $JQ -n --debug-dump-disasm '1-1' | wc -l)
 if [ $n -ne $nref ]; then
     echo "Constant expression folding for strings didn't work"
     exit 1
 fi
-n=`$VALGRIND $Q $JQ -n --debug-dump-disasm '2*3' | wc -l`
+n=$($VALGRIND $Q $JQ -n --debug-dump-disasm '2*3' | wc -l)
 if [ $n -ne $nref ]; then
     echo "Constant expression folding for strings didn't work"
     exit 1
 fi
-n=`$VALGRIND $Q $JQ -n --debug-dump-disasm '9/3' | wc -l`
+n=$($VALGRIND $Q $JQ -n --debug-dump-disasm '9/3' | wc -l)
 if [ $n -ne $nref ]; then
     echo "Constant expression folding for strings didn't work"
     exit 1
 fi
-n=`$VALGRIND $Q $JQ -n --debug-dump-disasm '9==3' | wc -l`
+n=$($VALGRIND $Q $JQ -n --debug-dump-disasm '9==3' | wc -l)
 if [ $n -ne $nref ]; then
     echo "Constant expression folding for strings didn't work"
     exit 1
 fi
-n=`$VALGRIND $Q $JQ -n --debug-dump-disasm '9!=3' | wc -l`
+n=$($VALGRIND $Q $JQ -n --debug-dump-disasm '9!=3' | wc -l)
 if [ $n -ne $nref ]; then
     echo "Constant expression folding for strings didn't work"
     exit 1
 fi
-n=`$VALGRIND $Q $JQ -n --debug-dump-disasm '9<=3' | wc -l`
+n=$($VALGRIND $Q $JQ -n --debug-dump-disasm '9<=3' | wc -l)
 if [ $n -ne $nref ]; then
     echo "Constant expression folding for strings didn't work"
     exit 1
 fi
-n=`$VALGRIND $Q $JQ -n --debug-dump-disasm '9>=3' | wc -l`
+n=$($VALGRIND $Q $JQ -n --debug-dump-disasm '9>=3' | wc -l)
 if [ $n -ne $nref ]; then
     echo "Constant expression folding for strings didn't work"
     exit 1
@@ -122,6 +122,14 @@ if $VALGRIND $Q $JQ -e . $d/input; then
   exit 2
 fi
 
+# Regression test for #1534
+echo "[1,2,3,4]" > $d/expected
+printf "[1,2][3,4]" | $JQ -cs add > $d/out 2>&1
+cmp $d/out $d/expected
+printf "[1,2][3,4]\n" | $JQ -cs add > $d/out 2>&1
+cmp $d/out $d/expected
+
+
 ## Test streaming parser
 
 ## If we add an option to stream to the `import ... as $symbol;` directive
@@ -137,15 +145,15 @@ if which seq > /dev/null 2>&1; then
     # test very, very slow when run with valgrind, and the whole point
     # is to run it with valgrind.
     #
-    #len=`wc -c < "$JQTESTDIR/torture/input0.json"`
+    #len=$(wc -c < "$JQTESTDIR/torture/input0.json")
     if [ -z "$VALGRIND" ]; then
         start=1
-        end=`wc -c < "$JQTESTDIR/torture/input0.json"`
+        end=$(wc -c < "$JQTESTDIR/torture/input0.json")
     else
         start=120
         end=151
     fi
-    for i in `seq $start $end`; do
+    for i in $(seq $start $end); do
         dd "if=tests/torture/input0.json" bs=$i count=1 2>/dev/null |
             $VALGRIND $JQ -c . > $d/out0 2>$d/err || true
         if [ -n "$VALGRIND" ]; then
@@ -170,7 +178,7 @@ fi
 
 ## Fuzz parser
 
-## XXX With a `urandom` builtin we could move this test into tests/all.test
+## XXX With a $(urandom) builtin we could move this test into tests/all.test
 clean=false
 if dd if=/dev/urandom bs=16 count=1024 > $d/rand 2>/dev/null; then
     # Have a /dev/urandom, good
@@ -189,12 +197,12 @@ clean=true
 
 # Check handling of ~/.jq; these can't move into jq_test.c yet because
 # they depend on $HOME
-if [ "`HOME="$mods" $VALGRIND $Q $JQ -nr fg`" != foobar ]; then
+if [ "$(HOME="$mods" $VALGRIND $Q $JQ -nr fg)" != foobar ]; then
     echo "Bug #479 appears to be back" 1>&2
     exit 1
 fi
 
-if [ `HOME="$mods" $VALGRIND $Q $JQ --debug-dump-disasm -n fg | grep '^[a-z]' | wc -l` -gt 3 ]; then
+if [ $(HOME="$mods" $VALGRIND $Q $JQ --debug-dump-disasm -n fg | grep '^[a-z]' | wc -l) -gt 3 ]; then
     echo "Binding too many defs into program" 1>&2
     exit 1
 fi
@@ -231,15 +239,15 @@ elif [ $? -ne 11 ]; then
     echo "jq halt_error(11) had wrong error code" 1>&2
     exit 1
 fi
-if [ -n "`$VALGRIND $Q $JQ -n 'halt_error(1)' 2>&1`" ]; then
+if [ -n "$($VALGRIND $Q $JQ -n 'halt_error(1)' 2>&1)" ]; then
     echo "jq halt_error(1) had unexpected output" 1>&2
     exit 1
 fi
-if [ -n "`$VALGRIND $Q $JQ -n '"xyz\n"|halt_error(1)' 2>/dev/null`" ]; then
+if [ -n "$($VALGRIND $Q $JQ -n '"xyz\n"|halt_error(1)' 2>/dev/null)" ]; then
     echo "jq halt_error(1) had unexpected output on stdout" 1>&2
     exit 1
 fi
-if [ "`$VALGRIND $Q $JQ -n '"xyz\n"|halt_error(1)' 2>&1`" != xyz ]; then
+if [ "$($VALGRIND $Q $JQ -n '"xyz\n"|halt_error(1)' 2>&1)" != xyz ]; then
     echo "jq halt_error(1) had unexpected output" 1>&2
     exit 1
 fi

tests/utf8test

@@ -2,7 +2,7 @@
 
 . "${0%/*}/setup" "$@"
 
-if [ "`$VALGRIND $Q $JQ -nf $JQTESTDIR/utf8-truncate.jq`" != "true" ]; then
+if [ "$($VALGRIND $Q $JQ -nf $JQTESTDIR/utf8-truncate.jq)" != "true" ]; then
 	echo "UTF-8 byte sequences that span the jv_load_file read buffer are mangled"
 	exit 1
 fi