Add --strip-comments option

Previously, bindings were parsed as the combination of two tokens:
  '$' IDENT

This meant that using a keyword as variable name (e.g. $then, $label)
did not work.
Attempts were made to allow $keyword to work by adding some '$' Keyword
rules in the parser, but this did not allow $keyword in all places:

  jq --arg label foo -n '$label'  # ok

  jq -n '"foo" as $label | .'     # error

Treating $foo as a single token is much simpler, in my opinion.

This patch also changes how LOC is lexed: "$__loc__" instead of as
"__loc__" that gets combined with '$' in the parser.

This patch also disallows having spaces after '$' when recalling a
variable  `$ foo'  since that was probably just an unintentional side
effect of the implementation, and it was not documented.

Fixes #2675

.gitattributes

@@ -1,3 +1,7 @@
 .gitattributes export-ignore
 .gitignore export-ignore
 * text=auto eol=lf
+
+# generated files
+src/lexer.[ch] linguist-generated=true
+src/parser.[ch] linguist-generated=true

.github/workflows/ci.yml

@@ -0,0 +1,364 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - "jq-*"
+  pull_request:
+
+jobs:
+  linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          [
+            amd64,
+            arm64,
+            armel,
+            armhf,
+            i386,
+            mips,
+            mips64,
+            mips64el,
+            mips64r6,
+            mips64r6el,
+            mipsel,
+            mipsr6,
+            mipsr6el,
+            powerpc,
+            ppc64el,
+            s390x,
+            riscv64,
+          ]
+        include:
+          - arch: amd64
+            CC: "x86_64-linux-gnu"
+          - arch: arm64
+            CC: "aarch64-linux-gnu"
+          - arch: armel
+            CC: "arm-linux-gnueabi"
+          - arch: armhf
+            CC: "arm-linux-gnueabihf"
+          - arch: i386
+            CC: "i686-linux-gnu"
+          - arch: mips
+            CC: "mips-linux-gnu"
+          - arch: mips64
+            CC: "mips64-linux-gnuabi64"
+          - arch: mips64el
+            CC: "mips64el-linux-gnuabi64"
+          - arch: mips64r6
+            CC: "mipsisa64r6-linux-gnuabi64"
+          - arch: mips64r6el
+            CC: "mipsisa64r6el-linux-gnuabi64"
+          - arch: mipsel
+            CC: "mipsel-linux-gnu"
+          - arch: mipsr6
+            CC: "mipsisa32r6-linux-gnu"
+          - arch: mipsr6el
+            CC: "mipsisa32r6el-linux-gnu"
+          - arch: powerpc
+            CC: "powerpc-linux-gnu"
+          - arch: ppc64el
+            CC: "powerpc64le-linux-gnu"
+          - arch: s390x
+            CC: "s390x-linux-gnu"
+          - arch: riscv64
+            CC: "riscv64-linux-gnu"
+    runs-on: ubuntu-22.04
+    env:
+      AR: ${{ matrix.CC }}-ar
+      CHOST: ${{ matrix.cc }}
+      CC: ${{ matrix.cc }}-gcc
+      CPP: ${{ matrix.cc }}-cpp
+      CXX: ${{ matrix.cc }}-g++
+      LDFLAGS: -s
+      SUFFIX: linux-${{ matrix.arch }}
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Install packages
+        run: |
+          sudo apt-get update
+          sudo apt-get upgrade
+          sudo apt-get install -y automake autoconf libtool crossbuild-essential-${{ matrix.arch }}
+      - name: Build
+        run: |
+          autoreconf -i
+          ./configure \
+            --host=${{ matrix.CC }} \
+            --disable-docs \
+            --disable-maintainer-mode \
+            --disable-valgrind \
+            --with-oniguruma=builtin \
+            --enable-static \
+            --enable-all-static
+          make -j$(nproc)
+          file ./jq
+          cp ./jq jq-${{ env.SUFFIX }}
+      - name: Test
+        # Only run tests for amd64 matching the CI machine arch
+        if: ${{ matrix.arch == 'amd64' }}
+        run: |
+          make check
+          git diff --exit-code
+      - name: Upload Test Logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-logs-${{ env.SUFFIX }}
+          retention-days: 7
+          path: |
+            test-suite.log
+            tests/*.log
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: jq-${{ env.SUFFIX }}
+          path: jq-${{ env.SUFFIX }}
+          if-no-files-found: error
+          retention-days: 7
+
+  macos:
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          [
+            amd64,
+            arm64,
+          ]
+        include:
+          - arch: amd64
+            target: "x86_64-apple-darwin"
+          - arch: arm64
+            target: "arm64-apple-darwin"
+    runs-on: macos-13
+    env:
+      LDFLAGS: -s
+      SUFFIX: macos-${{ matrix.arch }}
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Install packages
+        run: |
+          # brew update sometimes fails with "Fetching /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask failed!"
+          brew update || brew update-reset
+          brew install autoconf automake libtool
+      - name: "Set CC"
+        run: |
+          echo "CC=clang -target ${{ matrix.target }}$(uname -r)" >> $GITHUB_ENV
+      - name: Build
+        run: |
+          autoreconf -i
+          ./configure \
+            --host=${{ matrix.target }}$(uname -r) \
+            --disable-docs \
+            --disable-maintainer-mode \
+            --disable-valgrind \
+            --with-oniguruma=builtin \
+            --enable-static \
+            --enable-all-static
+          make -j$(nproc)
+          file ./jq
+          cp ./jq jq-${{ env.SUFFIX }}
+      - name: Test
+        # Only run tests for amd64 matching the CI machine arch
+        if: ${{ matrix.arch == 'amd64' }}
+        run: |
+          make check
+          git diff --exit-code
+      - name: Upload Test Logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-logs-${{ env.SUFFIX }}
+          retention-days: 7
+          path: |
+            test-suite.log
+            tests/*.log
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: jq-${{ env.SUFFIX }}
+          path: jq-${{ env.SUFFIX }}
+          if-no-files-found: error
+          retention-days: 7
+
+  windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          [
+            amd64,
+            i386,
+          ]
+        include:
+          - arch: amd64
+            CC: "x86_64-pc-msys"
+            toolchain: "gcc"
+            msystem: msys
+          - arch: i386
+            CC: "i686-w64-mingw32"
+            toolchain: "mingw-w64-i686-toolchain"
+            msystem: mingw32
+    runs-on: windows-2022
+    env:
+      CHOST: ${{ matrix.CC }}
+      CC: ${{ matrix.CC }}-gcc
+      LDFLAGS: -s
+      SUFFIX: windows-${{ matrix.arch }}
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - uses: msys2/setup-msys2@v2
+        with:
+          update: true
+          msystem: ${{ matrix.msystem }}
+          install: >-
+            base-devel
+            git
+            autoconf
+            automake
+            libtool
+            ${{ matrix.toolchain }}
+      - name: Build
+        shell: msys2 {0}
+        run: |
+          autoreconf -i
+          ./configure \
+            --host=${{ matrix.CC }} \
+            --disable-docs \
+            --disable-maintainer-mode \
+            --disable-valgrind \
+            --with-oniguruma=builtin \
+            --disable-shared \
+            --enable-static \
+            --enable-all-static
+          make -j$(nproc)
+          file ./jq.exe
+          cp ./jq.exe jq-${{ env.SUFFIX }}.exe
+      - name: Test
+        # Only run tests for amd64 matching the CI machine arch
+        if: ${{ matrix.arch == 'amd64' }}
+        shell: msys2 {0}
+        run: |
+          make check
+          git diff --exit-code --ignore-submodules
+      - name: Upload Test Logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-logs-${{ env.SUFFIX }}
+          retention-days: 7
+          path: |
+            test-suite.log
+            tests/*.log
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: jq-${{ env.SUFFIX }}
+          path: jq-${{ env.SUFFIX }}.exe
+          if-no-files-found: error
+          retention-days: 7
+
+  dist:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/jq-')
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Install packages
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y automake autoconf
+      - name: Create dist
+        run: |
+          autoreconf -i
+          ./configure \
+            --disable-docs \
+            --disable-maintainer-mode \
+            --disable-valgrind \
+            --with-oniguruma=builtin
+          make dist dist-zip
+          git diff --exit-code
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: jq-dist
+          if-no-files-found: error
+          retention-days: 7
+          path: |
+            jq-*.tar.gz
+            jq-*.zip
+
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    if: startsWith(github.ref, 'refs/tags/jq-')
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Docker metadata
+        uses: docker/metadata-action@v4
+        id: metadata
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: type=match,pattern=jq-(.*),group=1,value=${{ github.ref_name }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and release Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          provenance: false
+          platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
+          tags: ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}
+
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs: [linux, macos, windows, dist, docker]
+    if: startsWith(github.ref, 'refs/tags/jq-')
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+      - name: Merge built artifacts
+        uses: actions/download-artifact@v3
+      - name: Upload release
+        env:
+          TAG_NAME: ${{ github.ref_name }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p "release"
+          for files in jq-*; do
+              cp -rf "${files}/"* "release/"
+          done
+
+          gh release create "$TAG_NAME" --draft --title "jq ${TAG_NAME#jq-}" --generate-notes
+          gh release upload "$TAG_NAME" --clobber release/jq-*

.github/workflows/linux.yml

@@ -1,70 +0,0 @@
-name: Linux Build
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-jobs:
-  linux:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        name: [linux-clang, linux-gcc]
-        include:
-          - name: linux-clang
-            os: ubuntu-22.04
-            compiler: clang
-            cflags: ''
-          - name: linux-gcc
-            os: ubuntu-22.04
-            compiler: gcc
-            cflags: '-Wnonnull'
-    steps:
-      - name: Clone repository
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Install packages
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -y \
-            automake \
-            autoconf \
-            bison \
-            flex \
-            gdb \
-            python3 \
-            valgrind
-      - name: Build
-        env:
-          CC: ${{ matrix.compiler }}
-          MAKEVARS: ${{ matrix.makevars }}
-        run: |
-          autoreconf -fi
-          rm src/lexer.c src/lexer.h
-          rm src/parser.c src/parser.h
-          ./configure --disable-valgrind --with-oniguruma=builtin YACC="$(which bison) -y" $COVERAGE
-          make -j4
-          make dist
-      - name: Test
-        env:
-          CC: ${{ matrix.compiler }}
-          MAKEVARS: ${{ matrix.makevars }}
-        run: |
-          ulimit -c unlimited
-          make -j4 check
-      - name: Upload Test Logs
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-logs
-          path: |
-            test-suite.log
-            tests/*.log
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: jq-${{ matrix.name }}
-          path: |
-            jq

.github/workflows/macos.yml

@@ -1,62 +0,0 @@
-name: macOS Build
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-jobs:
-  macos:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        name: [macos-12-clang, macos-13-clang]
-        include:
-          - name: macos-12-clang
-            os: macos-12
-            compiler: clang
-          - name: macos-13-clang
-            os: macos-13
-            compiler: clang
-    steps:
-      - name: Clone repository
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Install packages
-        run: |
-          brew update
-          brew install autoconf automake libtool flex bison
-          sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac
-      - name: Build
-        env:
-          CC: ${{ matrix.compiler }}
-          MAKEVARS: ${{ matrix.makevars }}
-        run: |
-          autoreconf -fi
-          rm src/lexer.c src/lexer.h
-          rm src/parser.c src/parser.h
-          ./configure --disable-valgrind --with-oniguruma=builtin YACC="$(brew --prefix)/opt/bison/bin/bison -y" $COVERAGE
-          make -j4
-          make dist
-      - name: Test
-        env:
-          CC: ${{ matrix.compiler }}
-          MAKEVARS: ${{ matrix.makevars }}
-        run: |
-          ulimit -c unlimited
-          make -j4 check
-      - name: Upload Test Logs
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-logs
-          path: |
-            test-suite.log
-            tests/*.log
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: jq-${{ matrix.name }}
-          path: |
-            jq

.github/workflows/scanbuild.yml

@@ -48,7 +48,7 @@ jobs:
                 CC: ${{ matrix.compiler }}
                 MAKEVARS: ${{ matrix.makevars }}
               run: |
-                autoreconf -fi
+                autoreconf -i
                 rm src/lexer.c src/lexer.h
                 rm src/parser.c src/parser.h
                 ./configure --with-oniguruma=builtin YACC="$(which bison) -y" $COVERAGE

.github/workflows/windows.yml

@@ -1,45 +0,0 @@
-name: Windows Build
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-jobs:
-  windows:
-    runs-on: windows-latest
-    steps:
-      - name: Clone repository
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-      - uses: msys2/setup-msys2@v2
-        with:
-          update: true
-          install: >-
-            base-devel
-            git
-            clang
-            autoconf
-            automake
-            libtool
-            p7zip
-      - shell: msys2 {0}
-        run: |
-          autoreconf -fiv
-          ./configure --disable-valgrind --with-oniguruma=builtin --disable-shared --enable-static --enable-all-static
-          make -j4
-          make SUBDIRS= "TESTS=tests/mantest tests/jqtest tests/onigtest tests/base64test" check
-      - name: Upload Test Logs
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-logs
-          path: |
-            test-suite.log
-            tests/*.log
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: jq-windows
-          path: |
-            jq.exe

.gitignore

@@ -33,6 +33,7 @@ autom4te.cache
 INSTALL
 Makefile
 jq-*.tar.gz
+jq-*.zip
 configure
 aclocal.m4
 Makefile.in

Dockerfile

@@ -1,54 +1,36 @@
-FROM debian:9
+FROM debian:12-slim AS builder
 
 ENV DEBIAN_FRONTEND=noninteractive \
     DEBCONF_NONINTERACTIVE_SEEN=true \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8
 
+RUN apt-get update \
+ && apt-get install -y \
+      build-essential \
+      autoconf \
+      libtool \
+      git \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
 COPY . /app
+RUN autoreconf -i \
+ && ./configure \
+      --disable-docs \
+      --disable-maintainer-mode \
+      --disable-valgrind \
+      --with-oniguruma=builtin \
+      --enable-static \
+      --enable-all-static \
+      --prefix=/usr/local \
+ && make -j$(nproc) \
+ && make check \
+ && make install-strip
 
-# get dependencies, build, and remove anything we don't need for running jq.
-# valgrind seems to have trouble with pthreads TLS so it's off.
+FROM scratch
 
-RUN apt-get update && \
-    apt-get install -y \
-        build-essential \
-        autoconf \
-        libtool \
-        git \
-        bison \
-        flex \
-        python3 \
-        python3-pip \
-        wget && \
-    pip3 install pipenv && \
-    (cd /app/docs && pipenv sync) && \
-    (cd /app && \
-        git submodule init && \
-        git submodule update && \
-        autoreconf -i && \
-        ./configure --disable-valgrind --enable-all-static --prefix=/usr/local && \
-        make -j8 && \
-        make check && \
-        make install ) && \
-    (cd /app/modules/oniguruma && \
-        make uninstall ) && \
-    (cd /app && \
-        make distclean ) && \
-    apt-get purge -y \
-        build-essential \
-        autoconf \
-        libtool \
-        bison \
-        git \
-        flex \
-        python3 \
-        python3-pip && \
-    apt-get autoremove -y && \
-    rm -rf /app/modules/oniguruma/* && \
-    rm -rf /app/modules/oniguruma/.git && \
-    rm -rf /app/modules/oniguruma/.gitignore && \
-    rm -rf /var/lib/apt/lists/* /var/lib/gems
-
-ENTRYPOINT ["/usr/local/bin/jq"]
-CMD []
+COPY --from=builder /app/AUTHORS /app/COPYING /usr/local/bin/jq /
+RUN ["/jq", "--version"]
+ENTRYPOINT ["/jq"]

Makefile.am

@@ -132,7 +132,7 @@ endif
 
 ### Tests (make check)
 
-TESTS = tests/optionaltest tests/mantest tests/jqtest tests/onigtest tests/shtest tests/utf8test tests/base64test
+TESTS = tests/optionaltest tests/mantest tests/jqtest tests/onigtest tests/shtest tests/utf8test tests/base64test tests/commenttest
 TESTS_ENVIRONMENT = NO_VALGRIND=$(NO_VALGRIND)
 
 # This is a magic make variable that causes it to treat tests/man.test as a
@@ -215,7 +215,8 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
         tests/optional.test tests/optionaltest				\
         tests/utf8-truncate.jq tests/utf8test				\
         tests/base64.test tests/base64test 				\
-        tests/jq-f-test.sh tests/shtest
+        tests/jq-f-test.sh tests/shtest					\
+        tests/commenttest
 
 DISTCHECK_CONFIGURE_FLAGS=--disable-maintainer-mode --with-oniguruma=builtin
 

README.md

@@ -3,10 +3,6 @@ jq
 
 jq is a lightweight and flexible command-line JSON processor.
 
-[![Linux Build](https://github.com/jqlang/jq/actions/workflows/linux.yml/badge.svg?branch=master)](https://github.com/jqlang/jq/actions/workflows/linux.yml?query=branch%3Amaster)
-[![macOS Build](https://github.com/jqlang/jq/actions/workflows/macos.yml/badge.svg?branch=master)](https://github.com/jqlang/jq/actions/workflows/macos.yml?query=branch%3Amaster)
-[![Windows Build](https://github.com/jqlang/jq/actions/workflows/windows.yml/badge.svg?branch=master)](https://github.com/jqlang/jq/actions/workflows/windows.yml?query=branch%3Amaster)
-
 If you want to learn to use jq, read the documentation at
 [https://jqlang.github.io/jq](https://jqlang.github.io/jq).  This
 documentation is generated from the docs/ folder of this repository.
@@ -28,7 +24,7 @@ git submodule as per the instructions below.
 (note that jq's tests require regexp support to pass).  To build, run:
 
     git submodule update --init # if building from git to get oniguruma
-    autoreconf -fi              # if building from git
+    autoreconf -i               # if building from git
     ./configure --with-oniguruma=builtin
     make -j8
     make check

docs/content/manual/manual.yml

@@ -559,7 +559,7 @@ sections:
       hashes with only string keys), and "null".
 
       Booleans, null, strings and numbers are written the same way as
-      in javascript. Just like everything else in jq, these simple
+      in JSON. Just like everything else in jq, these simple
       values take an input and produce an output - `42` is a valid jq
       expression that takes an input, ignores it, and returns 42
       instead.
@@ -779,7 +779,7 @@ sections:
           Division by zero raises an error. `x % y` computes x modulo y.
 
           Multiplying a string by a number produces the concatenation of
-          that string that many times. `"x" * 0` produces **null**.
+          that string that many times. `"x" * 0` produces `""`.
 
           Dividing a string by another splits the first using the second
           as separators.
@@ -925,6 +925,24 @@ sections:
             input: '{"a": 1, "b": 2, "c": 3}'
             output: ['{"a": 2, "b": 3, "c": 4}']
 
+      - title: "`pick(pathexps)`"
+        body: |
+        
+          Emit the projection of the input object or array defined by the specified 
+          sequence of path expressions, such that if p is any one of these specifications,
+          then `(. | p)` will evaluate to the same value as `(. | pick(pathexps) | p)`.
+          For arrays, negative indices and .[m:n] specifications should not be used.
+
+        examples:
+          - program: 'pick(.a, .b.c, .x)'
+            input: '{"a": 1, "b": {"c": 2, "d": 3}, "e": 4}'
+            output: ['{"a":1,"b":{"c":2},"x":null}']
+
+          - program: '[1,2,3,4] | pick(.[2], .[0], .[0])'
+            input: '[1,2,3,4]'
+            output: ['[1,null,3]']
+
+
       - title: "`path(path_expression)`"
         body: |
 
@@ -1123,25 +1141,22 @@ sections:
             input: 'null'
             output: ['"{\"file\":\"<top-level>\",\"line\":1}"']
 
-      - title: "`paths`, `paths(node_filter)`, `leaf_paths`"
+      - title: "`paths`, `paths(node_filter)`"
         body: |
 
           `paths` outputs the paths to all the elements in its input
           (except it does not output the empty list, representing .
           itself).
 
-          `paths(f)` outputs the paths to any values for which `f` is true.
-          That is, `paths(numbers)` outputs the paths to all numeric
+          `paths(f)` outputs the paths to any values for which `f` is `true`.
+          That is, `paths(type == "number")` outputs the paths to all numeric
           values.
 
-          `leaf_paths` is an alias of `paths(scalars)`; `leaf_paths` is
-          *deprecated* and will be removed in the next major release.
-
         examples:
           - program: '[paths]'
             input: '[1,[[],{"a":2}]]'
             output: ['[[0],[1],[1,0],[1,1],[1,1,"a"]]']
-          - program: '[paths(scalars)]'
+          - program: '[paths(type == "number")]'
             input: '[1,[[],{"a":2}]]'
             output: ['[[0],[1,1,"a"]]']
 
@@ -1720,7 +1735,7 @@ sections:
             output: ['24']
 
 
-      - title: "`recurse(f)`, `recurse`, `recurse(f; condition)`, `recurse_down`"
+      - title: "`recurse(f)`, `recurse`, `recurse(f; condition)`"
         body: |
 
           The `recurse(f)` function allows you to search through a
@@ -1754,10 +1769,6 @@ sections:
           to generate all the integers, at least in principle, one
           could write `recurse(.+1; true)`.
 
-          For legacy reasons, `recurse_down` exists as an alias to
-          calling `recurse` without arguments. This alias is considered
-          *deprecated* and will be removed in the next major release.
-
           The recursive calls in `recurse` will not consume additional
           memory whenever `f` produces at most a single output for each
           input.
@@ -2116,16 +2127,24 @@ sections:
       - title: "`==`, `!=`"
         body: |
 
-          The expression 'a == b' will produce 'true' if the result of a and b
-          are equal (that is, if they represent equivalent JSON documents) and
+          The expression 'a == b' will produce 'true' if the results of evaluating
+          a and b are equal (that is, if they represent equivalent JSON values) and
           'false' otherwise. In particular, strings are never considered equal
-          to numbers. If you're coming from JavaScript, jq's == is like
-          JavaScript's === - considering values equal only when they have the
-          same type as well as the same value.
+          to numbers.  In checking for the equality of JSON objects, the ordering of keys
+          is irrelevant.  If you're coming from JavaScript, please note that jq's `==` is like
+          JavaScript's `===`, the "strict equality" operator.
 
           != is "not equal", and 'a != b' returns the opposite value of 'a == b'
 
         examples:
+          - program: '. == false'
+            input: 'null'
+            output: ['false']
+
+          - program: '. == {"b": {"d": (4 + 1e-20), "c": 3}, "a":1}'
+            input: '{"a":1, "b": {"c": 3, "d": 4}}'
+            output: ['true']
+
           - program: '.[] == 1'
             input: '[1, 1.0, "1", "banana"]'
             output: ['true', 'true', 'false', 'false']
@@ -2139,14 +2158,14 @@ sections:
 
           `if A then B end` is the same as `if A then B else .  end`.
           That is, the `else` branch is optional, and if absent is the
-          same as `.`.
+          same as `.`. This also applies to `elif` with absent ending `else` branch.
 
           Checking for false or null is a simpler notion of
           "truthiness" than is found in JavaScript or Python, but it
           means that you'll sometimes have to be more explicit about
           the condition you want.  You can't test whether, e.g. a
-          string is empty using `if .name then A else B end`, you'll
-          need something more like `if .name == "" then A else B end` instead.
+          string is empty using `if .name then A else B end`; you'll
+          need something like `if .name == "" then A else B end` instead.
 
           If the condition `A` produces multiple results, then `B` is evaluated
           once for each result that is not false or null, and `C` is evaluated
@@ -2473,32 +2492,43 @@ sections:
             input: '("ab,cd", "ef, gh")'
             output: ['"ab"', '"cd"', '"ef"', '"gh"']
 
-      - title: "`sub(regex; tostring)`, `sub(regex; string; flags)`"
+      - title: "`sub(regex; tostring)`, `sub(regex; tostring; flags)`"
         body: |
 
-          Emit the string obtained by replacing the first match of regex in the
-          input string with `tostring`, after interpolation.  `tostring` should
-          be a jq string, and may contain references to named captures. The
-          named captures are, in effect, presented as a JSON object (as
-          constructed by `capture`) to `tostring`, so a reference to a captured
-          variable named "x" would take the form: `"\(.x)"`.
+          Emit the string obtained by replacing the first match of
+          regex in the input string with `tostring`, after
+          interpolation.  `tostring` should be a jq string or a stream
+          of such strings, each of which may contain references to
+          named captures. The named captures are, in effect, presented
+          as a JSON object (as constructed by `capture`) to
+          `tostring`, so a reference to a captured variable named "x"
+          would take the form: `"\(.x)"`.
 
         example:
-          - program: 'sub("^[^a-z]*(?<x>[a-z]*).*")'
-            input: '"123abc456"'
-            output: '"ZabcZabc"'
 
+          - program: 'sub("[^a-z]*(?<x>[a-z]*)"; "Z\(.x)"; "g")'
+            input: '"123abc456def"'
+            output: ['"ZabcZdef"']
 
-      - title: "`gsub(regex; string)`, `gsub(regex; string; flags)`"
+          - program: '[sub("(?<a>.)"; "\(.a|ascii_upcase)", "\(.a|ascii_downcase)")]'
+            input: '"aB"'
+            output: ['["AB","aB"]']
+
+      - title: "`gsub(regex; tostring)`, `gsub(regex; tostring; flags)`"
         body: |
 
           `gsub` is like `sub` but all the non-overlapping occurrences of the regex are
-          replaced by the string, after interpolation.
+          replaced by `tostring`, after interpolation. If the second argument is a stream
+          of jq strings, then `gsub` will produce a corresponding stream of JSON strings.
 
         example:
           - program: 'gsub("(?<x>.)[^a]*"; "+\(.x)-")'
             input: '"Abcabc"'
-            output: '"+A-+a-"'
+            output: ['"+A-+a-"']
+
+          - program: '[gsub("p"; "a", "b")]'
+            input: '"p"'
+            output: ['["a","b"]']
 
 
   - title: Advanced features
@@ -2802,10 +2832,8 @@ sections:
           The `first(expr)` and `last(expr)` functions extract the first
           and last values from `expr`, respectively.
 
-          The `nth(n; expr)` function extracts the nth value output by
-          `expr`.  This can be defined as `def nth(n; expr):
-          last(limit(n + 1; expr));`.  Note that `nth(n; expr)` doesn't
-          support negative values of `n`.
+          The `nth(n; expr)` function extracts the nth value output by `expr`.
+          Note that `nth(n; expr)` doesn't support negative values of `n`.
 
         examples:
           - program: '[first(range(.)), last(range(.)), nth(./2; range(.))]'
@@ -3281,8 +3309,8 @@ sections:
       For paths starting with `~/`, the user's home directory is
       substituted for `~`.
 
-      For paths starting with `$ORIGIN/`, the path of the jq executable
-      is substituted for `$ORIGIN`.
+      For paths starting with `$ORIGIN/`, the directory where the jq
+      executable is located is substituted for `$ORIGIN`.
 
       For paths starting with `./` or paths that are `.`, the path of
       the including file is substituted for `.`.  For top-level programs

docs/content/manual/v1.5/manual.yml

@@ -891,8 +891,8 @@ sections:
           (except it does not output the empty list, representing .
           itself).
 
-          `paths(f)` outputs the paths to any values for which `f` is true.
-          That is, `paths(numbers)` outputs the paths to all numeric
+          `paths(f)` outputs the paths to any values for which `f` is `true`.
+          That is, `paths(type == "number")` outputs the paths to all numeric
           values.
 
           `leaf_paths` is an alias of `paths(scalars)`; `leaf_paths` is
@@ -902,7 +902,7 @@ sections:
           - program: '[paths]'
             input: '[1,[[],{"a":2}]]'
             output: ['[[0],[1],[1,0],[1,1],[1,1,"a"]]']
-          - program: '[paths(scalars)]'
+          - program: '[paths(type == "number")]'
             input: '[1,[[],{"a":2}]]'
             output: ['[[0],[1,1,"a"]]']
 
@@ -2765,8 +2765,8 @@ sections:
       For paths starting with "~/", the user's home directory is
       substituted for "~".
 
-      For paths starting with "$ORIGIN/", the path of the jq executable
-      is substituted for "$ORIGIN".
+      For paths starting with "$ORIGIN/", the directory where the jq
+      executable is located is substituted for "$ORIGIN".
 
       For paths starting with "./" or paths that are ".", the path of
       the including file is substituted for ".".  For top-level programs

docs/content/manual/v1.6/manual.yml

@@ -1052,8 +1052,8 @@ sections:
           (except it does not output the empty list, representing .
           itself).
 
-          `paths(f)` outputs the paths to any values for which `f` is true.
-          That is, `paths(numbers)` outputs the paths to all numeric
+          `paths(f)` outputs the paths to any values for which `f` is `true`.
+          That is, `paths(type == "number")` outputs the paths to all numeric
           values.
 
           `leaf_paths` is an alias of `paths(scalars)`; `leaf_paths` is
@@ -1063,7 +1063,7 @@ sections:
           - program: '[paths]'
             input: '[1,[[],{"a":2}]]'
             output: ['[[0],[1],[1,0],[1,1],[1,1,"a"]]']
-          - program: '[paths(scalars)]'
+          - program: '[paths(type == "number")]'
             input: '[1,[[],{"a":2}]]'
             output: ['[[0],[1,1,"a"]]']
 
@@ -3150,8 +3150,8 @@ sections:
       For paths starting with "~/", the user's home directory is
       substituted for "~".
 
-      For paths starting with "$ORIGIN/", the path of the jq executable
-      is substituted for "$ORIGIN".
+      For paths starting with "$ORIGIN/", the directory where the jq
+      executable is located is substituted for "$ORIGIN".
 
       For paths starting with "./" or paths that are ".", the path of
       the including file is substituted for ".".  For top-level programs

src/builtin.c

@@ -359,9 +359,12 @@ static jv f_multiply(jq_state *jq, jv input, jv a, jv b) {
       str = b;
       num = a;
     }
-    jv res = jv_null();
-    int n = jv_number_value(num);
-    if (n > 0) {
+    jv res;
+    double d = jv_number_value(num);
+    if (d < 0 || isnan(d)) {
+      res = jv_null();
+    } else {
+      int n = d;
       size_t alen = jv_string_length_bytes(jv_copy(str));
       res = jv_string_empty(alen * n);
       for (; n > 0; n--) {
@@ -381,7 +384,7 @@ static jv f_multiply(jq_state *jq, jv input, jv a, jv b) {
 static jv f_divide(jq_state *jq, jv input, jv a, jv b) {
   jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    if (jv_number_value(b) == 0.0)
+    if (jv_number_value(b) == 0.0 && jv_number_value(a) != 0.0)
       return type_error2(a, b, "cannot be divided because the divisor is zero");
     jv r = jv_number(jv_number_value(a) / jv_number_value(b));
     jv_free(a);
@@ -930,7 +933,8 @@ static jv f_match(jq_state *jq, jv input, jv regex, jv modifiers, jv testmode) {
         match = jv_object_set(match, jv_string("string"), jv_string(""));
         match = jv_object_set(match, jv_string("captures"), jv_array());
         result = jv_array_append(result, match);
-        start += 1;
+        // ensure '"qux" | match("(?=u)"; "g")' matches just once
+        start = (const UChar*)(input_string+region->end[0]+1);
         continue;
       }
 
@@ -998,7 +1002,7 @@ static jv f_match(jq_state *jq, jv input, jv regex, jv modifiers, jv testmode) {
             jv_string((char*)ebuf)));
       break;
     }
-  } while (global && start != end);
+  } while (global && start <= end);
   onig_region_free(region,1);
   region = NULL;
   if (region)

src/builtin.jq

@@ -12,31 +12,30 @@ def add: reduce .[] as $x (null; . + $x);
 def del(f): delpaths([path(f)]);
 def _assign(paths; $value): reduce path(paths) as $p (.; setpath($p; $value));
 def _modify(paths; update):
-    reduce path(paths) as $p (.;
+    reduce path(paths) as $p ([., []];
         . as $dot
       | null
       | label $out
-      | ($dot | getpath($p)) as $v
+      | ($dot[0] | getpath($p)) as $v
       | (
           (   $$$$v
             | update
             | (., break $out) as $v
             | $$$$dot
-            | setpath($p; $v)
+            | setpath([0] + $p; $v)
           ),
           (
               $$$$dot
-            | delpaths([$p])
+            | setpath([1, (.[1] | length)]; $p)
           )
         )
-    );
+    ) | . as $dot | $dot[0] | delpaths($dot[1]);
 def map_values(f): .[] |= f;
 
 # recurse
 def recurse(f): def r: ., (f | r); r;
 def recurse(f; cond): def r: ., (f | select(cond) | r); r;
 def recurse: recurse(.[]?);
-def recurse_down: recurse;
 
 def to_entries: [keys_unsorted[] as $k | {key: $k, value: .[$k]}];
 def from_entries: map({(.key // .Key // .name // .Name): (if has("value") then .value else .Value end)}) | add | .//={};
@@ -62,7 +61,6 @@ def strings: select(type == "string");
 def nulls: select(. == null);
 def values: select(. != null);
 def scalars: select(type|. != "array" and . != "object");
-def leaf_paths: paths(scalars);
 def join($x): reduce .[] as $i (null;
             (if .==null then "" else .+$x end) +
             ($i | if type=="boolean" or type=="number" then tostring else .//"" end)
@@ -90,17 +88,20 @@ def capture($val): ($val|type) as $vt | if $vt == "string" then capture($val; nu
    elif $vt == "array" and ($val | length) > 1 then capture($val[0]; $val[1])
    elif $vt == "array" and ($val | length) > 0 then capture($val[0]; null)
    else error( $vt + " not a string or array") end;
-def scan(re):
-  match(re; "g")
-  |  if (.captures|length > 0)
+def scan($re; $flags):
+  match($re; "g" + $flags)
+    | if (.captures|length > 0)
       then [ .captures | .[] | .string ]
       else .string
-      end ;
+      end;
+def scan($re): scan($re; null);
 #
 # If input is an array, then emit a stream of successive subarrays of length n (or less),
 # and similarly for strings.
-def _nwise(a; $n): if a|length <= $n then a else a[0:$n] , _nwise(a[$n:]; $n) end;
-def _nwise($n): _nwise(.; $n);
+def _nwise($n):
+  def n: if length <= $n then . else .[0:$n] , (.[$n:] | n) end;
+  n;
+def _nwise(a; $n): a | _nwise($n);
 #
 # splits/1 produces a stream; split/1 is retained for backward compatibility.
 def splits($re; flags): . as $s
@@ -114,47 +115,26 @@ def splits($re): splits($re; null);
 # split emits an array for backward compatibility
 def split($re; flags): [ splits($re; flags) ];
 #
-# If s contains capture variables, then create a capture object and pipe it to s
-def sub($re; s):
-  . as $in
-  | [match($re)]
-  | if length == 0 then $in
-    else .[0]
-    | . as $r
-#  # create the "capture" object:
-    | reduce ( $r | .captures | .[] | select(.name != null) | { (.name) : .string } ) as $pair
-        ({}; . + $pair)
-    | $in[0:$r.offset] + s + $in[$r.offset+$r.length:]
-    end ;
-#
-# If s contains capture variables, then create a capture object and pipe it to s
-def sub($re; s; flags):
-  def subg: [explode[] | select(. != 103)] | implode;
-  # "fla" should be flags with all occurrences of g removed; gs should be non-nil if flags has a g
-  def sub1(fla; gs):
-    def mysub:
-      . as $in
-      | [match($re; fla)]
-      | if length == 0 then $in
-        else .[0] as $edit
-        | ($edit | .offset + .length) as $len
-        # create the "capture" object:
-        | reduce ( $edit | .captures | .[] | select(.name != null) | { (.name) : .string } ) as $pair
-            ({}; . + $pair)
-        | $in[0:$edit.offset]
-          + s
-          + ($in[$len:] | if length > 0 and gs then mysub else . end)
-        end ;
-    mysub ;
-    (flags | index("g")) as $gs
-    | (flags | if $gs then subg else . end) as $fla
-    | sub1($fla; $gs);
+# If s contains capture variables, then create a capture object and pipe it to s, bearing
+# in mind that s could be a stream
+def sub($re; s; $flags):
+   . as $in
+   | (reduce match($re; $flags) as $edit
+        ({result: [], previous: 0};
+            $in[ .previous: ($edit | .offset) ] as $gap
+            # create the "capture" objects (one per item in s)
+            | [reduce ( $edit | .captures | .[] | select(.name != null) | { (.name) : .string } ) as $pair
+                 ({}; . + $pair) | s ] as $inserts
+            | reduce range(0; $inserts|length) as $ix (.; .result[$ix] += $gap + $inserts[$ix])
+            | .previous = ($edit | .offset + .length ) )
+          | .result[] + $in[.previous:] )
+      // $in;
 #
 def sub($re; s): sub($re; s; "");
-# repeated substitution of re (which may contain named captures)
+#
 def gsub($re; s; flags): sub($re; s; flags + "g");
 def gsub($re; s): sub($re; s; "g");
-
+#
 ########################################################################
 # generic iterator/generator
 def while(cond; update):
@@ -183,7 +163,9 @@ def any(condition): any(.[]; condition);
 def all: all(.[]; .);
 def any: any(.[]; .);
 def last(g): reduce g as $item (null; $item);
-def nth($n; g): if $n < 0 then error("nth doesn't support negative indices") else last(limit($n + 1; g)) end;
+def nth($n; g):
+  if $n < 0 then error("nth doesn't support negative indices")
+  else label $out | foreach g as $item ($n + 1; . - 1; if . <= 0 then $item, break $out else empty end) end;
 def first: .[0];
 def last: .[-1];
 def nth($n): .[$n];
@@ -206,7 +188,7 @@ def transpose:
   | length as $length
   | reduce range(0; $max) as $j
       ([]; . + [reduce range(0;$length) as $i ([]; . + [ $in[$i][$j] ] )] )
-	        end;
+  end;
 def in(xs): . as $x | xs | has($x);
 def inside(xs): . as $x | xs | contains($x);
 def repeat(exp):
@@ -237,7 +219,6 @@ def tostream:
   getpath($p) |
   reduce path(.[]?) as $q ([$p, .]; [$p+$q]);
 
-
 # Assuming the input array is sorted, bsearch/1 returns
 # the index of the target if the target is in the input array; and otherwise
 #  (-1 - ix), where ix is the insertion point that would leave the array sorted.
@@ -279,6 +260,12 @@ def walk(f):
   else f
   end;
 
+# pathexps could be a stream of dot-paths
+def pick(pathexps):
+  . as $in
+  | reduce path(pathexps) as $a (null;
+      setpath($a; $in|getpath($a)) );
+
 # SQL-ish operators here:
 def INDEX(stream; idx_expr):
   reduce stream as $row ({}; .[$row|idx_expr|tostring] = $row);

src/jv.c

@@ -216,17 +216,13 @@ enum {
 #define JVP_FLAGS_NUMBER_NATIVE_STR   JVP_MAKE_FLAGS(JV_KIND_NUMBER, JVP_MAKE_PFLAGS(JVP_NUMBER_NATIVE, 1))
 #define JVP_FLAGS_NUMBER_LITERAL      JVP_MAKE_FLAGS(JV_KIND_NUMBER, JVP_MAKE_PFLAGS(JVP_NUMBER_DECIMAL, 1))
 
-#define STR(x) #x
-#define XSTR(x) STR(x)
-#define DBL_MAX_STR XSTR(DBL_MAX)
-#define DBL_MIN_STR "-" XSTR(DBL_MAX)
-
 // the decimal precision of binary double
 #define BIN64_DEC_PRECISION  (17)
 #define DEC_NUMBER_STRING_GUARD (14)
 
-#include <jv_thread.h>
+#include "jv_thread.h"
 #ifdef WIN32
+#ifndef __MINGW32__
 /* Copied from Heimdal: thread-specific keys; see lib/base/dll.c in Heimdal */
 
 /*
@@ -491,6 +487,9 @@ pthread_getspecific(pthread_key_t key)
 #else
 #include <pthread.h>
 #endif
+#else
+#include <pthread.h>
+#endif
 
 static pthread_key_t dec_ctx_key;
 static pthread_key_t dec_ctx_dbl_key;
@@ -658,12 +657,12 @@ static const char* jvp_literal_number_literal(jv n) {
   }
 
   if (decNumberIsInfinite(pdec)) {
-    // For backward compatibility.
-    if (decNumberIsNegative(pdec)) {
-      return DBL_MIN_STR;
-    } else {
-      return DBL_MAX_STR;
-    }
+    // We cannot preserve the literal data of numbers outside the limited
+    // range of exponent. Since `decNumberToString` returns "Infinity"
+    // (or "-Infinity"), and to reduce stack allocations as possible, we
+    // normalize infinities in the callers instead of printing the maximum
+    // (or minimum) double here.
+    return NULL;
   }
 
   if (plit->literal_data == NULL) {

src/jv.h

@@ -223,6 +223,7 @@ enum {
   JV_PARSE_SEQ              = 1,
   JV_PARSE_STREAMING        = 2,
   JV_PARSE_STREAM_ERRORS    = 4,
+  JV_PARSE_STRIP_COMMENTS   = 8
 };
 
 jv jv_parse(const char* string);

src/jv_parse.c

@@ -31,6 +31,7 @@ enum last_seen {
 };
 
 struct jv_parser {
+  pfunc (*scan)(struct jv_parser* p, char ch, jv* out);
   const char* curr_buf;
   int curr_buf_length;
   int curr_buf_pos;
@@ -65,9 +66,14 @@ struct jv_parser {
   unsigned int last_ch_was_ws:1;
 };
 
+static pfunc scan_json(struct jv_parser* p, char ch, jv* out);
+static pfunc scan_line_comment(struct jv_parser* p, char ch, jv* out);
+static pfunc scan_slash_comment(struct jv_parser* p, char ch, jv* out);
+static pfunc scan_c_comment(struct jv_parser* p, char ch, jv* out);
 
 static void parser_init(struct jv_parser* p, int flags) {
   p->flags = flags;
+  p->scan = scan_json;
   if ((p->flags & JV_PARSE_STREAMING)) {
     p->path = jv_array();
   } else {
@@ -299,7 +305,7 @@ static pfunc stream_token(struct jv_parser* p, char ch) {
         p->output = JV_ARRAY(jv_copy(p->path), p->next);
         p->next = jv_invalid();
       }
-      p->path = jv_array_set(p->path, p->stacklen - 1, jv_true()); // ready for another name:value pair
+      p->path = jv_array_set(p->path, p->stacklen - 1, jv_null()); // ready for another key:value pair
       p->last_seen = JV_LAST_COMMA;
     } else {
       assert(k == JV_KIND_NULL);
@@ -613,12 +619,42 @@ static int stream_is_top_num(struct jv_parser* p) {
 #define is_top_num(p) \
    (((p)->flags & JV_PARSE_STREAMING) ? stream_is_top_num((p)) : parse_is_top_num((p)))
 
-static pfunc scan(struct jv_parser* p, char ch, jv* out) {
-  p->column++;
-  if (ch == '\n') {
-    p->line++;
-    p->column = 0;
+
+static pfunc scan_line_comment(struct jv_parser* p, char ch, jv* out) {
+  if(ch == '\n')
+    p->scan = scan_json;
+  return OK;
+}
+
+static pfunc scan_c_comment_close(struct jv_parser* p, char ch, jv* out) {
+  if(ch == '/') {
+    p->scan = scan_json;
+  } else {
+    p->scan = scan_c_comment;
   }
+  return OK;
+}
+
+static pfunc scan_c_comment(struct jv_parser* p, char ch, jv* out) {
+  if(ch == '*') {
+    p->scan = scan_c_comment_close;
+  }
+  return OK;
+}
+
+static pfunc scan_slash_comment(struct jv_parser* p, char ch, jv* out) {
+  if(ch == '/') {
+    p->scan = scan_line_comment;
+    return OK;
+  }
+  if(ch == '*') {
+    p->scan = scan_c_comment;
+    return OK;
+  }
+  return "Incomplete comment token; slash must be followed by another slash or asterisk";
+}
+
+static pfunc scan_json(struct jv_parser* p, char ch, jv* out) {
   if (ch == '\036' /* ASCII RS; see draft-ietf-json-sequence-07 */) {
     if (check_truncation(p)) {
       if (check_literal(p) == 0 && is_top_num(p))
@@ -635,9 +671,15 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
     *out = jv_invalid();
     return OK;
   }
+
   presult answer = 0;
   p->last_ch_was_ws = 0;
   if (p->st == JV_PARSER_NORMAL) {
+    if(ch == '/' && (p->flags & JV_PARSE_STRIP_COMMENTS)) {
+      p->scan = scan_slash_comment;
+      return OK;
+    }
+
     chclass cls = classify(ch);
     if (cls == WHITESPACE)
       p->last_ch_was_ws = 1;
@@ -678,6 +720,16 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
   return answer;
 }
 
+static pfunc scan(struct jv_parser* p, char ch, jv* out) {
+  p->column++;
+  if (ch == '\n') {
+    p->line++;
+    p->column = 0;
+  }
+  return p->scan(p, ch, out);
+}
+
+
 struct jv_parser* jv_parser_new(int flags) {
   struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser));
   parser_init(p, flags);

src/jv_thread.h

@@ -2,6 +2,7 @@
 #define JV_THREAD_H
 
 #ifdef WIN32
+#ifndef __MINGW32__
 #include <windows.h>
 #include <winnt.h>
 #include <errno.h>
@@ -69,4 +70,7 @@ void *pthread_getspecific(pthread_key_t);
 #else
 #include <pthread.h>
 #endif
+#else
+#include <pthread.h>
+#endif
 #endif /* JV_THREAD_H */

src/lexer.c

@@ -281,7 +281,6 @@ typedef int16_t flex_int16_t;
 typedef uint16_t flex_uint16_t;
 typedef int32_t flex_int32_t;
 typedef uint32_t flex_uint32_t;
-typedef uint64_t flex_uint64_t;
 #else
 typedef signed char flex_int8_t;
 typedef short int flex_int16_t;
@@ -446,7 +445,7 @@ struct yy_buffer_state
 	/* Number of characters read into yy_ch_buf, not including EOB
 	 * characters.
 	 */
-	yy_size_t yy_n_chars;
+	int yy_n_chars;
 
 	/* Whether we "own" the buffer - i.e., we know we created it,
 	 * and can realloc() it to grow it, and should free() it to
@@ -523,7 +522,7 @@ static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file , yyscan_t yyscanner
 
 YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size , yyscan_t yyscanner );
 YY_BUFFER_STATE yy_scan_string ( const char *yy_str , yyscan_t yyscanner );
-YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, yy_size_t len , yyscan_t yyscanner );
+YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len , yyscan_t yyscanner );
 
 void *yyalloc ( yy_size_t , yyscan_t yyscanner );
 void *yyrealloc ( void *, yy_size_t , yyscan_t yyscanner );
@@ -570,12 +569,12 @@ static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner );
  */
 #define YY_DO_BEFORE_ACTION \
 	yyg->yytext_ptr = yy_bp; \
-	yyleng = (yy_size_t) (yy_cp - yy_bp); \
+	yyleng = (int) (yy_cp - yy_bp); \
 	yyg->yy_hold_char = *yy_cp; \
 	*yy_cp = '\0'; \
 	yyg->yy_c_buf_p = yy_cp;
-#define YY_NUM_RULES 50
-#define YY_END_OF_BUFFER 51
+#define YY_NUM_RULES 51
+#define YY_END_OF_BUFFER 52
 /* This struct is not used in this scanner,
    but its presence is necessary. */
 struct yy_trans_info
@@ -583,25 +582,26 @@ struct yy_trans_info
 	flex_int32_t yy_verify;
 	flex_int32_t yy_nxt;
 	};
-static const flex_int16_t yy_accept[158] =
+static const flex_int16_t yy_accept[163] =
     {   0,
         0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
-        0,    0,   51,   49,   48,   48,   49,   40,    1,   35,
-       35,   36,   37,   35,   35,   35,   35,   35,   39,   35,
-       35,   35,   35,   49,   46,   46,   46,   46,   46,   46,
+        0,    0,   52,   50,   49,   49,   50,   40,    1,   35,
+       35,   36,   37,   35,   35,   35,   35,   35,   35,   39,
+       35,   35,   35,   35,   50,   46,   46,   46,   46,   46,
        46,   46,   46,   46,   46,   46,   46,   46,   35,   44,
-       44,   42,   45,   48,    2,    1,   29,   27,   25,   26,
-       33,   39,   47,   18,   28,   39,   39,    0,   31,    3,
-       32,    0,   38,   46,    0,   46,   46,    4,   46,   46,
-       46,   46,   46,   46,    9,   46,   46,   46,   46,   14,
-       46,   46,   46,   24,   44,   43,   41,   43,   47,   30,
+       44,   42,   45,   49,    2,    1,   48,   48,   29,   27,
+       25,   26,   33,   39,   47,   18,   28,   39,   39,    0,
+       31,    3,   32,    0,   38,   46,    0,   46,    4,   46,
+       46,   46,   46,   46,   46,    9,   46,   46,   46,   46,
+       14,   46,   46,   46,   24,   44,   43,   41,   43,   48,
 
-       39,    0,   39,   34,    0,   46,   13,   46,   46,    8,
-       46,   46,   15,   46,   46,   46,   46,   46,   46,   46,
-       19,    0,   43,   46,   46,   46,   46,   12,   11,   46,
-       46,   46,   46,   46,   46,   10,   43,   46,   22,   20,
-       46,   46,   46,   21,   46,   46,   43,   46,   46,    5,
-       46,    7,   16,   23,   17,    6,    0
+        0,   48,   47,   30,   39,    0,   39,   34,    0,   13,
+       46,   46,    8,   46,   46,   15,   46,   46,   46,   46,
+       46,   46,   46,   19,    0,   43,    0,   48,   46,   46,
+       46,   12,   11,   46,   46,   46,   46,   46,   46,   10,
+       43,   48,   22,   20,   46,   46,   46,   21,   46,   46,
+       43,   48,   46,    5,   46,    7,   16,   48,   17,    6,
+       23,    0
     } ;
 
 static const YY_CHAR yy_ec[256] =
@@ -646,96 +646,99 @@ static const YY_CHAR yy_meta[54] =
         1,    1,    1
     } ;
 
-static const flex_int16_t yy_base[171] =
+static const flex_int16_t yy_base[178] =
     {   0,
         0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
-       51,   52,  319,  320,   57,   59,  296,  320,    0,  320,
-      295,  320,  320,  294,  293,  292,   47,   47,   50,  291,
-      290,  289,  293,    0,  290,   48,   51,   37,   52,   53,
-       54,   55,   59,   56,   63,   57,   68,   71,  286,    0,
-        0,  320,   75,   89,  320,    0,  320,  320,  320,  320,
-      320,   87,    0,  285,  320,   92,   95,  103,  320,  320,
-      320,  289,    0,  286,  285,   74,  101,  284,   89,   81,
-       93,   87,  108,  113,  283,  116,  114,  118,  119,  282,
-      120,  121,  122,  320,    0,  271,  320,  270,    0,  320,
+       51,   52,  329,  330,   57,   59,  306,  330,    0,  296,
+      304,  330,  330,  303,  302,  330,  301,   47,   47,   50,
+      300,  299,  298,  302,    0,  299,   48,   37,   52,   51,
+       53,   54,   60,   56,   55,   59,   57,   63,  295,    0,
+        0,  330,   75,   87,  330,    0,  297,   73,  330,  330,
+      330,  330,  330,   89,    0,  293,  330,   90,   94,  100,
+      330,  330,  330,  297,    0,  294,  291,   87,  287,   92,
+       81,   95,  100,  101,  104,  283,  108,  112,  115,  114,
+      280,  116,  118,  119,  330,    0,  266,  330,  263,  271,
 
-      126,  280,  279,  320,    0,  123,  277,  126,  130,  274,
-      128,  127,  270,  133,  131,  137,  141,  147,  149,  151,
-      266,  161,  253,  259,  154,  155,  161,  256,  254,  157,
-      160,  162,  163,  164,  166,  242,  227,  171,  233,  228,
-      167,  165,  168,  226,  172,  173,  213,  188,  174,  221,
-      178,  211,  199,  198,  197,  196,  320,  219,  228,  234,
-      239,  244,  253,  262,  267,  272,  274,  279,  283,  287
+      268,  121,    0,  330,  125,  255,  250,  330,    0,  245,
+      123,  122,  240,  126,  138,  238,  139,  140,  125,  141,
+      145,  146,  148,  235,  158,  223,    0,  152,  229,  153,
+      151,  220,  208,  154,  157,  159,  160,  161,  163,  207,
+      196,  164,  204,  203,  166,  162,  169,  202,  173,  180,
+      191,  186,  168,  200,  192,  199,  196,  193,  195,  194,
+      171,  330,  228,  237,  240,  246,  251,  256,  265,  274,
+      279,  284,  289,  291,  296,  300,  304
     } ;
 
-static const flex_int16_t yy_def[171] =
+static const flex_int16_t yy_def[178] =
     {   0,
-      157,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-      158,  158,  157,  157,  157,  157,  157,  157,  159,  157,
-      157,  157,  157,  157,  157,  157,  160,  157,  157,  157,
-      157,  157,  157,  161,  162,  162,  162,  162,  162,  162,
-      162,  162,  162,  162,  162,  162,  162,  162,  157,  163,
-      163,  157,  164,  157,  157,  159,  157,  157,  157,  157,
-      157,  157,  165,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  161,  162,  157,  162,  162,  162,  162,  162,
-      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
-      162,  162,  162,  157,  163,  157,  157,  166,  165,  157,
+      162,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+      163,  163,  162,  162,  162,  162,  162,  162,  164,  165,
+      162,  162,  162,  162,  162,  162,  162,  166,  162,  162,
+      162,  162,  162,  162,  167,  168,  168,  168,  168,  168,
+      168,  168,  168,  168,  168,  168,  168,  168,  162,  169,
+      169,  162,  170,  162,  162,  164,  171,  171,  162,  162,
+      162,  162,  162,  162,  172,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  167,  168,  162,  168,  168,  168,
+      168,  168,  168,  168,  168,  168,  168,  168,  168,  168,
+      168,  168,  168,  168,  162,  169,  162,  162,  173,  171,
 
-      157,  157,  157,  157,  167,  162,  162,  162,  162,  162,
-      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
-      162,  164,  168,  162,  162,  162,  162,  162,  162,  162,
-      162,  162,  162,  162,  162,  162,  169,  162,  162,  162,
-      162,  162,  162,  162,  162,  162,  170,  162,  162,  162,
-      162,  162,  162,  162,  162,  162,    0,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157
+      162,  171,  172,  162,  162,  162,  162,  162,  174,  168,
+      168,  168,  168,  168,  168,  168,  168,  168,  168,  168,
+      168,  168,  168,  168,  170,  175,  165,  171,  168,  168,
+      168,  168,  168,  168,  168,  168,  168,  168,  168,  168,
+      176,  171,  168,  168,  168,  168,  168,  168,  168,  168,
+      177,  171,  168,  168,  168,  168,  168,  171,  168,  168,
+      171,    0,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162
     } ;
 
-static const flex_int16_t yy_nxt[374] =
+static const flex_int16_t yy_nxt[384] =
     {   0,
        14,   15,   16,   14,   17,   18,   19,   20,   21,   22,
-       23,   24,   25,   20,   26,   27,   28,   29,   20,   20,
-       30,   31,   32,   33,   34,   35,   35,   22,   14,   23,
-       36,   37,   38,   39,   40,   41,   42,   35,   43,   35,
-       44,   45,   35,   46,   35,   47,   35,   48,   35,   35,
-       22,   49,   23,   51,   51,   75,   52,   52,   54,   54,
-       54,   54,   61,   64,   62,   66,   75,   67,   65,   75,
-       75,   75,   75,   75,   75,   75,   68,   75,   76,   53,
-       53,   75,   79,   80,   97,   68,   75,   88,   81,   75,
-       54,   54,   75,   77,   82,   85,   83,   78,   84,   75,
+       23,   24,   25,   26,   27,   28,   29,   30,   26,   26,
+       31,   32,   33,   34,   35,   36,   36,   22,   14,   23,
+       36,   37,   38,   39,   40,   41,   42,   36,   43,   36,
+       44,   45,   36,   46,   36,   47,   36,   48,   36,   36,
+       22,   49,   23,   51,   51,   77,   52,   52,   54,   54,
+       54,   54,   63,   66,   64,   68,   77,   69,   67,   77,
+       77,   77,   77,   77,   77,   77,   70,   77,   77,   53,
+       53,   77,   80,   81,   98,   70,   82,   89,   54,   54,
+       78,  101,   92,   83,   79,   84,   86,   85,   90,   77,
 
-       86,   87,   90,   91,   62,   75,   89,   75,   92,  101,
-       66,   75,   67,   68,  106,  102,   93,  102,   68,   75,
-      103,   68,   68,   98,  108,  111,   75,   68,  109,  110,
-       68,   75,   75,  112,   75,  107,   75,   75,   75,   75,
-       75,   75,  113,  101,   75,   75,   75,  116,   75,   75,
-      117,   75,   68,  118,  119,   75,  120,  126,  114,   75,
-      115,   68,  129,  127,  128,   75,  125,   75,  130,   75,
-      157,  121,   75,   75,  131,   75,  133,  132,   75,   75,
-       75,   75,   75,   75,   75,   75,   75,  138,  141,   75,
-       75,   75,   75,  136,  139,  134,   75,  135,  140,  146,
+       93,   87,   88,  102,   91,   77,   64,  105,   94,   68,
+       77,   69,  106,   77,  106,   70,   70,  107,   77,   77,
+       70,  110,   77,   99,   70,   70,   77,  111,  112,   70,
+       77,  113,   77,   77,   77,  116,   77,   77,  114,  101,
+       77,   77,  105,   77,   77,  119,  115,  120,  121,  117,
+      122,   70,  118,  123,  130,  131,   77,   77,   77,   77,
+       70,  128,  132,   77,   77,  136,   77,  162,  124,   77,
+      101,   77,   77,  133,  134,   77,  137,   77,   77,   77,
+       77,   77,  101,  135,   77,  145,   77,   77,  144,  101,
+      140,   77,  143,  138,  139,  142,  150,  152,   77,  153,
 
-      149,  148,  151,  144,  145,  142,   75,  152,  153,   98,
-      143,  155,  150,  156,   75,   75,   75,   75,  154,   50,
-       50,   50,   50,   50,   50,   50,   50,   50,   56,   75,
-       56,   56,   56,   56,   56,   56,   56,   63,   63,   75,
-       63,  122,   63,   73,   75,   73,   75,   73,   74,   74,
-       74,   75,   74,   95,   95,  122,   95,   95,   95,   95,
-       75,   95,   96,   96,   96,   96,   96,   96,   96,   96,
-       96,   99,   75,   99,   75,   99,  123,   75,  123,  123,
-      124,  122,  124,  137,   75,  137,  137,  147,   75,  147,
-      147,   96,   75,   96,   96,   75,  103,  103,  122,  122,
+      148,  149,  146,  155,  101,  159,   99,  147,  156,  154,
+       77,  101,   77,   77,   77,  157,  158,   77,   77,  125,
+       77,   77,   77,  161,  125,   77,   77,  160,   50,   50,
+       50,   50,   50,   50,   50,   50,   50,   56,   77,   56,
+       56,   56,   56,   56,   56,   56,   57,   77,   57,   65,
+       65,  125,   65,   77,   65,   75,   77,   75,   77,   75,
+       76,   76,   76,   77,   76,   96,   96,  107,   96,   96,
+       96,   96,  107,   96,   97,   97,   97,   97,   97,   97,
+       97,   97,   97,  100,  100,  100,  127,  100,  103,  101,
+      103,  125,  103,  126,  125,  126,  126,  129,   77,  129,
 
-       75,   75,   75,  105,   75,  104,  100,   94,   75,   72,
-       71,   70,   69,   60,   59,   58,   57,   55,  157,   13,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157
+      141,   77,  141,  141,  151,   77,  151,  151,   97,  109,
+       97,   97,   77,  108,  104,  101,   95,   77,   74,   73,
+       72,   71,   62,   61,   60,   59,   58,   55,  162,   13,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162
     } ;
 
-static const flex_int16_t yy_chk[374] =
+static const flex_int16_t yy_chk[384] =
     {   0,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
@@ -743,41 +746,42 @@ static const flex_int16_t yy_chk[374] =
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,   11,   12,   38,   11,   12,   15,   15,
-       16,   16,   27,   28,   27,   29,   36,   29,   28,   37,
-       39,   40,   41,   42,   44,   46,   29,   43,   36,   11,
-       12,   45,   38,   39,   53,   29,   47,   44,   40,   48,
-       54,   54,   76,   37,   41,   43,   41,   37,   42,   80,
+       16,   16,   28,   29,   28,   30,   37,   30,   29,   40,
+       39,   41,   42,   45,   44,   47,   30,   46,   43,   11,
+       12,   48,   38,   39,   53,   30,   40,   44,   54,   54,
+       37,   58,   47,   41,   37,   41,   43,   42,   45,   81,
 
-       43,   43,   46,   47,   62,   82,   45,   79,   48,   66,
-       67,   81,   67,   62,   76,   68,   48,   68,   66,   77,
-       68,   67,   62,   53,   79,   82,   83,   66,   80,   81,
-       67,   84,   87,   82,   86,   77,   88,   89,   91,   92,
-       93,  106,   83,  101,  108,  112,  111,   87,  109,  115,
-       88,  114,  101,   89,   91,  116,   92,  108,   84,  117,
-       86,  101,  112,  109,  111,  118,  106,  119,  114,  120,
-      122,   93,  125,  126,  115,  130,  117,  116,  131,  127,
-      132,  133,  134,  142,  135,  141,  143,  125,  130,  138,
-      145,  146,  149,  120,  126,  118,  151,  119,  127,  135,
+       48,   43,   43,   58,   46,   78,   64,   68,   48,   69,
+       80,   69,   70,   82,   70,   64,   68,   70,   83,   84,
+       69,   78,   85,   53,   64,   68,   87,   80,   81,   69,
+       88,   82,   90,   89,   92,   84,   93,   94,   83,  102,
+      112,  111,  105,  119,  114,   88,   83,   89,   90,   85,
+       92,  105,   87,   93,  111,  112,  115,  117,  118,  120,
+      105,  102,  114,  121,  122,  119,  123,  125,   94,  131,
+      128,  130,  134,  115,  117,  135,  120,  136,  137,  138,
+      146,  139,  142,  118,  145,  134,  153,  147,  131,  161,
+      123,  149,  130,  121,  122,  128,  139,  142,  150,  145,
 
-      141,  138,  143,  133,  134,  131,  148,  145,  146,  122,
-      132,  149,  142,  151,  156,  155,  154,  153,  148,  158,
-      158,  158,  158,  158,  158,  158,  158,  158,  159,  152,
-      159,  159,  159,  159,  159,  159,  159,  160,  160,  150,
-      160,  147,  160,  161,  144,  161,  140,  161,  162,  162,
-      162,  139,  162,  163,  163,  137,  163,  163,  163,  163,
-      136,  163,  164,  164,  164,  164,  164,  164,  164,  164,
-      164,  165,  129,  165,  128,  165,  166,  124,  166,  166,
-      167,  123,  167,  168,  121,  168,  168,  169,  113,  169,
-      169,  170,  110,  170,  170,  107,  103,  102,   98,   96,
+      137,  138,  135,  147,  152,  153,  125,  136,  149,  146,
+      155,  158,  160,  159,  157,  150,  152,  156,  154,  151,
+      148,  144,  143,  158,  141,  140,  133,  155,  163,  163,
+      163,  163,  163,  163,  163,  163,  163,  164,  132,  164,
+      164,  164,  164,  164,  164,  164,  165,  129,  165,  166,
+      166,  126,  166,  124,  166,  167,  116,  167,  113,  167,
+      168,  168,  168,  110,  168,  169,  169,  107,  169,  169,
+      169,  169,  106,  169,  170,  170,  170,  170,  170,  170,
+      170,  170,  170,  171,  171,  171,  101,  171,  172,  100,
+      172,   99,  172,  173,   97,  173,  173,  174,   91,  174,
 
-       90,   85,   78,   75,   74,   72,   64,   49,   35,   33,
-       32,   31,   30,   26,   25,   24,   21,   17,   13,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157,  157,  157,  157,  157,  157,  157,  157,
-      157,  157,  157
+      175,   86,  175,  175,  176,   79,  176,  176,  177,   77,
+      177,  177,   76,   74,   66,   57,   49,   36,   34,   33,
+       32,   31,   27,   25,   24,   21,   20,   17,   13,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162,  162,  162,  162,  162,  162,  162,  162,
+      162,  162,  162
     } ;
 
 /* The intent behind this definition is that it'll catch
@@ -804,14 +808,14 @@ struct lexer_param;
     yyset_extra(yylloc->end, yyscanner);         \
   } while (0);
 
-#line 807 "src/lexer.c"
+#line 811 "src/lexer.c"
 
 #line 25 "src/lexer.l"
   static int enter(int opening, int state, yyscan_t yyscanner);
   static int try_exit(int closing, int state, yyscan_t yyscanner);
-#line 812 "src/lexer.c"
+#line 816 "src/lexer.c"
 #define YY_NO_INPUT 1
-#line 814 "src/lexer.c"
+#line 818 "src/lexer.c"
 
 #define INITIAL 0
 #define IN_PAREN 1
@@ -843,8 +847,8 @@ struct yyguts_t
     size_t yy_buffer_stack_max; /**< capacity of stack. */
     YY_BUFFER_STATE * yy_buffer_stack; /**< Stack as an array. */
     char yy_hold_char;
-    yy_size_t yy_n_chars;
-    yy_size_t yyleng_r;
+    int yy_n_chars;
+    int yyleng_r;
     char *yy_c_buf_p;
     int yy_init;
     int yy_start;
@@ -901,7 +905,7 @@ FILE *yyget_out ( yyscan_t yyscanner );
 
 void yyset_out  ( FILE * _out_str , yyscan_t yyscanner );
 
-			yy_size_t yyget_leng ( yyscan_t yyscanner );
+			int yyget_leng ( yyscan_t yyscanner );
 
 char *yyget_text ( yyscan_t yyscanner );
 
@@ -986,7 +990,7 @@ static int input ( yyscan_t yyscanner );
 	if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
 		{ \
 		int c = '*'; \
-		yy_size_t n; \
+		int n; \
 		for ( n = 0; n < max_size && \
 			     (c = getc( yyin )) != EOF && c != '\n'; ++n ) \
 			buf[n] = (char) c; \
@@ -1105,7 +1109,7 @@ YY_DECL
 #line 38 "src/lexer.l"
 
 
-#line 1108 "src/lexer.c"
+#line 1112 "src/lexer.c"
 
 	while ( /*CONSTCOND*/1 )		/* loops until end-of-file is reached */
 		{
@@ -1132,13 +1136,13 @@ yy_match:
 			while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 				{
 				yy_current_state = (int) yy_def[yy_current_state];
-				if ( yy_current_state >= 158 )
+				if ( yy_current_state >= 163 )
 					yy_c = yy_meta[yy_c];
 				}
 			yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
 			++yy_cp;
 			}
-		while ( yy_base[yy_current_state] != 320 );
+		while ( yy_base[yy_current_state] != 330 );
 
 yy_find_action:
 		yy_act = yy_accept[yy_current_state];
@@ -1429,22 +1433,27 @@ YY_RULE_SETUP
 { yylval->literal = jv_string(yytext+1); return FIELD;}
 	YY_BREAK
 case 48:
-/* rule 48 can match eol */
 YY_RULE_SETUP
-#line 126 "src/lexer.l"
-{}
+#line 125 "src/lexer.l"
+{ yylval->literal = jv_string(yytext+1); return BINDING;}
 	YY_BREAK
 case 49:
+/* rule 49 can match eol */
 YY_RULE_SETUP
-#line 128 "src/lexer.l"
-{ return INVALID_CHARACTER; }
+#line 127 "src/lexer.l"
+{}
 	YY_BREAK
 case 50:
 YY_RULE_SETUP
-#line 130 "src/lexer.l"
+#line 129 "src/lexer.l"
+{ return INVALID_CHARACTER; }
+	YY_BREAK
+case 51:
+YY_RULE_SETUP
+#line 131 "src/lexer.l"
 YY_FATAL_ERROR( "flex scanner jammed" );
 	YY_BREAK
-#line 1447 "src/lexer.c"
+#line 1456 "src/lexer.c"
 case YY_STATE_EOF(INITIAL):
 case YY_STATE_EOF(IN_PAREN):
 case YY_STATE_EOF(IN_BRACKET):
@@ -1637,7 +1646,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 
 	else
 		{
-			yy_size_t num_to_read =
+			int num_to_read =
 			YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1;
 
 		while ( num_to_read <= 0 )
@@ -1651,7 +1660,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 
 			if ( b->yy_is_our_buffer )
 				{
-				yy_size_t new_size = b->yy_buf_size * 2;
+				int new_size = b->yy_buf_size * 2;
 
 				if ( new_size <= 0 )
 					b->yy_buf_size += b->yy_buf_size / 8;
@@ -1709,7 +1718,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 
 	if ((yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) {
 		/* Extend the array by 50%, plus the number we really need. */
-		yy_size_t new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1);
+		int new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1);
 		YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc(
 			(void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size , yyscanner );
 		if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf )
@@ -1748,7 +1757,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 		while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 			{
 			yy_current_state = (int) yy_def[yy_current_state];
-			if ( yy_current_state >= 158 )
+			if ( yy_current_state >= 163 )
 				yy_c = yy_meta[yy_c];
 			}
 		yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
@@ -1777,11 +1786,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 	while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 		{
 		yy_current_state = (int) yy_def[yy_current_state];
-		if ( yy_current_state >= 158 )
+		if ( yy_current_state >= 163 )
 			yy_c = yy_meta[yy_c];
 		}
 	yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
-	yy_is_jam = (yy_current_state == 157);
+	yy_is_jam = (yy_current_state == 162);
 
 	(void)yyg;
 	return yy_is_jam ? 0 : yy_current_state;
@@ -1816,7 +1825,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 
 		else
 			{ /* need more input */
-			yy_size_t offset = yyg->yy_c_buf_p - yyg->yytext_ptr;
+			int offset = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr);
 			++yyg->yy_c_buf_p;
 
 			switch ( yy_get_next_buffer( yyscanner ) )
@@ -2194,12 +2203,12 @@ YY_BUFFER_STATE yy_scan_string (const char * yystr , yyscan_t yyscanner)
  * @param yyscanner The scanner object.
  * @return the newly allocated buffer state object.
  */
-YY_BUFFER_STATE yy_scan_bytes  (const char * yybytes, yy_size_t  _yybytes_len , yyscan_t yyscanner)
+YY_BUFFER_STATE yy_scan_bytes  (const char * yybytes, int  _yybytes_len , yyscan_t yyscanner)
 {
 	YY_BUFFER_STATE b;
 	char *buf;
 	yy_size_t n;
-	yy_size_t i;
+	int i;
     
 	/* Get memory for full buffer, including space for trailing EOB's. */
 	n = (yy_size_t) (_yybytes_len + 2);
@@ -2284,7 +2293,7 @@ static void yynoreturn yy_fatal_error (const char* msg , yyscan_t yyscanner)
 	do \
 		{ \
 		/* Undo effects of setting up yytext. */ \
-        yy_size_t yyless_macro_arg = (n); \
+        int yyless_macro_arg = (n); \
         YY_LESS_LINENO(yyless_macro_arg);\
 		yytext[yyleng] = yyg->yy_hold_char; \
 		yyg->yy_c_buf_p = yytext + yyless_macro_arg; \
@@ -2352,7 +2361,7 @@ FILE *yyget_out  (yyscan_t yyscanner)
 /** Get the length of the current token.
  * @param yyscanner The scanner object.
  */
-yy_size_t yyget_leng  (yyscan_t yyscanner)
+int yyget_leng  (yyscan_t yyscanner)
 {
     struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
     return yyleng;
@@ -2616,7 +2625,7 @@ static int yy_flex_strlen (const char * s , yyscan_t yyscanner)
 
 #define YYTABLES_NAME "yytables"
 
-#line 130 "src/lexer.l"
+#line 131 "src/lexer.l"
 
 /* perhaps these should be calls... */
 /*

src/lexer.h

@@ -285,7 +285,6 @@ typedef int16_t flex_int16_t;
 typedef uint16_t flex_uint16_t;
 typedef int32_t flex_int32_t;
 typedef uint32_t flex_uint32_t;
-typedef uint64_t flex_uint64_t;
 #else
 typedef signed char flex_int8_t;
 typedef short int flex_int16_t;
@@ -399,7 +398,7 @@ struct yy_buffer_state
 	/* Number of characters read into yy_ch_buf, not including EOB
 	 * characters.
 	 */
-	yy_size_t yy_n_chars;
+	int yy_n_chars;
 
 	/* Whether we "own" the buffer - i.e., we know we created it,
 	 * and can realloc() it to grow it, and should free() it to
@@ -443,7 +442,7 @@ void yypop_buffer_state ( yyscan_t yyscanner );
 
 YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size , yyscan_t yyscanner );
 YY_BUFFER_STATE yy_scan_string ( const char *yy_str , yyscan_t yyscanner );
-YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, yy_size_t len , yyscan_t yyscanner );
+YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len , yyscan_t yyscanner );
 
 void *yyalloc ( yy_size_t , yyscan_t yyscanner );
 void *yyrealloc ( void *, yy_size_t , yyscan_t yyscanner );
@@ -501,7 +500,7 @@ FILE *yyget_out ( yyscan_t yyscanner );
 
 void yyset_out  ( FILE * _out_str , yyscan_t yyscanner );
 
-			yy_size_t yyget_leng ( yyscan_t yyscanner );
+			int yyget_leng ( yyscan_t yyscanner );
 
 char *yyget_text ( yyscan_t yyscanner );
 
@@ -732,9 +731,9 @@ extern int yylex \
 #undef yyTABLES_NAME
 #endif
 
-#line 130 "src/lexer.l"
+#line 131 "src/lexer.l"
 
 
-#line 738 "src/lexer.h"
+#line 737 "src/lexer.h"
 #undef jq_yyIN_HEADER
 #endif /* jq_yyHEADER_H */

src/lexer.l

@@ -60,7 +60,7 @@ struct lexer_param;
 "catch" { return CATCH; }
 "label" { return LABEL; }
 "break" { return BREAK; }
-"__loc__" { return LOC; }
+"$__loc__" { return LOC; }
 "|=" { return SETPIPE; }
 "+=" { return SETPLUS; }
 "-=" { return SETMINUS; }
@@ -122,6 +122,7 @@ struct lexer_param;
 
 ([a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext); return IDENT;}
 \.[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext+1); return FIELD;}
+\$([a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext+1); return BINDING;}
 
 [ \n\t]+  {}
 

src/main.c

@@ -81,6 +81,7 @@ static void usage(int code, int keep_it_short) {
       "  -M               monochrome (don't colorize JSON);\n"
       "  -S               sort keys of objects on output;\n"
       "  --tab            use tabs for indentation;\n"
+      "  --strip-comments accept and strip comments from input;\n"
       "  --arg a v        set variable $a to value <v>;\n"
       "  --argjson a v    set variable $a to JSON value <v>;\n"
       "  --slurpfile a f  set variable $a to an array of JSON texts read from <f>;\n"
@@ -317,7 +318,12 @@ int main(int argc, char* argv[]) {
       if (further_args_are_strings) {
         ARGS = jv_array_append(ARGS, jv_string(argv[i]));
       } else if (further_args_are_json) {
-        ARGS = jv_array_append(ARGS, jv_parse(argv[i]));
+        jv v =  jv_parse(argv[i]);
+        if (!jv_is_valid(v)) {
+          fprintf(stderr, "%s: invalid JSON text passed to --jsonargs\n", progname);
+          die();
+        }
+        ARGS = jv_array_append(ARGS, v);
       } else {
         jq_util_input_add_input(input_state, argv[i]);
         nfiles++;
@@ -330,7 +336,12 @@ int main(int argc, char* argv[]) {
         if (further_args_are_strings) {
           ARGS = jv_array_append(ARGS, jv_string(argv[i]));
         } else if (further_args_are_json) {
-          ARGS = jv_array_append(ARGS, jv_parse(argv[i]));
+          jv v =  jv_parse(argv[i]);
+          if (!jv_is_valid(v)) {
+            fprintf(stderr, "%s: invalid JSON text passed to --jsonargs\n", progname);
+            die();
+          }
+          ARGS = jv_array_append(ARGS, v);
         } else {
           jq_util_input_add_input(input_state, argv[i]);
           nfiles++;
@@ -444,6 +455,10 @@ int main(int argc, char* argv[]) {
         parser_flags |= JV_PARSE_STREAMING;
         continue;
       }
+      if (isoption(argv[i], 0, "strip-comments", &short_opts)) {
+        parser_flags |= JV_PARSE_STRIP_COMMENTS;
+        continue;
+      }
       if (isoption(argv[i], 0, "stream-errors", &short_opts)) {
         parser_flags |= JV_PARSE_STREAM_ERRORS;
         continue;
@@ -592,7 +607,7 @@ int main(int argc, char* argv[]) {
     // Default search path list
     lib_search_paths = JV_ARRAY(jv_string("~/.jq"),
                                 jv_string("$ORIGIN/../lib/jq"),
-                                jv_string("$ORIGIN/lib"));
+                                jv_string("$ORIGIN/../lib"));
   }
   jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_search_paths);
 
@@ -682,18 +697,20 @@ int main(int argc, char* argv[]) {
         ret = process(jq, value, jq_flags, dumpopts);
         if (ret <= 0 && ret != JQ_OK_NO_OUTPUT)
           last_result = (ret != JQ_OK_NULL_KIND);
+        if (jq_halted(jq))
+          break;
         continue;
       }
 
       // Parse error
       jv msg = jv_invalid_get_msg(value);
       if (!(options & SEQ)) {
-        // --seq -> errors are not fatal
-        ret = JQ_OK_NO_OUTPUT;
+        ret = JQ_ERROR_UNKNOWN;
         fprintf(stderr, "jq: parse error: %s\n", jv_string_value(msg));
         jv_free(msg);
         break;
       }
+      // --seq -> errors are not fatal
       fprintf(stderr, "jq: ignoring parse error: %s\n", jv_string_value(msg));
       jv_free(msg);
     }

src/parser.h

@@ -76,48 +76,49 @@ struct lexer_param;
     INVALID_CHARACTER = 258,       /* INVALID_CHARACTER  */
     IDENT = 259,                   /* IDENT  */
     FIELD = 260,                   /* FIELD  */
-    LITERAL = 261,                 /* LITERAL  */
-    FORMAT = 262,                  /* FORMAT  */
-    REC = 263,                     /* ".."  */
-    SETMOD = 264,                  /* "%="  */
-    EQ = 265,                      /* "=="  */
-    NEQ = 266,                     /* "!="  */
-    DEFINEDOR = 267,               /* "//"  */
-    AS = 268,                      /* "as"  */
-    DEF = 269,                     /* "def"  */
-    MODULE = 270,                  /* "module"  */
-    IMPORT = 271,                  /* "import"  */
-    INCLUDE = 272,                 /* "include"  */
-    IF = 273,                      /* "if"  */
-    THEN = 274,                    /* "then"  */
-    ELSE = 275,                    /* "else"  */
-    ELSE_IF = 276,                 /* "elif"  */
-    REDUCE = 277,                  /* "reduce"  */
-    FOREACH = 278,                 /* "foreach"  */
-    END = 279,                     /* "end"  */
-    AND = 280,                     /* "and"  */
-    OR = 281,                      /* "or"  */
-    TRY = 282,                     /* "try"  */
-    CATCH = 283,                   /* "catch"  */
-    LABEL = 284,                   /* "label"  */
-    BREAK = 285,                   /* "break"  */
-    LOC = 286,                     /* "__loc__"  */
-    SETPIPE = 287,                 /* "|="  */
-    SETPLUS = 288,                 /* "+="  */
-    SETMINUS = 289,                /* "-="  */
-    SETMULT = 290,                 /* "*="  */
-    SETDIV = 291,                  /* "/="  */
-    SETDEFINEDOR = 292,            /* "//="  */
-    LESSEQ = 293,                  /* "<="  */
-    GREATEREQ = 294,               /* ">="  */
-    ALTERNATION = 295,             /* "?//"  */
-    QQSTRING_START = 296,          /* QQSTRING_START  */
-    QQSTRING_TEXT = 297,           /* QQSTRING_TEXT  */
-    QQSTRING_INTERP_START = 298,   /* QQSTRING_INTERP_START  */
-    QQSTRING_INTERP_END = 299,     /* QQSTRING_INTERP_END  */
-    QQSTRING_END = 300,            /* QQSTRING_END  */
-    FUNCDEF = 301,                 /* FUNCDEF  */
-    NONOPT = 302                   /* NONOPT  */
+    BINDING = 261,                 /* BINDING  */
+    LITERAL = 262,                 /* LITERAL  */
+    FORMAT = 263,                  /* FORMAT  */
+    REC = 264,                     /* ".."  */
+    SETMOD = 265,                  /* "%="  */
+    EQ = 266,                      /* "=="  */
+    NEQ = 267,                     /* "!="  */
+    DEFINEDOR = 268,               /* "//"  */
+    AS = 269,                      /* "as"  */
+    DEF = 270,                     /* "def"  */
+    MODULE = 271,                  /* "module"  */
+    IMPORT = 272,                  /* "import"  */
+    INCLUDE = 273,                 /* "include"  */
+    IF = 274,                      /* "if"  */
+    THEN = 275,                    /* "then"  */
+    ELSE = 276,                    /* "else"  */
+    ELSE_IF = 277,                 /* "elif"  */
+    REDUCE = 278,                  /* "reduce"  */
+    FOREACH = 279,                 /* "foreach"  */
+    END = 280,                     /* "end"  */
+    AND = 281,                     /* "and"  */
+    OR = 282,                      /* "or"  */
+    TRY = 283,                     /* "try"  */
+    CATCH = 284,                   /* "catch"  */
+    LABEL = 285,                   /* "label"  */
+    BREAK = 286,                   /* "break"  */
+    LOC = 287,                     /* "$__loc__"  */
+    SETPIPE = 288,                 /* "|="  */
+    SETPLUS = 289,                 /* "+="  */
+    SETMINUS = 290,                /* "-="  */
+    SETMULT = 291,                 /* "*="  */
+    SETDIV = 292,                  /* "/="  */
+    SETDEFINEDOR = 293,            /* "//="  */
+    LESSEQ = 294,                  /* "<="  */
+    GREATEREQ = 295,               /* ">="  */
+    ALTERNATION = 296,             /* "?//"  */
+    QQSTRING_START = 297,          /* QQSTRING_START  */
+    QQSTRING_TEXT = 298,           /* QQSTRING_TEXT  */
+    QQSTRING_INTERP_START = 299,   /* QQSTRING_INTERP_START  */
+    QQSTRING_INTERP_END = 300,     /* QQSTRING_INTERP_END  */
+    QQSTRING_END = 301,            /* QQSTRING_END  */
+    FUNCDEF = 302,                 /* FUNCDEF  */
+    NONOPT = 303                   /* NONOPT  */
   };
   typedef enum yytokentype yytoken_kind_t;
 #endif
@@ -129,48 +130,49 @@ struct lexer_param;
 #define INVALID_CHARACTER 258
 #define IDENT 259
 #define FIELD 260
-#define LITERAL 261
-#define FORMAT 262
-#define REC 263
-#define SETMOD 264
-#define EQ 265
-#define NEQ 266
-#define DEFINEDOR 267
-#define AS 268
-#define DEF 269
-#define MODULE 270
-#define IMPORT 271
-#define INCLUDE 272
-#define IF 273
-#define THEN 274
-#define ELSE 275
-#define ELSE_IF 276
-#define REDUCE 277
-#define FOREACH 278
-#define END 279
-#define AND 280
-#define OR 281
-#define TRY 282
-#define CATCH 283
-#define LABEL 284
-#define BREAK 285
-#define LOC 286
-#define SETPIPE 287
-#define SETPLUS 288
-#define SETMINUS 289
-#define SETMULT 290
-#define SETDIV 291
-#define SETDEFINEDOR 292
-#define LESSEQ 293
-#define GREATEREQ 294
-#define ALTERNATION 295
-#define QQSTRING_START 296
-#define QQSTRING_TEXT 297
-#define QQSTRING_INTERP_START 298
-#define QQSTRING_INTERP_END 299
-#define QQSTRING_END 300
-#define FUNCDEF 301
-#define NONOPT 302
+#define BINDING 261
+#define LITERAL 262
+#define FORMAT 263
+#define REC 264
+#define SETMOD 265
+#define EQ 266
+#define NEQ 267
+#define DEFINEDOR 268
+#define AS 269
+#define DEF 270
+#define MODULE 271
+#define IMPORT 272
+#define INCLUDE 273
+#define IF 274
+#define THEN 275
+#define ELSE 276
+#define ELSE_IF 277
+#define REDUCE 278
+#define FOREACH 279
+#define END 280
+#define AND 281
+#define OR 282
+#define TRY 283
+#define CATCH 284
+#define LABEL 285
+#define BREAK 286
+#define LOC 287
+#define SETPIPE 288
+#define SETPLUS 289
+#define SETMINUS 290
+#define SETMULT 291
+#define SETDIV 292
+#define SETDEFINEDOR 293
+#define LESSEQ 294
+#define GREATEREQ 295
+#define ALTERNATION 296
+#define QQSTRING_START 297
+#define QQSTRING_TEXT 298
+#define QQSTRING_INTERP_START 299
+#define QQSTRING_INTERP_END 300
+#define QQSTRING_END 301
+#define FUNCDEF 302
+#define NONOPT 303
 
 /* Value type.  */
 #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
@@ -181,7 +183,7 @@ union YYSTYPE
   jv literal;
   block blk;
 
-#line 185 "src/parser.h"
+#line 187 "src/parser.h"
 
 };
 typedef union YYSTYPE YYSTYPE;

src/parser.y

@@ -49,6 +49,7 @@ struct lexer_param;
 %token INVALID_CHARACTER
 %token <literal> IDENT
 %token <literal> FIELD
+%token <literal> BINDING
 %token <literal> LITERAL
 %token <literal> FORMAT
 %token REC ".."
@@ -74,7 +75,7 @@ struct lexer_param;
 %token CATCH "catch"
 %token LABEL "label"
 %token BREAK "break"
-%token LOC "__loc__"
+%token LOC "$__loc__"
 %token SETPIPE "|="
 %token SETPLUS "+="
 %token SETMINUS "-="
@@ -367,9 +368,6 @@ Term "as" Patterns '|' Exp {
 "if" Exp "then" Exp ElseBody {
   $$ = gen_cond($2, $4, $5);
 } |
-"if" Exp "then" Exp "end" {
-  $$ = gen_cond($2, $4, gen_noop());
-} |
 "if" Exp "then" error {
   FAIL(@$, "Possibly unterminated 'if' statement");
   $$ = $2;
@@ -388,10 +386,10 @@ Term "as" Patterns '|' Exp {
   $$ = $2;
 } |
 
-"label" '$' IDENT '|' Exp {
-  jv v = jv_string_fmt("*label-%s", jv_string_value($3));
-  $$ = gen_location(@$, locations, gen_label(jv_string_value(v), $5));
-  jv_free($3);
+"label" BINDING '|' Exp {
+  jv v = jv_string_fmt("*label-%s", jv_string_value($2));
+  $$ = gen_location(@$, locations, gen_label(jv_string_value(v), $4));
+  jv_free($2);
   jv_free(v);
 } |
 
@@ -528,13 +526,13 @@ ImportWhat Exp ';' {
 }
 
 ImportWhat:
-"import" ImportFrom "as" '$' IDENT {
+"import" ImportFrom "as" BINDING {
   jv v = block_const($2);
   // XXX Make gen_import take only blocks and the int is_data so we
   // don't have to free so much stuff here
-  $$ = gen_import(jv_string_value(v), jv_string_value($5), 1);
+  $$ = gen_import(jv_string_value(v), jv_string_value($4), 1);
   block_free($2);
-  jv_free($5);
+  jv_free($4);
   jv_free(v);
 } |
 "import" ImportFrom "as" IDENT {
@@ -582,13 +580,9 @@ Params ';' Param {
 }
 
 Param:
-'$' IDENT {
-  $$ = gen_param_regular(jv_string_value($2));
-  jv_free($2);
-} |
-'$' Keyword {
-  $$ = gen_param_regular(jv_string_value($2));
-  jv_free($2);
+BINDING {
+  $$ = gen_param_regular(jv_string_value($1));
+  jv_free($1);
 } |
 IDENT {
   $$ = gen_param(jv_string_value($1));
@@ -625,6 +619,9 @@ ElseBody:
 } |
 "else" Exp "end" {
   $$ = $2;
+} |
+"end" {
+  $$ = gen_noop();
 }
 
 ExpD:
@@ -646,13 +643,13 @@ Term:
 REC {
   $$ = gen_call("recurse", gen_noop());
 } |
-BREAK '$' IDENT {
-  jv v = jv_string_fmt("*label-%s", jv_string_value($3));     // impossible symbol
+BREAK BINDING {
+  jv v = jv_string_fmt("*label-%s", jv_string_value($2));     // impossible symbol
   $$ = gen_location(@$, locations,
                     BLOCK(gen_op_unbound(LOADV, jv_string_value(v)),
                     gen_call("error", gen_noop())));
   jv_free(v);
-  jv_free($3);
+  jv_free($2);
 } |
 BREAK error {
   FAIL(@$, "break requires a label to break to");
@@ -710,6 +707,12 @@ Term '[' ']' '?' {
 Term '[' ']' %prec NONOPT {
   $$ = block_join($1, gen_op_simple(EACH));
 } |
+Term '.' '[' ']' '?' {
+  $$ = block_join($1, gen_op_simple(EACH_OPT));
+} |
+Term '.' '[' ']' %prec NONOPT {
+  $$ = block_join($1, gen_op_simple(EACH));
+} |
 Term '[' Exp ':' Exp ']' '?' {
   $$ = gen_slice_index($1, $3, $5, INDEX_OPT);
 } |
@@ -768,22 +771,17 @@ FORMAT {
  * DO NOT USE!!  I will break your jq code if you do use this outside
  * src/builtin.jq.
  */
-'$' '$' '$' '$' IDENT {
-  $$ = gen_location(@$, locations, gen_op_unbound(LOADVN, jv_string_value($5)));
-  jv_free($5);
+'$' '$' '$' BINDING {
+  $$ = gen_location(@$, locations, gen_op_unbound(LOADVN, jv_string_value($4)));
+  jv_free($4);
 } |
-'$' IDENT {
-  $$ = gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($2)));
-  jv_free($2);
+BINDING {
+  $$ = gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($1)));
+  jv_free($1);
 } |
-'$' Keyword {
-  if (strcmp(jv_string_value($2), "__loc__") == 0) {
-    $$ = gen_const(JV_OBJECT(jv_string("file"), jv_copy(locations->fname),
-                             jv_string("line"), jv_number(locfile_get_line(locations, @$.start) + 1)));
-  } else {
-    $$ = gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($2)));
-  }
-  jv_free($2);
+"$__loc__" {
+  $$ = gen_const(JV_OBJECT(jv_string("file"), jv_copy(locations->fname),
+                           jv_string("line"), jv_number(locfile_get_line(locations, @$.start) + 1)));
 } |
 IDENT {
   const char *s = jv_string_value($1);
@@ -837,9 +835,9 @@ Pattern {
 }
 
 Pattern:
-'$' IDENT {
-  $$ = gen_op_unbound(STOREV, jv_string_value($2));
-  jv_free($2);
+BINDING {
+  $$ = gen_op_unbound(STOREV, jv_string_value($1));
+  jv_free($1);
 } |
 '[' ArrayPats ']' {
   $$ = BLOCK($2, gen_op_simple(POP));
@@ -865,11 +863,11 @@ ObjPats ',' ObjPat {
 }
 
 ObjPat:
-'$' IDENT {
-  $$ = gen_object_matcher(gen_const($2), gen_op_unbound(STOREV, jv_string_value($2)));
+BINDING {
+  $$ = gen_object_matcher(gen_const($1), gen_op_unbound(STOREV, jv_string_value($1)));
 } |
-'$' IDENT ':' Pattern {
-  $$ = gen_object_matcher(gen_const($2), BLOCK(gen_op_simple(DUP), gen_op_unbound(STOREV, jv_string_value($2)), $4));
+BINDING ':' Pattern {
+  $$ = gen_object_matcher(gen_const($1), BLOCK(gen_op_simple(DUP), gen_op_unbound(STOREV, jv_string_value($1)), $3));
 } |
 IDENT ':' Pattern {
   $$ = gen_object_matcher(gen_const($1), $3);
@@ -947,9 +945,6 @@ Keyword:
 } |
 "break" {
   $$ = jv_string("break");
-} |
-"__loc__" {
-  $$ = jv_string("__loc__");
 }
 
 MkDict:
@@ -974,17 +969,13 @@ IDENT ':' ExpD {
   $$ = gen_dictpair($1, BLOCK(gen_op_simple(POP), gen_op_simple(DUP2),
                               gen_op_simple(DUP2), gen_op_simple(INDEX)));
   }
-| '$' IDENT ':' ExpD {
-  $$ = gen_dictpair(gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($2))),
-                    $4);
+| BINDING ':' ExpD {
+  $$ = gen_dictpair(gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($1))),
+                    $3);
   }
-| '$' IDENT {
-  $$ = gen_dictpair(gen_const($2),
-                    gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($2))));
-  }
-| '$' Keyword {
-  $$ = gen_dictpair(gen_const($2),
-                    gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($2))));
+| BINDING {
+  $$ = gen_dictpair(gen_const($1),
+                    gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($1))));
   }
 | IDENT {
   $$ = gen_dictpair(gen_const(jv_copy($1)),

tests/commenttest

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${0%/*}/setup" "$@"
+
+if [ "$(echo '{"hi":"there"/*comment*/}' | $VALGRIND $Q $JQ --strip-comments '.hi')" != '"there"' ]; then
+	echo "C-style comment test failed"
+	exit 1
+fi
+
+if [ "$(printf '{"hi"://comment\n"there"}' | $VALGRIND $Q $JQ --strip-comments '.hi')" != '"there"' ]; then
+	echo "C++-style comment test failed"
+	exit 1
+fi
+
+exit 0

tests/jq.test

@@ -173,6 +173,11 @@ jq: error: Cannot use number (0) as object key at <top-level>, line 1:
 [1,null,true,false,"abcdef",{},{"a":1,"b":2},[],[1,2,3,4,5],[1,2]]
 [null,"bc",[],[2,3],[2]]
 
+# chaining/suffix-list, with and without dot
+map(try .a[] catch ., try .a.[] catch ., .a[]?, .a.[]?)
+[{"a": [1,2]}, {"a": 123}]
+[1,2,1,2,1,2,1,2,"Cannot iterate over number (123)","Cannot iterate over number (123)"]
+
 #
 # Negative array indices
 #
@@ -319,9 +324,13 @@ null
 "badness"
 [1]
 
-[first(range(.)), last(range(.)), nth(0; range(.)), nth(5; range(.)), try nth(-1; range(.)) catch .]
+[first(range(.)), last(range(.))]
 10
-[0,9,0,5,"nth doesn't support negative indices"]
+[0,9]
+
+[nth(0,5,9,10,15; range(.)), try nth(-1; range(.)) catch .]
+10
+[0,5,9,"nth doesn't support negative indices"]
 
 # Check that first(g) does not extract more than one value from g
 first(1,error("foo"))
@@ -952,10 +961,6 @@ path(.a[path(.b)[0]])
 [1,[[],{"a":2}]]
 [[0],[1],[1,0],[1,1],[1,1,"a"]]
 
-[leaf_paths]
-[1,[[],{"a":2}]]
-[[0],[1,1,"a"]]
-
 ["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
 {"bar": 42, "foo": ["a", "b", "c", "d"]}
 "b"
@@ -1048,6 +1053,19 @@ def inc(x): x |= .+1; inc(.[].a)
 {"a":[{"b":5}]}
 {"a":[{"c":3,"b":5}]}
 
+# #2051, deletion using assigning empty against arrays
+(.[] | select(. >= 2)) |= empty
+[1,5,3,0,7]
+[1,0]
+
+.[] |= select(. % 2 == 0)
+[0,1,2,3,4,5]
+[0,2,4]
+
+.foo[1,4,2,3] |= empty
+{"foo":[0,1,2,3,4,5]}
+{"foo":[0,5]}
+
 .[2][3] = 1
 [4]
 [4, null, [null, null, null, 1]]
@@ -1116,6 +1134,13 @@ null
 7
 [7]
 
+[if false then 3 elif false then 4 end]
+7
+[7]
+
+[if false then 3 elif false then 4 else . end]
+7
+[7]
 
 [.[] | [.foo[] // .bar]]
 [{"foo":[1,2], "bar": 42}, {"foo":[1], "bar": null}, {"foo":[null,false,3], "bar": 18}, {"foo":[], "bar":42}, {"foo": [null,false,null], "bar": 41}]
@@ -1268,6 +1293,14 @@ indices(", ")
 ["a", "ab", "abc"]
 ["aaa", "ababab", "abcabcabc"]
 
+[.[] * "abc"]
+[-1.0, -0.5, 0.0, 0.5, 1.0, 1.5, 3.7, 10.0]
+[null,null,"","","abc","abc","abcabcabc","abcabcabcabcabcabcabcabcabcabc"]
+
+[. * (nan,-nan)]
+"abc"
+[null,null]
+
 [.[] / ","]
 ["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
 [["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]]
@@ -1571,6 +1604,11 @@ try (1/.) catch .
 0
 "number (1) and number (0) cannot be divided because the divisor is zero"
 
+0/0, (0 as $x | $x/0) | isnan
+0
+true
+true
+
 try (1%.) catch .
 0
 "number (1) and number (0) cannot be divided (remainder) because the divisor is zero"
@@ -1731,3 +1769,26 @@ false
 . |= try . catch .
 1
 1
+
+
+# Using a keyword as variable/label name
+
+123 as $label | $label
+null
+123
+
+[ label $if | range(10) | ., (select(. == 5) | break $if) ]
+null
+[0,1,2,3,4,5]
+
+reduce .[] as $then (4 as $else | $else; . as $elif | . + $then * $elif)
+[1,2,3]
+96
+
+1 as $foreach | 2 as $and | 3 as $or | { $foreach, $and, $or, a }
+{"a":4,"b":5}
+{"foreach":1,"and":2,"or":3,"a":4}
+
+[ foreach .[] as $try (1 as $catch | $catch - 1; . + $try; .) ]
+[10,9,8,7]
+[10,19,27,34]

tests/man.test

@@ -219,6 +219,14 @@ map_values(.+1)
 {"a": 1, "b": 2, "c": 3}
 {"a": 2, "b": 3, "c": 4}
 
+pick(.a, .b.c, .x)
+{"a": 1, "b": {"c": 2, "d": 3}, "e": 4}
+{"a":1,"b":{"c":2},"x":null}
+
+[1,2,3,4] | pick(.[2], .[0], .[0])
+[1,2,3,4]
+[1,null,3]
+
 path(.a[0].b)
 null
 ["a",0,"b"]
@@ -300,7 +308,7 @@ null
 [1,[[],{"a":2}]]
 [[0],[1],[1,0],[1,1],[1,1,"a"]]
 
-[paths(scalars)]
+[paths(type == "number")]
 [1,[[],{"a":2}]]
 [[0],[1,1,"a"]]
 
@@ -668,6 +676,14 @@ strptime("%Y-%m-%dT%H:%M:%SZ")|mktime
 "2015-03-05T23:51:47Z"
 1425599507
 
+. == false
+null
+false
+
+. == {"b": {"d": (4 + 1e-20), "c": 3}, "a":1}
+{"a":1, "b": {"c": 3, "d": 4}}
+true
+
 .[] == 1
 [1, 1.0, "1", "banana"]
 true

tests/onig.test

@@ -1,12 +1,16 @@
 # match builtin
 [match("( )*"; "g")]
 "abc"
-[{"offset":0, "length":0, "string":"", "captures":[]},{"offset":1, "length":0, "string":"", "captures":[]},{"offset":2, "length":0, "string":"", "captures":[]}]
+[{"offset":0, "length":0, "string":"", "captures":[]}, {"offset":1, "length":0, "string":"", "captures":[]}, {"offset":2, "length":0, "string":"", "captures":[]}, {"offset":3, "length":0, "string":"", "captures":[]}]
 
 [match("( )*"; "gn")]
 "abc"
 []
 
+[match(""; "g")]
+"ab"
+[{"offset":0,"length":0,"string":"","captures":[]},{"offset":1,"length":0,"string":"","captures":[]},{"offset":2,"length":0,"string":"","captures":[]}]
+
 [match("a"; "gi")]
 "āáàä"
 []
@@ -71,10 +75,47 @@ gsub("a";"b")
 "aaaaa"
 "bbbbb"
 
-gsub( "(.*)"; "";  "x")
+gsub("(.*)"; ""; "x")
 ""
 ""
 
+gsub(""; "a"; "g")
+""
+"a"
+
+gsub("^"; ""; "g")
+"a"
+"a"
+
+gsub(""; "a"; "g")
+"a"
+"aaa"
+
+gsub("$"; "a"; "g")
+"a"
+"aa"
+
+gsub("^"; "a")
+""
+"a"
+
+gsub("(?=u)"; "u")
+"qux"
+"quux"
+
+gsub("^.*a"; "b")
+"aaa"
+"b"
+
+gsub("^.*?a"; "b")
+"aaa"
+"baa"
+
+# The following is for regression testing and should not be construed as a requirement:
+[gsub("a"; "b", "c")]
+"a"
+["b","c"]
+
 [.[] | scan(", ")]
 ["a,b, c, d, e,f",", a,b, c, d, e,f, "]
 [", ",", ",", ",", ",", ",", ",", ",", "]
@@ -87,12 +128,42 @@ gsub( "(.*)"; "";  "x")
 ["a,b, c, d, e,f",", a,b, c, d, e,f, "]
 [[["a,b:c, d, e,f"],["a,b:c:d:e,f"],[", ",", ",", "]],[[":a,b, c, d, e,f, "],[":a,b:c:d:e,f:"],[", ",", ",", ",", ",", "]]]
 
+[.[] | scan("b+"; "i")]
+["","bBb","abcABBBCabbbc"]
+["bBb","b","BBB","bbb"]
+
 # reference to named captures
 gsub("(?<x>.)[^a]*"; "+\(.x)-")
 "Abcabc"
 "+A-+a-"
 
+gsub("(?<x>.)(?<y>[0-9])"; "\(.x|ascii_downcase)\(.y)")
+"A1 B2 CD"
+"a1 b2 CD"
+
+gsub("\\b(?<x>.)"; "\(.x|ascii_downcase)")
+"ABC DEF"
+"aBC dEF"
+
 # utf-8
 sub("(?<x>.)"; "\(.x)!")
 "’"
 "’!"
+
+[sub("a"; "b", "c")]
+"a"
+["b","c"]
+
+[sub("(?<a>.)"; "\(.a|ascii_upcase)", "\(.a|ascii_downcase)", "c")]
+"aB"
+["AB","aB","cB"]
+
+[gsub("(?<a>.)"; "\(.a|ascii_upcase)", "\(.a|ascii_downcase)", "c")]
+"aB"
+["AB","ab","cc"]
+
+# splits and _nwise
+[splits("")]
+"ab"
+["","a","b",""]
+

tests/shtest

@@ -143,6 +143,21 @@ if $VALGRIND $Q $JQ -e . $d/input; then
   exit 2
 fi
 
+# Regression test for #2146
+if echo "foobar" | $JQ .; then
+  printf 'Issue #2146 is back?\n' 1>&2
+  exit 1
+elif [ $? -ne 5 ]; then
+  echo "Invalid input had wrong error code" 1>&2
+  exit 1
+fi
+
+# Regression test for #2367; make sure to call jq twice
+if ! echo '{"a": 1E9999999999}' | $JQ . | $JQ -e .a; then
+  printf 'Issue #2367 is back?\n' 1>&2
+  exit 1
+fi
+
 # Regression test for #1534
 echo "[1,2,3,4]" > $d/expected
 printf "[1,2][3,4]" | $JQ -cs add > $d/out 2>&1
@@ -203,6 +218,23 @@ else
     echo "Not doing torture tests"
 fi
 
+## Regression test for issue #2378 assert when stream parse broken object pair
+echo '{"a":1,"b",' | $JQ --stream  > /dev/null 2> $d/err || true
+grep 'Objects must consist of key:value pairs' $d/err > /dev/null
+
+## Regression test for issue #2572 assert when using --jsonargs and invalid JSON
+$JQ -n --jsonargs null invalid && EC=$? || EC=$?
+if [ "$EC" -ne 2 ]; then
+    echo "--jsonargs exited with wrong exit code, expected 2 got $EC" 1>&2
+    exit 1
+fi
+# this tests the args_done code path "--"
+$JQ -n --jsonargs null -- invalid && EC=$? || EC=$?
+if [ "$EC" -ne 2 ]; then
+    echo "--jsonargs exited with wrong exit code, expected 2 got $EC" 1>&2
+    exit 1
+fi
+
 ## Fuzz parser
 
 ## XXX With a $(urandom) builtin we could move this test into tests/all.test