Some syntax tweaks for AIX

Now that #1313 is fixed, |= no longer outputs null when the RHS update
expression outputs empty.

When a user wants to keep the current value of the LHS they would have
the RHS update expression output `.`, so having `empty` achieve the same
thing would be redundant.  The obvious thing to do is to delete the LHS
when the RHS update outputs `empty` (i.e., doesn't output any values).

It's reasonable to think that existing programs won't be broken by this
change, because reduce and |= not handling empty well is clearly a bug.
(Though it's possible that some programs were using empty to quickly
terminate reduce or |=, it's not likely.  They should use label/break
instead.)

Prior to this change |= would use the _last_ value output by the RHS
update expression.  With this change |= will use the _first_ value
instead.  This change _is_ a minor backwards-incompatible change.  It
may or may not be acceptable; we'll see.  It is a useful change in that
it makes |= faster when the update expression produces multiple values.

.travis.yml

@@ -1,3 +1,7 @@
+os:
+    - linux
+    - osx
+
 sudo: false
 
 language: c
@@ -10,38 +14,54 @@ matrix:
     include:
         - compiler: gcc
           env: COVERAGE="--disable-valgrind --enable-gcov"
+          os: linux
 
 addons:
     apt:
         packages:
             - libonig-dev
             - valgrind
+            - bison
 
 before_install:
+    - echo "$TRAVIS_OS_NAME"
+    - uname -s
+    - brew update               || true;
+      brew install flex         || true;
+      brew install bison        || true;
+      brew install oniguruma    || true;
     - rm src/{lexer,parser}.{c,h}
 
 install:
     - bundle install --gemfile=docs/Gemfile
-
     - wget http://ftp.debian.org/debian/pool/main/b/bison/bison_3.0.2.dfsg-2_amd64.deb
     - ar p bison_3.0.2.dfsg-2_amd64.deb data.tar.xz | tar xJ
-
     - if [ -n "$COVERAGE" ]; then pip install --user cpp-coveralls; fi
 
 before_script:
+    # If this is OS X we'll get bison from brew, else we'll get bison
+    # from the .deb unpacked above in the install section.
+    - PATH=/usr/local/opt/bison/bin:$PWD/usr/bin:$PATH
+    - echo SHELL=$SHELL
+    - echo PATH=$PATH
+    - which bison
+    - bison --version
     - autoreconf -i
-    - ./configure
-        YACC="usr/bin/bison -y"
-        $COVERAGE
+    - ./configure YACC="$(which bison) -y" $COVERAGE
 
 script:
-    - make -j4 BISON_PKGDATADIR=$(pwd)/usr/share/bison
+    # When using the bison from Debian we need to tell that bison where
+    # to find its data.  Yay non-relocatable code.  Not.
+    - echo PATH=$PATH
+    - which bison
+    - make BISON_PKGDATADIR=$PWD/usr/share/bison src/parser.c || make src/parser.c
+    - make -j4
     - make check -j4
 
 after_script:
     - |
         if [ -n "$COVERAGE" ]; then
-            rm -rf src/.libs usr # don't care about coverage for libjq and bison
+            rm -rf src/.libs # don't care about coverage for libjq
             coveralls --gcov-options '\-lp' \
                 -e src/lexer.c -e src/parser.c -e src/jv_dtoa.c
         fi

AUTHORS

@@ -4,7 +4,6 @@ Stephen Dolan        <mu@netsoc.tcd.ie>
 Maintained by:
 Nicolas Williams     <nico@cryptonector.com>
 William Langford     <wlangfor@gmail.com>
-David Tolnay         <dtolnay@gmail.com>
 
 Contributions by:
 Aaron Peschel        <apeschel@zendesk.com>

KEYS

@@ -6,8 +6,3 @@ pub   2048R/D15684DB 2015-10-12 [expires: 2017-10-11]
       Key fingerprint = 7F6C 7BD3 0412 AFD5 8C1A  5007 EB26 A4F8 D156 84DB
 uid                  Nicolas Williams <nico@cryptonector.com>
 sub   2048R/9C9CCD6A 2015-10-12 [expires: 2017-10-11]
-
-pub   4096R/95FF6D82 2011-11-13
-      Key fingerprint = 4628 C5D8 2CFF 65D6 8164  A78A F9BA 143B 95FF 6D82
-uid                  David Tolnay <dtolnay@gmail.com>
-sub   4096R/9572BD05 2011-11-13

Makefile.am

@@ -115,7 +115,7 @@ endif
 
 ### Tests (make check)
 
-TESTS = tests/optionaltest tests/mantest tests/jqtest tests/onigtest tests/shtest
+TESTS = tests/optionaltest tests/mantest tests/jqtest tests/onigtest tests/shtest tests/utf8test
 TESTS_ENVIRONMENT = NO_VALGRIND=$(NO_VALGRIND)
 
 

README.md

@@ -15,7 +15,9 @@ You can also try it online at [jqplay.org](https://jqplay.org).
 
 If you want to hack on jq, feel free, but be warned that its internals
 are not well-documented at the moment. Bring a hard hat and a
-shovel.  Also, read the wiki: https://github.com/stedolan/jq/wiki
+shovel.  Also, read the wiki: https://github.com/stedolan/jq/wiki, where
+you will find cookbooks, discussion of advanced topics, internals,
+release engineering, and more.
 
 Source tarball and built executable releases can be found on the
 homepage and on the github release page, https://github.com/stedolan/jq/releases

configure.ac

@@ -179,29 +179,59 @@ AC_CHECK_MATH_FUNC(acos, [.5])
 AC_CHECK_MATH_FUNC(acosh, [.5])
 AC_CHECK_MATH_FUNC(asin, [.5])
 AC_CHECK_MATH_FUNC(asinh, [.5])
-AC_CHECK_MATH_FUNC(atan, [.5])
 AC_CHECK_MATH_FUNC(atan2, [.5,.5])
+AC_CHECK_MATH_FUNC(atan, [.5])
 AC_CHECK_MATH_FUNC(atanh, [.5])
 AC_CHECK_MATH_FUNC(cbrt, [.5])
+AC_CHECK_MATH_FUNC(ceil,[.5])
+AC_CHECK_MATH_FUNC(copysign,[.5,1.0])
 AC_CHECK_MATH_FUNC(cos, [.5])
 AC_CHECK_MATH_FUNC(cosh, [.5])
-AC_CHECK_MATH_FUNC(exp, [.5])
+AC_CHECK_MATH_FUNC(drem,[.5,1.0])
+AC_CHECK_MATH_FUNC(erf,[.5])
+AC_CHECK_MATH_FUNC(erfc,[.5])
+AC_CHECK_MATH_FUNC(exp10,[.5])
 AC_CHECK_MATH_FUNC(exp2, [.5])
+AC_CHECK_MATH_FUNC(exp, [.5])
+AC_CHECK_MATH_FUNC(expm1,[.5])
+AC_CHECK_MATH_FUNC(fabs,[.5])
+AC_CHECK_MATH_FUNC(fdim,[.5,1.0])
 AC_CHECK_MATH_FUNC(floor, [.5])
+AC_CHECK_MATH_FUNC(fma,[.5,1.0,1.5])
+AC_CHECK_MATH_FUNC(fmax,[.5,1.0])
+AC_CHECK_MATH_FUNC(fmin,[.5,1.0])
+AC_CHECK_MATH_FUNC(fmod,[.5,1.0])
+AC_CHECK_MATH_FUNC(frexp,[.5,1.0])
+AC_CHECK_MATH_FUNC(gamma,[.5])
 AC_CHECK_MATH_FUNC(hypot, [.5,.5])
 AC_CHECK_MATH_FUNC(j0, [.5])
 AC_CHECK_MATH_FUNC(j1, [.5])
-AC_CHECK_MATH_FUNC(log, [.5])
+AC_CHECK_MATH_FUNC(ldexp,[.5,2])
+AC_CHECK_MATH_FUNC(lgamma,[.5])
 AC_CHECK_MATH_FUNC(log10, [.5])
+AC_CHECK_MATH_FUNC(log1p,[.5])
 AC_CHECK_MATH_FUNC(log2, [.5])
+AC_CHECK_MATH_FUNC(log, [.5])
+AC_CHECK_MATH_FUNC(logb,[.5])
+AC_CHECK_MATH_FUNC(modf,[.5,1.0])
+AC_CHECK_MATH_FUNC(nearbyint,[.5])
+AC_CHECK_MATH_FUNC(nextafter,[.5,1.0])
+AC_CHECK_MATH_FUNC(nexttoward,[.5,1.0])
+AC_CHECK_MATH_FUNC(pow10,[.5])
 AC_CHECK_MATH_FUNC(pow, [2,2])
 AC_CHECK_MATH_FUNC(remainder, [3,2])
+AC_CHECK_MATH_FUNC(rint,[.5])
+AC_CHECK_MATH_FUNC(round,[.5])
+AC_CHECK_MATH_FUNC(scalb,[.5,1.0])
+AC_CHECK_MATH_FUNC(scalbln,[.5,2])
+AC_CHECK_MATH_FUNC(significand,[.5])
 AC_CHECK_MATH_FUNC(sin, [.5])
 AC_CHECK_MATH_FUNC(sinh, [.5])
 AC_CHECK_MATH_FUNC(sqrt, [.5])
 AC_CHECK_MATH_FUNC(tan, [.5])
 AC_CHECK_MATH_FUNC(tanh, [.5])
 AC_CHECK_MATH_FUNC(tgamma, [.5])
+AC_CHECK_MATH_FUNC(trunc,[.5])
 AC_CHECK_MATH_FUNC(y0, [.5])
 AC_CHECK_MATH_FUNC(y1, [.5])
 
@@ -230,4 +260,3 @@ AC_CONFIG_MACRO_DIR([config/m4])
 AC_CONFIG_FILES([Makefile])
 AC_OUTPUT
 
-

docs/content/2.download/default.yml

@@ -27,11 +27,9 @@ body:
        * jq 1.4 is in the official [openSUSE](https://software.opensuse.org/package/jq)
          repository. Install using `sudo zypper install jq`.
 
-       * For Arch users, a PKGBUILD is in the
-         [AUR](https://aur.archlinux.org/packages/jq-git/).
-         Refer to the
-         [ArchWiki](https://wiki.archlinux.org/index.php/Arch_User_Repository)
-         for how to install from AUR.
+       * jq 1.5 is in the official
+         [Arch](https://www.archlinux.org/packages/?sort=&q=jq&maintainer=&flagged=)
+         repository.  Install using `sudo pacman -Sy jq`.
 
        * jq 1.5 binaries for
          [64-bit](https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64)

docs/content/3.manual/manual.yml

@@ -107,7 +107,7 @@ sections:
         output and an ASCII LF (line feed) is printed after every
         output.  Input JSON texts that fail to parse are ignored (but
         warned about), discarding all subsequent input until the next
-        RS.  This more also parses the output of jq without the `--seq`
+        RS.  This mode also parses the output of jq without the `--seq`
         option.
 
       * `--stream`:
@@ -433,6 +433,7 @@ sections:
 
       - title: "`|`"
         body: |
+
           The | operator combines two filters by feeding the output(s) of
           the one on the left into the input of the one on the right. It's
           pretty much the same as the Unix shell's pipe, if you're used to
@@ -443,11 +444,29 @@ sections:
           expression `.[] | .foo` retrieves the "foo" field of each
           element of the input array.
 
+          Note that `.a.b.c` is the same as `.a | .b | .c`.
+
+          Note too that `.` is the input value at the particular stage
+          in a "pipeline", specifically: where the `.` expression appears.
+          Thus `.a | . | .b` is the same as `.a.b`, as the `.` in the
+          middle refers to whatever value `.a` produced.
+
         examples:
           - program: '.[] | .name'
             input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
             output: ['"JSON"', '"XML"']
 
+      - title: "Parenthesis"
+        body: |
+
+          Parenthesis work as a grouping operator just as in any typical
+          programming language.
+
+        examples:
+          - program: '(. + 2) * 5'
+            input: '1'
+            output: [15]
+
   - title: Types and Values
     body: |
 
@@ -723,7 +742,7 @@ sections:
       - title: "`in`"
         body: |
 
-          The builtin function `in` returns the input key is in the
+          The builtin function `in` returns whether or not the input key is in the
           given object, or the input index corresponds to an element
           in the given array. It is, essentially, an inversed version
           of `has`.
@@ -740,7 +759,7 @@ sections:
         body: |
 
           For any filter `x`, `map(x)` will run that filter for each
-          element of the input array, and produce the outputs a new
+          element of the input array, and return the outputs in a new
           array. `map(.+1)` will increment each element of an array of numbers.
 
           Similarly, `map_values(x)` will run that filter for each element,
@@ -762,8 +781,8 @@ sections:
         body: |
 
           Outputs array representations of the given path expression
-          in `.`.  The outputs are arrays of strings (keys in objects0
-          and/or numbers (array indices.
+          in `.`.  The outputs are arrays of strings (object keys)
+          and/or numbers (array indices).
 
           Path expressions are jq expressions like `.a`, but also `.[]`.
           There are two types of path expressions: ones that can match
@@ -834,6 +853,18 @@ sections:
             input: 'null'
             output: ['[{"a":1}]']
 
+      - title: "`delpaths(PATHS)`"
+        body: |
+
+          The builtin function `delpaths` sets the `PATHS` in `.`.
+          `PATHS` must be an array of paths, where each path is an array
+          of strings and numbers.
+
+        examples:
+          - program: 'delpaths([["a","b"]])'
+            input: '{"a":{"b":1},"x":{"y":2}}'
+            output: ['{"a":{},"x":{"y":2}}']
+
       - title: "`to_entries`, `from_entries`, `with_entries`"
         body: |
 
@@ -1835,6 +1866,46 @@ sections:
             input: '"2015-03-05T23:51:47Z"'
             output: ['1425599507']
 
+      - title: "SQL-Style Operators"
+        body: |
+
+          jq provides a few SQL-style operators.
+
+           * INDEX(stream; index_expression):
+
+             This builtin produces an object whose keys are computed by
+             the given index expression applied to each value from the
+             given stream.
+
+           * JOIN($idx; stream; idx_expr; join_expr):
+
+             This builtin joins the values from the given stream to the
+             given index.  The index's keys are computed by applying the
+             given index expression to each value from the given stream.
+             An array of the value in the stream and the corresponding
+             value from the index is fed to the given join expression to
+             produce each result.
+
+           * JOIN($idx; stream; idx_expr):
+
+             Same as `JOIN($idx; stream; idx_expr; .)`.
+
+           * JOIN($idx; idx_expr):
+
+             This builtin joins the input `.` to the given index, applying
+             the given index expression to `.` to compute the index key.
+             The join operation is as described above.
+
+           * IN(s):
+
+             This builtin outputs `true` if `.` appears in the given
+             stream, otherwise it outputs `false`.
+
+           * IN(source; s):
+
+             This builtin outputs `true` if any value in the source stream
+             appears in the second stream, otherwise it outputs `false`.
+
   - title: Conditionals and Comparisons
     entries:
       - title: "`==`, `!=`"
@@ -2349,17 +2420,17 @@ sections:
               def increment: . + 1;
 
           From then on, `increment` is usable as a filter just like a
-          builtin function (in fact, this is how some of the builtins
+          builtin function (in fact, this is how many of the builtins
           are defined). A function may take arguments:
 
               def map(f): [.[] | f];
 
-          Arguments are passed as filters, not as values. The
-          same argument may be referenced multiple times with
-          different inputs (here `f` is run for each element of the
-          input array). Arguments to a function work more like
-          callbacks than like value arguments.  This is important to
-          understand.  Consider:
+          Arguments are passed as _filters_ (functions with no
+          arguments), _not_ as values. The same argument may be
+          referenced multiple times with different inputs (here `f` is
+          run for each element of the input array).  Arguments to a
+          function work more like callbacks than like value arguments.
+          This is important to understand.  Consider:
 
               def foo(f): f|f;
               5|foo(.*2)
@@ -2380,12 +2451,16 @@ sections:
               def addvalue($f): ...;
 
           With either definition, `addvalue(.foo)` will add the current
-          input's `.foo` field to each element of the array.
+          input's `.foo` field to each element of the array.  Do note
+          that calling `addvalue(.[])` will cause the `map(. + $f)` part
+          to be evaluated once per value in the value of `.` at the call
+          site.
 
           Multiple definitions using the same function name are allowed.
           Each re-definition replaces the previous one for the same
           number of function arguments, but only for references from
           functions (or main program) subsequent to the re-definition.
+          See also the section below on scoping.
 
         examples:
           - program: 'def addvalue(f): . + [f]; map(addvalue(.[0]))'
@@ -2395,6 +2470,23 @@ sections:
             input: '[[1,2],[10,20]]'
             output: ['[[1,2,1,2], [10,20,1,2]]']
 
+      - title: 'Scoping'
+        body: |
+
+          There are two types of symbols in jq: value bindings (a.k.a.,
+          "variables"), and functions.  Both are scoped lexically,
+          with expressions being able to refer only to symbols that
+          have been defined "to the left" of them.  The only exception
+          to this rule is that functions can refer to themselves so as
+          to be able to create recursive functions.
+
+          For example, in the following expression there is a binding
+          which is visible "to the right" of it, `... | .*3 as
+          $times_three | [.  + $times_three] | ...`, but not "to the
+          left".  Consider this expression now, `... | (.*3 as
+          $times_three | [.+ $times_three]) | ...`: here the binding
+          `$times_three` is _not_ visible past the closing parenthesis.
+
       - title: Reduce
         body: |
 
@@ -2413,10 +2505,18 @@ sections:
                   (2 as $item | . + $item) |
                   (1 as $item | . + $item)
 
+          If the reduction update expression outputs `empty` (that is,
+          no values), then the reduction state is left as-is.  For
+          example, `reduce range(4) as $n ({}; if .==2 then empty else
+          .[$n|tostring] |= $n)` will produce `{"0":0,"1":1,"3":3}`.
+
         examples:
           - program: 'reduce .[] as $item (0; . + $item)'
             input: '[10,2,5,3]'
             output: ['20']
+          - program: 'reduce .[] as $n ([]; if $n%2==0 then empty else . + [$n] end)'
+            input: '[0,1,2,3,4,5]'
+            output: ['[1,3,5]']
 
       - title: "`limit(n; exp)`"
         body: |
@@ -2569,13 +2669,30 @@ sections:
       that take a single input argument (e.g., `sin()`) are available as
       zero-argument jq functions.  C math functions that take two input
       arguments (e.g., `pow()`) are available as two-argument jq
-      functions that ignore `.`.
+      functions that ignore `.`.  C math functions that take three input
+      arguments are available as three-argument jq functions that ignore
+      `.`.
 
       Availability of standard math functions depends on the
       availability of the corresponding math functions in your operating
       system and C math library.  Unavailable math functions will be
       defined but will raise an error.
 
+      One-input C math functions: `acos` `acosh` `asin` `asinh` `atan`
+      `atanh` `cbrt` `ceil` `cos` `cosh` `erf` `erfc` `exp` `exp10`
+      `exp2` `expm1` `fabs` `floor` `gamma` `j0` `j1` `lgamma` `log`
+      `log10` `log1p` `log2` `logb` `nearbyint` `pow10` `rint` `round`
+      `significand` `sin` `sinh` `sqrt` `tan` `tanh` `tgamma` `trunc`
+      `y0` `y1`.
+
+      Two-input C math functions: `atan2` `copysign` `drem` `fdim`
+      `fmax` `fmin` `fmod` `frexp` `hypot` `jn` `ldexp` `modf`
+      `nextafter` `nexttoward` `pow` `remainder` `scalb` `scalbln` `yn`.
+
+      Three-input C math functions: `fma`.
+
+      See your system's manual for more information on each of these.
+
   - title: 'I/O'
     body: |
 
@@ -2592,6 +2709,10 @@ sections:
       application-specific behavior, such as for executables that use
       the libjq C API but aren't the jq executable itself.
 
+      Most jq builtins are referentially transparent, and yield constant
+      and repeatable value streams when applied to constant inputs.
+      This is not true of I/O builtins.
+
     entries:
       - title: "`input`"
         body: |
@@ -2703,6 +2824,15 @@ sections:
       All the assignment operators in jq have path expressions on the
       left-hand side.
 
+      Values in jq are always immutable.  Internally, assignment works
+      by using a reduction to compute new, replacement values for `.` that
+      have had all the desired assignments applied to `.`, then
+      outputting the modified value.  This might be made clear by this
+      example: `{a:{b:{c:1}}} | (.a.b|=3), .`.  This will output
+      `{"a":{"b":3}}` and `{"a":{"b":{"c":1}}}` because the last
+      sub-expression, `.`, sees the original value, not the modified
+      value.
+
     entries:
       - title: "`=`"
         body: |
@@ -2770,8 +2900,12 @@ sections:
           not a valid or useful path expression in `.`); use `$var |
           .foo |= . + 1` instead.
 
-          If the right-hand side outputs multiple values, only the last
-          one will be used.
+          If the right-hand side outputs no values (i.e., `empty`), then
+          the left-hand side path will be deleted, as with `del(path)`.
+
+          If the right-hand side outputs multiple values, only the first
+          one will be used (NOTE: it used to be that only the last one was
+          used).
 
         examples:
 

docs/content/3.manual/v1.5/manual.yml

@@ -336,7 +336,7 @@ sections:
           to the last element, -2 referring to the next to last element,
           and so on.
 
-          The `.foo` syntax only works for simply keys i.e. keys that
+          The `.foo` syntax only works for simple keys i.e. keys that
           are all alphanumeric characters. `.[<string>]` works with
           keys that contain special characters such as colons and dots.
           For example `.["foo::bar"]` and `.["foo.bar"]` work while
@@ -711,7 +711,7 @@ sections:
       - title: "`in`"
         body: |
 
-          The builtin function `in` returns the input key is in the
+          The builtin function `in` returns whether or not the input key is in the
           given object, or the input index corresponds to an element
           in the given array. It is, essentially, an inversed version
           of `has`.
@@ -728,8 +728,8 @@ sections:
         body: |
 
           Outputs array representations of the given path expression
-          in `.`.  The outputs are arrays of strings (keys in objects0
-          and/or numbers (array indices.
+          in `.`.  The outputs are arrays of strings (object keys)
+          and/or numbers (array indices).
 
           Path expressions are jq expressions like `.a`, but also `.[]`.
           There are two types of path expressions: ones that can match
@@ -866,7 +866,7 @@ sections:
         body: |
 
           For any filter `x`, `map(x)` will run that filter for each
-          element of the input array, and produce the outputs a new
+          element of the input array, and return the outputs in a new
           array. `map(.+1)` will increment each element of an array of numbers.
 
           Similarly, `map_values(x)` will run that filter for each element,

jq.1.prebuilt

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "JQ" "1" "July 2015" "" ""
+.TH "JQ" "1" "February 2017" "" ""
 .
 .SH "NAME"
 \fBjq\fR \- Command\-line JSON processor
@@ -52,7 +52,7 @@ Output the jq version and exit with zero\.
 \fB\-\-seq\fR:
 .
 .IP
-Use the \fBapplication/json\-seq\fR MIME type scheme for separating JSON texts in jq\'s input and output\. This means that an ASCII RS (record separator) character is printed before each value on output and an ASCII LF (line feed) is printed after every output\. Input JSON texts that fail to parse are ignored (but warned about), discarding all subsequent input until the next RS\. This more also parses the output of jq without the \fB\-\-seq\fR option\.
+Use the \fBapplication/json\-seq\fR MIME type scheme for separating JSON texts in jq\'s input and output\. This means that an ASCII RS (record separator) character is printed before each value on output and an ASCII LF (line feed) is printed after every output\. Input JSON texts that fail to parse are ignored (but warned about), discarding all subsequent input until the next RS\. This mode also parses the output of jq without the \fB\-\-seq\fR option\.
 .
 .IP "\(bu" 4
 \fB\-\-stream\fR:
@@ -178,13 +178,13 @@ This option reads all the JSON texts in the named file and binds an array of the
 Do not use\. Use \fB\-\-slurpfile\fR instead\.
 .
 .IP
-(This option is like \fB\-\-slurpfile\fR, but when the file has just one text, then that is used, else an array of texts is used as in \fB\-\-slurfile\fR\.)
+(This option is like \fB\-\-slurpfile\fR, but when the file has just one text, then that is used, else an array of texts is used as in \fB\-\-slurpfile\fR\.)
 .
 .IP "\(bu" 4
 \fB\-\-run\-tests [filename]\fR:
 .
 .IP
-Runs the tests in the given file or standard input\. This must be the last option given and does not honor all preceding options\. The input consts of comment lines, empty lines, and program lines followed by one input line, as many lines of output as are expected (one per output), and a terminating empty line\. Compilation failure tests start with a line containing only "%%FAIL", then a line containing the program to compile, then a line containing an error message to compare to the actual\.
+Runs the tests in the given file or standard input\. This must be the last option given and does not honor all preceding options\. The input consists of comment lines, empty lines, and program lines followed by one input line, as many lines of output as are expected (one per output), and a terminating empty line\. Compilation failure tests start with a line containing only "%%FAIL", then a line containing the program to compile, then a line containing an error message to compare to the actual\.
 .
 .IP
 Be warned that this option can change backwards\-incompatibly\.
@@ -277,7 +277,7 @@ The \fB\.[10:15]\fR syntax can be used to return a subarray of an array or subst
 The \fB\.[2]\fR syntax can be used to return the element at the given index\. Negative indices are allowed, with \-1 referring to the last element, \-2 referring to the next to last element, and so on\.
 .
 .P
-The \fB\.foo\fR syntax only works for simply keys i\.e\. keys that are all alphanumeric characters\. \fB\.[<string>]\fR works with keys that contain special charactors such as colons and dots\. For example \fB\.["foo::bar"]\fR and \fB\.["foo\.bar"]\fR work while \fB\.foo::bar\fR and \fB\.foo\.bar\fR would not\.
+The \fB\.foo\fR syntax only works for simply keys i\.e\. keys that are all alphanumeric characters\. \fB\.[<string>]\fR works with keys that contain special characters such as colons and dots\. For example \fB\.["foo::bar"]\fR and \fB\.["foo\.bar"]\fR work while \fB\.foo::bar\fR and \fB\.foo\.bar\fR would not\.
 .
 .P
 The \fB?\fR "operator" can also be used with the slice operator, as in \fB\.[10:15]?\fR, which outputs values where the inputs are slice\-able\.
@@ -376,6 +376,12 @@ The | operator combines two filters by feeding the output(s) of the one on the l
 .P
 If the one on the left produces multiple results, the one on the right will be run for each of those results\. So, the expression \fB\.[] | \.foo\fR retrieves the "foo" field of each element of the input array\.
 .
+.P
+Note that \fB\.a\.b\.c\fR is the same as \fB\.a | \.b | \.c\fR\.
+.
+.P
+Note too that \fB\.\fR is the input value at the particular stage in a "pipeline", specifically: where the \fB\.\fR expression appears\. Thus \fB\.a | \. | \.b\fR is the same as \fB\.a\.b\fR, as the \fB\.\fR in the middle refers to whatever value \fB\.a\fR produced\.
+.
 .IP "" 4
 .
 .nf
@@ -388,6 +394,21 @@ jq \'\.[] | \.name\'
 .
 .IP "" 0
 .
+.SS "Parenthesis"
+Parenthesis work as a grouping operator just as in any typical programming language\.
+.
+.IP "" 4
+.
+.nf
+
+jq \'(\. + 2) * 5\'
+   1
+=> 15
+.
+.fi
+.
+.IP "" 0
+.
 .SH "TYPES AND VALUES"
 jq supports the same set of datatypes as JSON \- numbers, strings, booleans, arrays, objects (which in JSON\-speak are hashes with only string keys), and "null"\.
 .
@@ -573,7 +594,7 @@ jq \'{a: 1} + {b: 2} + {c: 3} + {a: 42}\'
 .IP "" 0
 .
 .SS "Subtraction \- \-"
-As well as normal arithmetic subtraction on numbers, the \fB\-\fR operator can be used on arrays to remove all occurences of the second array\'s elements from the first array\.
+As well as normal arithmetic subtraction on numbers, the \fB\-\fR operator can be used on arrays to remove all occurrences of the second array\'s elements from the first array\.
 .
 .IP "" 4
 .
@@ -592,13 +613,13 @@ jq \'\. \- ["xml", "yaml"]\'
 .IP "" 0
 .
 .SS "Multiplication, division, modulo \- *, /, and %"
-These operators only work on numbers, and do the expected\.
+These infix operators behave as expected when given two numbers\. Division by zero raises an error\. \fBx % y\fR computes x modulo y\.
 .
 .P
-Multiplying a string by a number produces the concatenation of that string that many times\.
+Multiplying a string by a number produces the concatenation of that string that many times\. \fB"x" * 0\fR produces \fBnull\fR\.
 .
 .P
-Dividing a string by another splits the first using the second as separators\. Division by zero raises an error\.
+Dividing a string by another splits the first using the second as separators\.
 .
 .P
 Multiplying two objects will merge them recursively: this works like addition but if both objects contain a value for the same key, and the values are objects, the two are merged with the same strategy\.
@@ -647,6 +668,21 @@ jq \'\.[] | length\' [[1,2], "string", {"a":2}, null] => 2, 6, 1, 0
 .
 .IP "" 0
 .
+.SS "utf8bytelength"
+The builtin function \fButf8bytelength\fR outputs the number of bytes used to encode a string in UTF\-8\.
+.
+.IP "" 4
+.
+.nf
+
+jq \'utf8bytelength\'
+   "\eu03bc"
+=> 2
+.
+.fi
+.
+.IP "" 0
+.
 .SS "keys, keys_unsorted"
 The builtin function \fBkeys\fR, when given an object, returns its keys in an array\.
 .
@@ -698,7 +734,7 @@ jq \'map(has(2))\'
 .IP "" 0
 .
 .SS "in"
-The builtin function \fBin\fR returns the input key is in the given object, or the input index corresponds to an element in the given array\. It is, essentially, an inversed version of \fBhas\fR\.
+The builtin function \fBin\fR returns whether or not the input key is in the given object, or the input index corresponds to an element in the given array\. It is, essentially, an inversed version of \fBhas\fR\.
 .
 .IP "" 4
 .
@@ -716,8 +752,33 @@ jq \'map(in([0,1]))\'
 .
 .IP "" 0
 .
+.SS "map(x), map_values(x)"
+For any filter \fBx\fR, \fBmap(x)\fR will run that filter for each element of the input array, and return the outputs in a new array\. \fBmap(\.+1)\fR will increment each element of an array of numbers\.
+.
+.P
+Similarly, \fBmap_values(x)\fR will run that filter for each element, but it will return an object when an object is passed\.
+.
+.P
+\fBmap(x)\fR is equivalent to \fB[\.[] | x]\fR\. In fact, this is how it\'s defined\. Similarly, \fBmap_values(x)\fR is defined as \fB\.[] |= x\fR\.
+.
+.IP "" 4
+.
+.nf
+
+jq \'map(\.+1)\'
+   [1,2,3]
+=> [2,3,4]
+
+jq \'map_values(\.+1)\'
+   {"a": 1, "b": 2, "c": 3}
+=> {"a": 2, "b": 3, "c": 4}
+.
+.fi
+.
+.IP "" 0
+.
 .SS "path(path_expression)"
-Outputs array representations of the given path expression in \fB\.\fR\. The outputs are arrays of strings (keys in objects0 and/or numbers (array indices\.
+Outputs array representations of the given path expression in \fB\.\fR\. The outputs are arrays of strings (object keys) and/or numbers (array indices)\.
 .
 .P
 Path expressions are jq expressions like \fB\.a\fR, but also \fB\.[]\fR\. There are two types of path expressions: ones that can match exactly, and ones that cannot\. For example, \fB\.a\.b\.c\fR is an exact match path expression, while \fB\.a[]\.b\fR is not\.
@@ -766,11 +827,68 @@ jq \'del(\.[1, 2])\'
 .
 .IP "" 0
 .
+.SS "getpath(PATHS)"
+The builtin function \fBgetpath\fR outputs the values in \fB\.\fR found at each path in \fBPATHS\fR\.
+.
+.IP "" 4
+.
+.nf
+
+jq \'getpath(["a","b"])\'
+   null
+=> null
+
+jq \'[getpath(["a","b"], ["a","c"])]\'
+   {"a":{"b":0, "c":1}}
+=> [0, 1]
+.
+.fi
+.
+.IP "" 0
+.
+.SS "setpath(PATHS; VALUE)"
+The builtin function \fBsetpath\fR sets the \fBPATHS\fR in \fB\.\fR to \fBVALUE\fR\.
+.
+.IP "" 4
+.
+.nf
+
+jq \'setpath(["a","b"]; 1)\'
+   null
+=> {"a": {"b": 1}}
+
+jq \'setpath(["a","b"]; 1)\'
+   {"a":{"b":0}}
+=> {"a": {"b": 1}}
+
+jq \'setpath([0,"a"]; 1)\'
+   null
+=> [{"a":1}]
+.
+.fi
+.
+.IP "" 0
+.
+.SS "delpaths(PATHS)"
+The builtin function \fBdelpaths\fR sets the \fBPATHS\fR in \fB\.\fR\. \fBPATHS\fR must be an array of paths, where each path is an array of strings and numbers\.
+.
+.IP "" 4
+.
+.nf
+
+jq \'delpaths([["a","b"]])\'
+   {"a":{"b":1},"x":{"y":2}}
+=> {"a":{},"x":{"y":2}}
+.
+.fi
+.
+.IP "" 0
+.
 .SS "to_entries, from_entries, with_entries"
 These functions convert between an object and an array of key\-value pairs\. If \fBto_entries\fR is passed an object, then for each \fBk: v\fR entry in the input, the output array includes \fB{"key": k, "value": v}\fR\.
 .
 .P
-\fBfrom_entries\fR does the opposite conversion, and \fBwith_entries(foo)\fR is a shorthand for \fBto_entries | map(foo) | from_entries\fR, useful for doing some operation to all keys and values of an object\. \fBfrom_entries\fR accepts key, Key, Name, value and Value as keys\.
+\fBfrom_entries\fR does the opposite conversion, and \fBwith_entries(foo)\fR is a shorthand for \fBto_entries | map(foo) | from_entries\fR, useful for doing some operation to all keys and values of an object\. \fBfrom_entries\fR accepts key, Key, name, Name, value and Value as keys\.
 .
 .IP "" 4
 .
@@ -869,31 +987,6 @@ jq \'try error("\e($__loc__)") catch \.\'
 .
 .IP "" 0
 .
-.SS "map(x), map_values(x)"
-For any filter \fBx\fR, \fBmap(x)\fR will run that filter for each element of the input array, and produce the outputs a new array\. \fBmap(\.+1)\fR will increment each element of an array of numbers\.
-.
-.P
-Similarly, \fBmap_values(x)\fR will run that filter for each element, but it will return an object when an object is passed\.
-.
-.P
-\fBmap(x)\fR is equivalent to \fB[\.[] | x]\fR\. In fact, this is how it\'s defined\. Similarly, \fBmap_values(x)\fR is defined as \fB\.[] |= x\fR\.
-.
-.IP "" 4
-.
-.nf
-
-jq \'map(\.+1)\'
-   [1,2,3]
-=> [2,3,4]
-
-jq \'map_values(\.+1)\'
-   {"a": 1, "b": 2, "c": 3}
-=> {"a": 2, "b": 3, "c": 4}
-.
-.fi
-.
-.IP "" 0
-.
 .SS "paths, paths(node_filter), leaf_paths"
 \fBpaths\fR outputs the paths to all the elements in its input (except it does not output the empty list, representing \. itself)\.
 .
@@ -946,7 +1039,7 @@ jq \'add\'
 .IP "" 0
 .
 .SS "any, any(condition), any(generator; condition)"
-The filter \fBany\fR takes as input an array of boolean values, and produces \fBtrue\fR as output if any of the the elements of the array is \fBtrue\fR\.
+The filter \fBany\fR takes as input an array of boolean values, and produces \fBtrue\fR as output if any of the elements of the array are \fBtrue\fR\.
 .
 .P
 If the input is an empty array, \fBany\fR returns \fBfalse\fR\.
@@ -978,7 +1071,7 @@ jq \'any\'
 .IP "" 0
 .
 .SS "all, all(condition), all(generator; condition)"
-The filter \fBall\fR takes as input an array of boolean values, and produces \fBtrue\fR as output if all of the the elements of the array are \fBtrue\fR\.
+The filter \fBall\fR takes as input an array of boolean values, and produces \fBtrue\fR as output if all of the elements of the array are \fBtrue\fR\.
 .
 .P
 The \fBall(condition)\fR form applies the given condition to the elements of the input array\.
@@ -1009,7 +1102,7 @@ jq \'all\'
 .
 .IP "" 0
 .
-.SS "[Requires 1\.5] flatten, flatten(depth)"
+.SS "flatten, flatten(depth)"
 The filter \fBflatten\fR takes as input an array of nested arrays, and produces a flat array in which all arrays inside the original array have been recursively replaced by their values\. You can pass an argument to it to specify how many levels of nesting to flatten\.
 .
 .P
@@ -1528,7 +1621,7 @@ jq \'implode\'
 .
 .IP "" 0
 .
-.SS "split"
+.SS "split(str)"
 Splits an input string on the separator argument\.
 .
 .IP "" 4
@@ -1546,6 +1639,9 @@ jq \'split(", ")\'
 .SS "join(str)"
 Joins the array of elements given as input, using the argument as separator\. It is the inverse of \fBsplit\fR: that is, running \fBsplit("foo") | join("foo")\fR over any input string returns said input string\.
 .
+.P
+Numbers and booleans in the input are converted to strings\. Null values are treated as empty strings\. Arrays and objects in the input are not supported\.
+.
 .IP "" 4
 .
 .nf
@@ -1553,6 +1649,10 @@ Joins the array of elements given as input, using the argument as separator\. It
 jq \'join(", ")\'
    ["a","b,c,d","e"]
 => "a, b,c,d, e"
+
+jq \'join(" ")\'
+   ["a",1,2\.3,true,null,false]
+=> "a 1 2\.3 true  false"
 .
 .fi
 .
@@ -1664,6 +1764,25 @@ jq \'recurse(\. * \.; \. < 20)\'
 .
 .IP "" 0
 .
+.SS "walk(f)"
+The \fBwalk(f)\fR function applies f recursively to every component of the input entity\. When an array is encountered, f is first applied to its elements and then to the array itself; when an object is encountered, f is first applied to all the values and then to the object\. In practice, f will usually test the type of its input, as illustrated in the following examples\. The first example highlights the usefulness of processing the elements of an array of arrays before processing the array itself\. The second example shows how all the keys of all the objects within the input can be considered for alteration\.
+.
+.IP "" 4
+.
+.nf
+
+jq \'walk(if type == "array" then sort else \. end)\'
+   [[4, 1, 7], [8, 5, 2], [3, 6, 9]]
+=> [[1,4,7],[2,5,8],[3,6,9]]
+
+jq \'walk( if type == "object" then with_entries( \.key |= sub( "^_+"; "") ) else \. end )\'
+   [ { "_a": { "__b": 2 } } ]
+=> [{"a":{"b":2}}]
+.
+.fi
+.
+.IP "" 0
+.
 .SS "\.\."
 Short\-hand for \fBrecurse\fR without arguments\. This is intended to resemble the XPath \fB//\fR operator\. Note that \fB\.\.a\fR does not work; use \fB\.\.|a\fR instead\. In the example below we use \fB\.\.|\.a?\fR to find all the values of object keys "a" in any object found "below" \fB\.\fR\.
 .
@@ -1783,7 +1902,7 @@ Calls \fBtostring\fR, see that function for details\.
 \fB@json\fR:
 .
 .IP
-Serialises the input as JSON\.
+Serializes the input as JSON\.
 .
 .TP
 \fB@html\fR:
@@ -1828,7 +1947,7 @@ This syntax can be combined with string interpolation in a useful way\. You can
 .
 .nf
 
-@uri "http://www\.google\.com/search?q=\e(\.search)"
+@uri "https://www\.google\.com/search?q=\e(\.search)"
 .
 .fi
 .
@@ -1841,7 +1960,7 @@ will produce the following output for the input \fB{"search":"what is jq?"}\fR:
 .
 .nf
 
-"http://www\.google\.com/search?q=what%20is%20jq%3F"
+"https://www\.google\.com/search?q=what%20is%20jq%3F"
 .
 .fi
 .
@@ -1922,6 +2041,45 @@ jq \'strptime("%Y\-%m\-%dT%H:%M:%SZ")|mktime\'
 .
 .IP "" 0
 .
+.SS "SQL\-Style Operators"
+jq provides a few SQL\-style operators\.
+.
+.TP
+INDEX(stream; index_expression):
+.
+.IP
+This builtin produces an object whose keys are computed by the given index expression applied to each value from the given stream\.
+.
+.TP
+JOIN($idx; stream; idx_expr; join_expr):
+.
+.IP
+This builtin joins the values from the given stream to the given index\. The index\'s keys are computed by applying the given index expression to each value from the given stream\. An array of the value in the stream and the corresponding value from the index is fed to the given join expression to produce each result\.
+.
+.TP
+JOIN($idx; stream; idx_expr):
+.
+.IP
+Same as \fBJOIN($idx; stream; idx_expr; \.)\fR\.
+.
+.TP
+JOIN($idx; idx_expr):
+.
+.IP
+This builtin joins the input \fB\.\fR to the given index, applying the given index expression to \fB\.\fR to compute the index key\. The join operation is as described above\.
+.
+.TP
+IN(s):
+.
+.IP
+This builtin outputs \fBtrue\fR if \fB\.\fR appears in the given stream, otherwise it outputs \fBfalse\fR\.
+.
+.TP
+IN(source; s):
+.
+.IP
+This builtin outputs \fBtrue\fR if any value in the source stream appears in the second stream, otherwise it outputs \fBfalse\fR\.
+.
 .SH "CONDITIONALS AND COMPARISONS"
 .
 .SS "==, !="
@@ -1949,7 +2107,7 @@ jq \'\.[] == 1\'
 Checking for false or null is a simpler notion of "truthiness" than is found in Javascript or Python, but it means that you\'ll sometimes have to be more explicit about the condition you want: you can\'t test whether, e\.g\. a string is empty using \fBif \.name then A else B end\fR, you\'ll need something more like \fBif (\.name | length) > 0 then A else B end\fR instead\.
 .
 .P
-If the condition A produces multiple results, it is considered "true" if any of those results is not false or null\. If it produces zero results, it\'s considered false\.
+If the condition \fBA\fR produces multiple results, then \fBB\fR is evaluated once for each result that is not false or null, and \fBC\fR is evaluated once for each false or null\.
 .
 .P
 More cases can be added to an if using \fBelif A then B\fR syntax\.
@@ -1965,18 +2123,7 @@ jq \'if \. == 0 then
 .IP "" 0
 .
 .P
-"zero" elif \. == 1 then "one" else "many" end\'
-.
-.IP "" 4
-.
-.nf
-
-   2
-=> "many"
-.
-.fi
-.
-.IP "" 0
+"zero" elif \. == 1 then "one" else "many" end\' 2 => "many"
 .
 .SS ">, >=, <=, <"
 The comparison operators \fB>\fR, \fB>=\fR, \fB<=\fR, \fB<\fR return whether their left argument is greater than, greater than or equal to, less than or equal to or less than their right argument (respectively)\.
@@ -2232,7 +2379,7 @@ jq \-n \'("test", "TEst", "teST", "TEST") | test( "(?i)te(?\-i)st" )\'
 .P
 evaluates to: true, true, false, false\.
 .
-.SS "[Requires 1\.5] test(val), test(regex; flags)"
+.SS "test(val), test(regex; flags)"
 Like \fBmatch\fR, but does not return match objects, only \fBtrue\fR or \fBfalse\fR for whether or not the regex matches the input\.
 .
 .IP "" 4
@@ -2251,7 +2398,7 @@ jq \'\.[] | test("a b c # spaces are ignored"; "ix")\'
 .
 .IP "" 0
 .
-.SS "[Requires 1\.5] match(val), match(regex; flags)"
+.SS "match(val), match(regex; flags)"
 \fBmatch\fR outputs an object for each match it finds\. Matches have the following fields:
 .
 .IP "\(bu" 4
@@ -2316,7 +2463,7 @@ jq \'[ match("\."; "g")] | length\'
 .
 .IP "" 0
 .
-.SS "[Requires 1\.5] capture(val), capture(regex; flags)"
+.SS "capture(val), capture(regex; flags)"
 Collects the named captures in a JSON object, with the name of each capture as the key, and the matched string as the corresponding value\.
 .
 .IP "" 4
@@ -2331,19 +2478,19 @@ jq \'capture("(?<a>[a\-z]+)\-(?<n>[0\-9]+)")\'
 .
 .IP "" 0
 .
-.SS "[Requires 1\.5] scan(regex), scan(regex; flags)"
+.SS "scan(regex), scan(regex; flags)"
 Emit a stream of the non\-overlapping substrings of the input that match the regex in accordance with the flags, if any have been specified\. If there is no match, the stream is empty\. To capture all the matches for each input string, use the idiom \fB[ expr ]\fR, e\.g\. \fB[ scan(regex) ]\fR\.
 .
 .SS "split(regex; flags)"
 For backwards compatibility, \fBsplit\fR splits on a string, not a regex\.
 .
-.SS "[Requires 1\.5] splits(regex), splits(regex; flags)"
+.SS "splits(regex), splits(regex; flags)"
 These provide the same results as their \fBsplit\fR counterparts, but as a stream instead of an array\.
 .
-.SS "[Requires 1\.5] sub(regex; tostring) sub(regex; string; flags)"
+.SS "sub(regex; tostring) sub(regex; string; flags)"
 Emit the string obtained by replacing the first match of regex in the input string with \fBtostring\fR, after interpolation\. \fBtostring\fR should be a jq string, and may contain references to named captures\. The named captures are, in effect, presented as a JSON object (as constructed by \fBcapture\fR) to \fBtostring\fR, so a reference to a captured variable named "x" would take the form: "(\.x)"\.
 .
-.SS "[Requires 1\.5] gsub(regex; string), gsub(regex; string; flags)"
+.SS "gsub(regex; string), gsub(regex; string; flags)"
 \fBgsub\fR is like \fBsub\fR but all the non\-overlapping occurrences of the regex are replaced by the string, after interpolation\.
 .
 .SH "ADVANCED FEATURES"
@@ -2524,7 +2671,7 @@ def increment: \. + 1;
 .IP "" 0
 .
 .P
-From then on, \fBincrement\fR is usable as a filter just like a builtin function (in fact, this is how some of the builtins are defined)\. A function may take arguments:
+From then on, \fBincrement\fR is usable as a filter just like a builtin function (in fact, this is how many of the builtins are defined)\. A function may take arguments:
 .
 .IP "" 4
 .
@@ -2537,7 +2684,7 @@ def map(f): [\.[] | f];
 .IP "" 0
 .
 .P
-Arguments are passed as filters, not as values\. The same argument may be referenced multiple times with different inputs (here \fBf\fR is run for each element of the input array)\. Arguments to a function work more like callbacks than like value arguments\. This is important to understand\. Consider:
+Arguments are passed as \fIfilters\fR (functions with no arguments), \fInot\fR as values\. The same argument may be referenced multiple times with different inputs (here \fBf\fR is run for each element of the input array)\. Arguments to a function work more like callbacks than like value arguments\. This is important to understand\. Consider:
 .
 .IP "" 4
 .
@@ -2580,10 +2727,10 @@ def addvalue($f): \.\.\.;
 .IP "" 0
 .
 .P
-With either definition, \fBaddvalue(\.foo)\fR will add the current input\'s \fB\.foo\fR field to each element of the array\.
+With either definition, \fBaddvalue(\.foo)\fR will add the current input\'s \fB\.foo\fR field to each element of the array\. Do note that calling \fBaddvalue(\.[])\fR will cause the \fBmap(\. + $f)\fR part to be evaluated once per value in the value of \fB\.\fR at the call site\.
 .
 .P
-Multiple definitions using the same function name are allowed\. Each re\-definition replaces the previous one for the same number of function arguments, but only for references from functions (or main program) subsequent to the re\-definition\.
+Multiple definitions using the same function name are allowed\. Each re\-definition replaces the previous one for the same number of function arguments, but only for references from functions (or main program) subsequent to the re\-definition\. See also the section below on scoping\.
 .
 .IP "" 4
 .
@@ -2601,6 +2748,12 @@ jq \'def addvalue(f): f as $x | map(\. + $x); addvalue(\.[0])\'
 .
 .IP "" 0
 .
+.SS "Scoping"
+There are two types of symbols in jq: value bindings (a\.k\.a\., "variables"), and functions\. Both are scoped lexically, with expressions being able to refer only to symbols that have been defined "to the left" of them\. The only exception to this rule is that functions can refer to themselves so as to be able to create recursive functions\.
+.
+.P
+For example, in the following expression there is a binding which is visible "to the right" of it, \fB\.\.\. | \.*3 as $times_three | [\. + $times_three] | \.\.\.\fR, but not "to the left"\. Consider this expression now, \fB\.\.\. | (\.*3 as $times_three | [\.+ $times_three]) | \.\.\.\fR: here the binding \fB$times_three\fR is \fInot\fR visible past the closing parenthesis\.
+.
 .SS "Reduce"
 The \fBreduce\fR syntax in jq allows you to combine all of the results of an expression by accumulating them into a single answer\. As an example, we\'ll pass \fB[3,2,1]\fR to this expression:
 .
@@ -2624,10 +2777,25 @@ For each result that \fB\.[]\fR produces, \fB\. + $item\fR is run to accumulate
 0 | (3 as $item | \. + $item) |
     (2 as $item | \. + $item) |
     (1 as $item | \. + $item)
+.
+.fi
+.
+.IP "" 0
+.
+.P
+If the reduction update expression outputs \fBempty\fR (that is, no values), then the reduction state is left as\-is\. For example, \fBreduce range(4) as $n ({}; if \.==2 then empty else \.[$n|tostring] |= $n)\fR will produce \fB{"0":0,"1":1,"3":3}\fR\.
+.
+.IP "" 4
+.
+.nf
 
 jq \'reduce \.[] as $item (0; \. + $item)\'
    [10,2,5,3]
 => 20
+
+jq \'reduce \.[] as $n ([]; if $n%2==0 then empty else \. + [$n] end)\'
+   [0,1,2,3,4,5]
+=> [1,3,5]
 .
 .fi
 .
@@ -2709,7 +2877,7 @@ jq \'[foreach \.[] as $item ([[],[]]; if $item == null then [[],\.[0]] else [(\.
 As described above, \fBrecurse\fR uses recursion, and any jq function can be recursive\. The \fBwhile\fR builtin is also implemented in terms of recursion\.
 .
 .P
-Tail calls are optmized whenever the expression to the left of the recursive call outputs its last value\. In practice this means that the expression to the left of the recursive call should not produce more than one output for each input\.
+Tail calls are optimized whenever the expression to the left of the recursive call outputs its last value\. In practice this means that the expression to the left of the recursive call should not produce more than one output for each input\.
 .
 .P
 For example:
@@ -2766,17 +2934,32 @@ jq \'def while(cond; update): def _while: if cond then \., (update | _while) els
 jq currently only has IEEE754 double\-precision (64\-bit) floating point number support\.
 .
 .P
-Besides simple arithmetic operators such as \fB+\fR, jq also has most standard math functions from the C math library\. C math functions that take a single input argument (e\.g\., \fBsin()\fR) are available as zero\-argument jq functions\. C math functions that take two input arguments (e\.g\., \fBpow()\fR) are available as two\-argument jq functions that ignore \fB\.\fR\.
+Besides simple arithmetic operators such as \fB+\fR, jq also has most standard math functions from the C math library\. C math functions that take a single input argument (e\.g\., \fBsin()\fR) are available as zero\-argument jq functions\. C math functions that take two input arguments (e\.g\., \fBpow()\fR) are available as two\-argument jq functions that ignore \fB\.\fR\. C math functions that take three input arguments are available as three\-argument jq functions that ignore \fB\.\fR\.
 .
 .P
 Availability of standard math functions depends on the availability of the corresponding math functions in your operating system and C math library\. Unavailable math functions will be defined but will raise an error\.
 .
+.P
+One\-input C math functions: \fBacos\fR \fBacosh\fR \fBasin\fR \fBasinh\fR \fBatan\fR \fBatanh\fR \fBcbrt\fR \fBceil\fR \fBcos\fR \fBcosh\fR \fBerf\fR \fBerfc\fR \fBexp\fR \fBexp10\fR \fBexp2\fR \fBexpm1\fR \fBfabs\fR \fBfloor\fR \fBgamma\fR \fBj0\fR \fBj1\fR \fBlgamma\fR \fBlog\fR \fBlog10\fR \fBlog1p\fR \fBlog2\fR \fBlogb\fR \fBnearbyint\fR \fBpow10\fR \fBrint\fR \fBround\fR \fBsignificand\fR \fBsin\fR \fBsinh\fR \fBsqrt\fR \fBtan\fR \fBtanh\fR \fBtgamma\fR \fBtrunc\fR \fBy0\fR \fBy1\fR\.
+.
+.P
+Two\-input C math functions: \fBatan2\fR \fBcopysign\fR \fBdrem\fR \fBfdim\fR \fBfmax\fR \fBfmin\fR \fBfmod\fR \fBfrexp\fR \fBhypot\fR \fBjn\fR \fBldexp\fR \fBmodf\fR \fBnextafter\fR \fBnexttoward\fR \fBpow\fR \fBremainder\fR \fBscalb\fR \fBscalbln\fR \fByn\fR\.
+.
+.P
+Three\-input C math functions: \fBfma\fR\.
+.
+.P
+See your system\'s manual for more information on each of these\.
+.
 .SH "I/O"
 At this time jq has minimal support for I/O, mostly in the form of control over when inputs are read\. Two builtins functions are provided for this, \fBinput\fR and \fBinputs\fR, that read from the same sources (e\.g\., \fBstdin\fR, files named on the command\-line) as jq itself\. These two builtins, and jq\'s own reading actions, can be interleaved with each other\.
 .
 .P
 One builtin provides minimal output capabilities, \fBdebug\fR\. (Recall that a jq program\'s output values are always output as JSON texts on \fBstdout\fR\.) The \fBdebug\fR builtin can have application\-specific behavior, such as for executables that use the libjq C API but aren\'t the jq executable itself\.
 .
+.P
+Most jq builtins are referentially transparent, and yield constant and repeatable value streams when applied to constant inputs\. This is not true of I/O builtins\.
+.
 .SS "input"
 Outputs one new input\.
 .
@@ -2805,7 +2988,7 @@ However, streaming isn\'t easy to deal with as the jq program will have \fB[<pat
 Several builtins are provided to make handling streams easier\.
 .
 .P
-The examples below use the the streamed form of \fB[0,[1]]\fR, which is \fB[[0],1],[[1,0],2],[[1,0]],[[1]])]\fR\.
+The examples below use the streamed form of \fB[0,[1]]\fR, which is \fB[[0],0],[[1,0],1],[[1,0]],[[1]]\fR\.
 .
 .P
 Streaming forms include \fB[<path>, <leaf\-value>]\fR (to indicate any scalar value, empty array, or empty object), and \fB[<path>]\fR (to indicate the end of an array or object)\. Future versions of jq run with \fB\-\-stream\fR and \fB\-seq\fR may output additional forms such as \fB["error message"]\fR when an input text fails to parse\.
@@ -2861,6 +3044,12 @@ Assignment works a little differently in jq than in most programming languages\.
 .P
 If an object has two fields which are arrays, \fB\.foo\fR and \fB\.bar\fR, and you append something to \fB\.foo\fR, then \fB\.bar\fR will not get bigger\. Even if you\'ve just set \fB\.bar = \.foo\fR\. If you\'re used to programming in languages like Python, Java, Ruby, Javascript, etc\. then you can think of it as though jq does a full deep copy of every object before it does the assignment (for performance, it doesn\'t actually do that, but that\'s the general idea)\.
 .
+.P
+All the assignment operators in jq have path expressions on the left\-hand side\.
+.
+.P
+Values in jq are always immutable\. Internally, assignment works by using a reduction to compute new, replacement values for \fB\.\fR that have had all the desired assignments applied to \fB\.\fR, then outputting the modified value\. This might be made clear by this example: \fB{a:{b:{c:1}}} | (\.a\.b|=3), \.\fR\. This will output \fB{"a":{"b":3}}\fR and \fB{"a":{"b":{"c":1}}}\fR because the last sub\-expression, \fB\.\fR, sees the original value, not the modified value\.
+.
 .SS "="
 The filter \fB\.foo = 1\fR will take as input an object and produce as output an object with the "foo" field set to 1\. There is no notion of "modifying" or "changing" something in jq \- all jq values are immutable\. For instance,
 .
@@ -2874,7 +3063,13 @@ will not have the side\-effect of setting \.bar\.baz to be set to 1, as the simi
 This means that it\'s impossible to build circular values in jq (such as an array whose first element is itself)\. This is quite intentional, and ensures that anything a jq program can produce can be represented in JSON\.
 .
 .P
-Note that the left\-hand side of \'=\' refers to a value in \fB\.\fR\. Thus \fB$var\.foo = 1\fR won\'t work as expected; use \fB$var | \.foo = 1\fR instead\.
+Note that the left\-hand side of \'=\' refers to a value in \fB\.\fR\. Thus \fB$var\.foo = 1\fR won\'t work as expected (\fB$var\.foo\fR is not a valid or useful path expression in \fB\.\fR); use \fB$var | \.foo = 1\fR instead\.
+.
+.P
+If the right\-hand side of \'=\' produces multiple values, then for each such value jq will set the paths on the left\-hand side to the value and then it will output the modified \fB\.\fR\. For example, \fB(\.a,\.b)=range(2)\fR outputs \fB{"a":0,"b":0}\fR, then \fB{"a":1,"b":1}\fR\. The "update" assignment forms (see below) do not do this\.
+.
+.P
+Note too that \fB\.a,\.b=0\fR does not set \fB\.a\fR and \fB\.b\fR, but \fB(\.a,\.b)=0\fR sets both\.
 .
 .SS "|="
 As well as the assignment operator \'=\', jq provides the "update" operator \'|=\', which takes a filter on the right\-hand side and works out the new value for the property of \fB\.\fR being assigned to by running the old value through this expression\. For instance, \.foo |= \.+1 will build an object with the "foo" field set to the input\'s "foo" plus 1\.
@@ -2895,7 +3090,13 @@ The former will set the "a" field of the input to the "b" field of the input, an
 The left\-hand side can be any general path expression; see \fBpath()\fR\.
 .
 .P
-Note that the left\-hand side of \'|=\' refers to a value in \fB\.\fR\. Thus \fB$var\.foo |= \. + 1\fR won\'t work as expected; use \fB$var | \.foo |= \. + 1\fR instead\.
+Note that the left\-hand side of \'|=\' refers to a value in \fB\.\fR\. Thus \fB$var\.foo |= \. + 1\fR won\'t work as expected (\fB$var\.foo\fR is not a valid or useful path expression in \fB\.\fR); use \fB$var | \.foo |= \. + 1\fR instead\.
+.
+.P
+If the right\-hand side outputs no values (i\.e\., \fBempty\fR), then the left\-hand side path will be deleted, as with \fBdel(path)\fR\.
+.
+.P
+If the right\-hand side outputs multiple values, only the first one will be used (NOTE: it used to be that only the last one was used)\.
 .
 .IP "" 4
 .

scripts/crosscompile

@@ -38,5 +38,6 @@ set -x
 for jq in `find . -type f \( -name jq -o -name jq.exe \) -print`; do
     cp "$jq" ..
 done
+cd ..
 rm -rf tmp
 

scripts/gen_utf8_tables.py

@@ -7,7 +7,6 @@ mask = lambda n: (1 << n) - 1
 
 def print_table(type, name, t):
     print("static const %s %s[] =" % (type, name))
-    first = True
     for i in range(0,len(t),16):
         print ((" {" if i == 0 else "  ") +
                ", ".join("0x%02x"%n for n in t[i:i+16]) +

src/builtin.c

@@ -1,7 +1,12 @@
 #define _BSD_SOURCE
 #define _GNU_SOURCE
-#define _XOPEN_SOURCE
-#define _XOPEN_SOURCE_EXTENDED 1
+#ifndef __sun__
+# define _XOPEN_SOURCE
+# define _XOPEN_SOURCE_EXTENDED 1
+#else
+# define _XPG6
+# define __EXTENSIONS__
+#endif
 #include <sys/time.h>
 #include <stdlib.h>
 #include <stddef.h>
@@ -105,9 +110,24 @@ static jv f_ ## name(jq_state *jq, jv input, jv a, jv b) { \
   return ret; \
 }
 #define LIBM_DDD_NO(name)
+
+#define LIBM_DDDD(name) \
+static jv f_ ## name(jq_state *jq, jv input, jv a, jv b, jv c) { \
+  if (jv_get_kind(a) != JV_KIND_NUMBER || jv_get_kind(b) != JV_KIND_NUMBER) \
+    return type_error(input, "number required"); \
+  jv_free(input); \
+  jv ret = jv_number(name(jv_number_value(a), jv_number_value(b), jv_number_value(c))); \
+  jv_free(a); \
+  jv_free(b); \
+  jv_free(c); \
+  return ret; \
+}
+#define LIBM_DDDD_NO(name)
 #include "libm.h"
+#undef LIBM_DDDD_NO
 #undef LIBM_DDD_NO
 #undef LIBM_DD_NO
+#undef LIBM_DDDD
 #undef LIBM_DDD
 #undef LIBM_DD
 
@@ -1227,14 +1247,17 @@ static jv f_now(jq_state *jq, jv a) {
 }
 #endif
 
-static jv f_current_filename(jq_state *jq) {
+static jv f_current_filename(jq_state *jq, jv a) {
+  jv_free(a);
+
   jv r = jq_util_input_get_current_filename(jq);
   if (jv_is_valid(r))
     return r;
   jv_free(r);
   return jv_null();
 }
-static jv f_current_line(jq_state *jq) {
+static jv f_current_line(jq_state *jq, jv a) {
+  jv_free(a);
   return jq_util_input_get_current_line(jq);
 }
 
@@ -1246,6 +1269,10 @@ static jv f_current_line(jq_state *jq) {
   {(cfunction_ptr)f_ ## name, "_" #name, 3},
 #define LIBM_DDD_NO(name)
 
+#define LIBM_DDDD(name) \
+  {(cfunction_ptr)f_ ## name, "_" #name, 4},
+#define LIBM_DDDD_NO(name)
+
 static const struct cfunction function_list[] = {
 #include "libm.h"
   {(cfunction_ptr)f_plus, "_plus", 3},
@@ -1313,8 +1340,10 @@ static const struct cfunction function_list[] = {
   {(cfunction_ptr)f_current_filename, "input_filename", 1},
   {(cfunction_ptr)f_current_line, "input_line_number", 1},
 };
+#undef LIBM_DDDD_NO
 #undef LIBM_DDD_NO
 #undef LIBM_DD_NO
+#undef LIBM_DDDD
 #undef LIBM_DDD
 #undef LIBM_DD
 
@@ -1367,16 +1396,37 @@ static block bind_bytecoded_builtins(block b) {
 
 #define LIBM_DD(name) "def " #name ": _" #name ";"
 #define LIBM_DDD(name) "def " #name "(a;b): _" #name "(a;b);"
-#define LIBM_DD_NO(name) "def " #name ": \"Error: " #name "() not found at build time\"|error;"
-#define LIBM_DDD_NO(name) "def " #name "(a;b): \"Error: " #name "() not found at build time\"|error;"
+#define LIBM_DDDD(name) "def " #name "(a;b;c): _" #name "(a;b;c);"
+#define LIBM_DD_NO(name)
+#define LIBM_DDD_NO(name)
+#define LIBM_DDDD_NO(name)
 
 static const char* const jq_builtins =
+/* Include supported math functions first */
 #include "libm.h"
+/* Include jq-coded builtins next (some depend on math) */
 #include "src/builtin.inc"
-;
 
+/* Include unsupported math functions next */
+#undef LIBM_DDDD_NO
 #undef LIBM_DDD_NO
 #undef LIBM_DD_NO
+#undef LIBM_DDDD
+#undef LIBM_DDD
+#undef LIBM_DD
+#define LIBM_DD(name)
+#define LIBM_DDD(name)
+#define LIBM_DDDD(name)
+#define LIBM_DD_NO(name) "def " #name ": \"Error: " #name "() not found at build time\"|error;"
+#define LIBM_DDD_NO(name) "def " #name "(a;b): \"Error: " #name "() not found at build time\"|error;"
+#define LIBM_DDDD_NO(name) "def " #name "(a;b;c): \"Error: " #name "() not found at build time\"|error;"
+#include "libm.h"
+;
+
+#undef LIBM_DDDD_NO
+#undef LIBM_DDD_NO
+#undef LIBM_DD_NO
+#undef LIBM_DDDD
 #undef LIBM_DDD
 #undef LIBM_DD
 

src/builtin.jq

@@ -10,11 +10,11 @@ def min_by(f): _min_by_impl(map([f]));
 def add: reduce .[] as $x (null; . + $x);
 def del(f): delpaths([path(f)]);
 def _assign(paths; value): value as $v | reduce path(paths) as $p (.; setpath($p; $v));
-def _modify(paths; update): reduce path(paths) as $p (.; setpath($p; getpath($p) | update));
+def _modify(paths; update): reduce path(paths) as $p (.; label $out | (setpath($p; getpath($p) | update) | ., break $out), delpaths([$p]));
 def map_values(f): .[] |= f;
 
 # recurse
-def recurse(f): def r: ., (f | select(. != null) | r); r;
+def recurse(f): def r: ., (f | r); r;
 def recurse(f; cond): def r: ., (f | select(cond) | r); r;
 def recurse: recurse(.[]?);
 def recurse_down: recurse;
@@ -167,7 +167,7 @@ def until(cond; next):
          if cond then . else (next|_until) end;
      _until;
 def limit($n; exp): if $n < 0 then exp else label $out | foreach exp as $item ([$n, null]; if .[0] < 1 then break $out else [.[0] -1, $item] end; .[1]) end;
-def first(g): label $out | foreach g as $item ([false, null]; if .[0]==true then break $out else [true, $item] end; .[1]);
+def first(g): label $out | g | ., break $out;
 def last(g): reduce g as $item (null; $item);
 def nth($n; g): if $n < 0 then error("nth doesn't support negative indices") else last(limit($n + 1; g)) end;
 def first: .[0];
@@ -210,7 +210,7 @@ def ascii_upcase:
 
 # Streaming utilities
 def truncate_stream(stream):
-  . as $n | null | stream | . as $input | if (.[0]|length) > $n then setpath([0];$input[0][1:]) else empty end;
+  . as $n | null | stream | . as $input | if (.[0]|length) > $n then setpath([0];$input[0][$n:]) else empty end;
 def fromstream(i):
   foreach i as $item (
     [null,false,null,false];
@@ -287,3 +287,20 @@ def walk(f):
   elif type == "array" then map( walk(f) ) | f
   else f
   end;
+
+# SQL-ish operators here:
+def INDEX(stream; idx_expr):
+  reduce stream as $row ({};
+    .[$row|idx_expr|
+      if type != "string" then tojson
+      else .
+      end] |= $row);
+def INDEX(idx_expr): INDEX(.[]; idx_expr);
+def JOIN($idx; idx_expr):
+  [.[] | [., $idx[idx_expr]]];
+def JOIN($idx; stream; idx_expr):
+  stream | [., $idx[idx_expr]];
+def JOIN($idx; stream; idx_expr; join_expr):
+  stream | [., $idx[idx_expr]] | join_expr;
+def IN(s): reduce (first(select(. == s)) | true) as $v (false; if . or $v then true else false end);
+def IN(src; s): reduce (src|IN(s)) as $v (false; if . or $v then true else false end);

src/compile.c

@@ -704,12 +704,27 @@ static block bind_matcher(block matcher, block body) {
 
 block gen_reduce(block source, block matcher, block init, block body) {
   block res_var = gen_op_var_fresh(STOREV, "reduce");
+  block update_var = gen_op_bound(STOREV, res_var);
+  block jmp = gen_op_target(JUMP, body);
   block loop = BLOCK(gen_op_simple(DUPN),
                      source,
                      bind_matcher(matcher,
                                   BLOCK(gen_op_bound(LOADVN, res_var),
+                                        /*
+                                         * We fork to the body, jump to
+                                         * the STOREV.  This means that
+                                         * if body produces no results
+                                         * (i.e., it just does empty)
+                                         * then we keep the current
+                                         * reduction state as-is.
+                                         *
+                                         * To break out of a
+                                         * reduction... use break.
+                                         */
+                                        gen_op_target(FORK, jmp),
+                                        jmp,
                                         body,
-                                        gen_op_bound(STOREV, res_var))),
+                                        update_var)),
                      gen_op_simple(BACKTRACK));
   return BLOCK(gen_op_simple(DUP),
                init,

src/jv.h

@@ -99,6 +99,9 @@ jv jv_array_indexes(jv, jv);
   __attribute__ ((__format__( __printf__, fmt_arg_num, args_num)))
 #define JV_VPRINTF_LIKE(fmt_arg_num) \
   __attribute__ ((__format__( __printf__, fmt_arg_num, 0)))
+#else
+#define JV_PRINTF_LIKE(fmt_arg_num, args_num)
+#define JV_VPRINTF_LIKE(fmt_arg_num)
 #endif
 
 
@@ -112,8 +115,8 @@ const char* jv_string_value(jv);
 jv jv_string_indexes(jv j, jv k);
 jv jv_string_slice(jv j, int start, int end);
 jv jv_string_concat(jv, jv);
-jv jv_string_vfmt(const char*, va_list) JV_VPRINTF_LIKE(1);
-jv jv_string_fmt(const char*, ...) JV_PRINTF_LIKE(1, 2);
+jv jv_string_vfmt(const char* fmt, va_list ap) JV_VPRINTF_LIKE(1);
+jv jv_string_fmt(const char* fmt, ...) JV_PRINTF_LIKE(1, 2);
 jv jv_string_append_codepoint(jv a, uint32_t c);
 jv jv_string_append_buf(jv a, const char* buf, int len);
 jv jv_string_append_str(jv a, const char* str);
@@ -185,7 +188,7 @@ jv jv_object_iter_value(jv, int);
 
 int jv_get_refcnt(jv);
 
-enum jv_print_flags {
+enum {
   JV_PRINT_PRETTY   = 1,
   JV_PRINT_ASCII    = 2,
   JV_PRINT_COLOR    = 4,

src/jv_file.c

@@ -4,6 +4,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "jv.h"
+#include "jv_unicode.h"
 
 jv jv_load_file(const char* filename, int raw) {
   FILE* file = fopen(filename, "r");
@@ -20,11 +21,23 @@ jv jv_load_file(const char* filename, int raw) {
     data = jv_array();
     parser = jv_parser_new(0);
   }
+
+  // To avoid mangling UTF-8 multi-byte sequences that cross the end of our read
+  // buffer, we need to be able to read the remainder of a sequence and add that
+  // before appending.
+  const int max_utf8_len = 4;
+  char buf[4096+max_utf8_len];
   while (!feof(file) && !ferror(file)) {
-    char buf[4096];
-    size_t n = fread(buf, 1, sizeof(buf), file);
+    size_t n = fread(buf, 1, sizeof(buf)-max_utf8_len, file);
+    int len = 0;
+    if (jvp_utf8_backtrack(buf+(n-1), buf, &len) && len > 0) {
+      if (!feof(file) && !ferror(file)) {
+        n += fread(buf+n, 1, len, file);
+      }
+    }
+
     if (raw) {
-      data = jv_string_concat(data, jv_string_sized(buf, (int)n));
+      data = jv_string_append_buf(data, buf, n);
     } else {
       jv_parser_set_buf(parser, buf, n, !feof(file));
       jv value;

src/jv_parse.c

@@ -10,6 +10,10 @@
 
 typedef const char* presult;
 
+#ifndef MAX_PARSING_DEPTH
+#define MAX_PARSING_DEPTH (256)
+#endif
+
 #define TRY(x) do {presult msg__ = (x); if (msg__) return msg__; } while(0)
 #ifdef __GNUC__
 #define pfunc __attribute__((warn_unused_result)) presult
@@ -147,11 +151,13 @@ static void push(struct jv_parser* p, jv v) {
 static pfunc parse_token(struct jv_parser* p, char ch) {
   switch (ch) {
   case '[':
+    if (p->stackpos >= MAX_PARSING_DEPTH) return "Exceeds depth limit for parsing";
     if (jv_is_valid(p->next)) return "Expected separator between values";
     push(p, jv_array());
     break;
 
   case '{':
+    if (p->stackpos >= MAX_PARSING_DEPTH) return "Exceeds depth limit for parsing";
     if (jv_is_valid(p->next)) return "Expected separator between values";
     push(p, jv_object());
     break;

src/jv_print.c

@@ -13,6 +13,10 @@
 #include "jv_dtoa.h"
 #include "jv_unicode.h"
 
+#ifndef MAX_PRINT_DEPTH
+#define MAX_PRINT_DEPTH (256)
+#endif
+
 #define ESC "\033"
 #define COL(c) (ESC "[" c "m")
 #define COLRESET (ESC "[0m")
@@ -150,7 +154,9 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       }
     }
   }
-  switch (jv_get_kind(x)) {
+  if (indent > MAX_PRINT_DEPTH) {
+    put_str("<skipped: too deep>", F, S, flags & JV_PRINT_ISATTY);
+  } else switch (jv_get_kind(x)) {
   default:
   case JV_KIND_INVALID:
     if (flags & JV_PRINT_INVALID) {

src/jv_unicode.c

@@ -3,6 +3,29 @@
 #include "jv_unicode.h"
 #include "jv_utf8_tables.h"
 
+// jvp_utf8_backtrack returns the beginning of the last codepoint in the
+// string, assuming that start is the last byte in the string.
+// If the last codepoint is incomplete, returns the number of missing bytes via
+// *missing_bytes.  If there are no leading bytes or an invalid byte is
+// encountered, NULL is returned and *missing_bytes is not altered.
+const char* jvp_utf8_backtrack(const char* start, const char* min, int *missing_bytes) {
+  assert(min < start);
+  if (min == start) {
+    return min;
+  }
+  int length = 0;
+  int seen = 1;
+  while (start >= min && (length = utf8_coding_length[(unsigned char)*start]) == UTF8_CONTINUATION_BYTE) {
+    start--;
+    seen++;
+  }
+  if (length == 0 || length == UTF8_CONTINUATION_BYTE || length - seen < 0) {
+    return NULL;
+  }
+  if (missing_bytes) *missing_bytes = length - seen;
+  return start;
+}
+
 const char* jvp_utf8_next(const char* in, const char* end, int* codepoint_ret) {
   assert(in <= end);
   if (in == end) {

src/jv_unicode.h

@@ -1,6 +1,7 @@
 #ifndef JV_UNICODE_H
 #define JV_UNICODE_H
 
+const char* jvp_utf8_backtrack(const char* start, const char* min, int *missing_bytes);
 const char* jvp_utf8_next(const char* in, const char* end, int* codepoint);
 int jvp_utf8_is_valid(const char* in, const char* end);
 

src/libm.h

@@ -149,3 +149,153 @@ LIBM_DID(jn)
 #ifdef HAVE_YN
 LIBM_DID(yn)
 #endif
+#ifdef HAVE_CEIL
+LIBM_DD(ceil)
+#else
+LIBM_DD_NO(ceil)
+#endif
+#ifdef HAVE_COPYSIGN
+LIBM_DDD(copysign)
+#else
+LIBM_DDD_NO(copysign)
+#endif
+#ifdef HAVE_DREM
+LIBM_DDD(drem)
+#else
+LIBM_DDD_NO(drem)
+#endif
+#ifdef HAVE_ERF
+LIBM_DD(erf)
+#else
+LIBM_DD_NO(erf)
+#endif
+#ifdef HAVE_ERFC
+LIBM_DD(erfc)
+#else
+LIBM_DD_NO(erfc)
+#endif
+#ifdef HAVE_EXP10
+LIBM_DD(exp10)
+#else
+LIBM_DD_NO(exp10)
+#endif
+#ifdef HAVE_EXPM1
+LIBM_DD(expm1)
+#else
+LIBM_DD_NO(expm1)
+#endif
+#ifdef HAVE_FABS
+LIBM_DD(fabs)
+#else
+LIBM_DD_NO(fabs)
+#endif
+#ifdef HAVE_FDIM
+LIBM_DDD(fdim)
+#else
+LIBM_DDD_NO(fdim)
+#endif
+#ifdef HAVE_FMA
+LIBM_DDDD(fma)
+#else
+LIBM_DDDD_NO(fma)
+#endif
+#ifdef HAVE_FMAX
+LIBM_DDD(fmax)
+#else
+LIBM_DDD_NO(fmax)
+#endif
+#ifdef HAVE_FMIN
+LIBM_DDD(fmin)
+#else
+LIBM_DDD_NO(fmin)
+#endif
+#ifdef HAVE_FMOD
+LIBM_DDD(fmod)
+#else
+LIBM_DDD_NO(fmod)
+#endif
+#ifdef HAVE_FREXP
+LIBM_DDD(frexp)
+#else
+LIBM_DDD_NO(frexp)
+#endif
+#ifdef HAVE_GAMMA
+LIBM_DD(gamma)
+#else
+LIBM_DD_NO(gamma)
+#endif
+#ifdef HAVE_LGAMMA
+LIBM_DD(lgamma)
+#else
+LIBM_DD_NO(lgamma)
+#endif
+#ifdef HAVE_LOG1P
+LIBM_DD(log1p)
+#else
+LIBM_DD_NO(log1p)
+#endif
+#ifdef HAVE_LOGB
+LIBM_DD(logb)
+#else
+LIBM_DD_NO(logb)
+#endif
+#ifdef HAVE_MODF
+LIBM_DDD(modf)
+#else
+LIBM_DDD_NO(modf)
+#endif
+#ifdef HAVE_NEARBYINT
+LIBM_DD(nearbyint)
+#else
+LIBM_DD_NO(nearbyint)
+#endif
+#ifdef HAVE_NEXTAFTER
+LIBM_DDD(nextafter)
+#else
+LIBM_DDD_NO(nextafter)
+#endif
+#ifdef HAVE_NEXTTOWARD
+LIBM_DDD(nexttoward)
+#else
+LIBM_DDD_NO(nexttoward)
+#endif
+#ifdef HAVE_POW10
+LIBM_DD(pow10)
+#else
+LIBM_DD_NO(pow10)
+#endif
+#ifdef HAVE_RINT
+LIBM_DD(rint)
+#else
+LIBM_DD_NO(rint)
+#endif
+#ifdef HAVE_ROUND
+LIBM_DD(round)
+#else
+LIBM_DD_NO(round)
+#endif
+#ifdef HAVE_SCALB
+LIBM_DDD(scalb)
+#else
+LIBM_DDD_NO(scalb)
+#endif
+#ifdef HAVE_SCALBLN
+LIBM_DDD(scalbln)
+#else
+LIBM_DDD_NO(scalbln)
+#endif
+#ifdef HAVE_SIGNIFICAND
+LIBM_DD(significand)
+#else
+LIBM_DD_NO(significand)
+#endif
+#ifdef HAVE_TRUNC
+LIBM_DD(trunc)
+#else
+LIBM_DD_NO(trunc)
+#endif
+#ifdef HAVE_LDEXP
+LIBM_DDD(ldexp)
+#else
+LIBM_DDD_NO(ldexp)
+#endif

tests/jq.test

@@ -303,6 +303,11 @@ null
 10
 [0,9,0,5,"nth doesn't support negative indices"]
 
+# Check that first(g) does not extract more than one value from g
+first(1,error("foo"))
+null
+1
+
 #
 # Check that various builtins evalute all arguments where appropriate,
 # doing cartesian products where appropriate.
@@ -686,6 +691,11 @@ reduce [[1,2,10], [3,4,10]][] as [$i,$j] (0; . + $i * $j)
 null
 14
 
+# Test fix for #1313 (reduce should handle empty updates)
+reduce range(5) as $n ([]; select($n%2 == 1) | . + [$n])
+null
+[1,3]
+
 . as $dot|any($dot[];not)
 [1,2,3,4,true,false,1,2,3,4,5]
 true
@@ -1044,7 +1054,13 @@ sort
 [42,[2,5,3,11],10,{"a":42,"b":2},{"a":42},true,2,[2,6],"hello",null,[2,5,6],{"a":[],"b":1},"abc","ab",[3,10],{},false,"abcd",null]
 [null,null,false,true,2,10,42,"ab","abc","abcd","hello",[2,5,3,11],[2,5,6],[2,6],[3,10],{},{"a":42},{"a":42,"b":2},{"a":[],"b":1}]
 
-# Tests of `sort_by` and `group_by` have been moved to tests/sort.test
+(sort_by(.b) | sort_by(.a)), sort_by(.a, .b), sort_by(.b, .c), group_by(.b), group_by(.a + .b - .c == 2)
+[{"a": 1, "b": 4, "c": 14}, {"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}, {"a": 0, "b": 2, "c": 43}]
+[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}]
+[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}]
+[{"a": 4, "b": 1, "c": 3}, {"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 3}, {"a": 1, "b": 4, "c": 14}]
+[[{"a": 4, "b": 1, "c": 3}], [{"a": 0, "b": 2, "c": 43}], [{"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}]]
+[[{"a": 1, "b": 4, "c": 14}, {"a": 0, "b": 2, "c": 43}], [{"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}]]
 
 unique
 [1,2,5,3,5,3,1,3]
@@ -1306,4 +1322,32 @@ jq: error: syntax error, unexpected INVALID_CHARACTER, expecting $end (Unix shel
 (.[{}] = 0)?
 null
 
+INDEX(range(5)|[., "foo\(.)"]; .[0])
+null
+{"0":[0,"foo0"],"1":[1,"foo1"],"2":[2,"foo2"],"3":[3,"foo3"],"4":[4,"foo4"]}
+
+JOIN({"0":[0,"abc"],"1":[1,"bcd"],"2":[2,"def"],"3":[3,"efg"],"4":[4,"fgh"]}; .[0]|tostring)
+[[5,"foo"],[3,"bar"],[1,"foobar"]]
+[[[5,"foo"],null],[[3,"bar"],[3,"efg"]],[[1,"foobar"],[1,"bcd"]]]
+
+range(5;10)|IN(range(10))
+null
+true
+true
+true
+true
+true
+
+range(10;12)|IN(range(10))
+null
+false
+false
+
+IN(range(10;20); range(10))
+null
+false
+
+IN(range(5;20); range(10))
+null
+true
 

tests/optional.test

@@ -1,16 +1,5 @@
 # See tests/jq.test and the jq manual for more information.
 
-# As long as we use the C qsort() function, this test can fail if the C
-# qsort() implementation we get is not stable (Quicksort is not stable).
-
-(sort_by(.b) | sort_by(.a)), sort_by(.a, .b), sort_by(.b, .c), group_by(.b), group_by(.a + .b - .c == 2)
-[{"a": 1, "b": 4, "c": 14}, {"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}, {"a": 0, "b": 2, "c": 43}]
-[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}]
-[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}]
-[{"a": 4, "b": 1, "c": 3}, {"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 3}, {"a": 1, "b": 4, "c": 14}]
-[[{"a": 4, "b": 1, "c": 3}], [{"a": 0, "b": 2, "c": 43}], [{"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}]]
-[[{"a": 1, "b": 4, "c": 14}, {"a": 0, "b": 2, "c": 43}], [{"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}]]
-
 # strptime() is not available on mingw/WIN32
 [strptime("%Y-%m-%dT%H:%M:%SZ")|(.,mktime)]
 "2015-03-05T23:51:47Z"

tests/utf8-truncate.jq

@@ -0,0 +1,3 @@
+def fill:"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+def e:"日本語";
+e == "日本語"

tests/utf8test

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+. "${0%/*}/setup" "$@"
+
+if [ "`$VALGRIND $Q $JQ -nf $JQTESTDIR/utf8-truncate.jq`" != "true" ]; then
+	echo "UTF-8 byte sequences that span the jv_load_file read buffer are mangled"
+	exit 1
+fi
+
+exit 0