fixup! Configure appveyor build to use python

2026-10-31 11:05:54 +00:00 · 2019-02-22 20:05:09 -05:00 · 2019-02-22 20:02:44 -05:00 · 2019-02-22 19:53:14 -05:00 · 2019-02-22 19:50:26 -05:00 · 2019-02-22 19:50:26 -05:00
50 changed files with 2380 additions and 1704 deletions
@@ -17,6 +17,7 @@ jobs:
            stage: test
            os: linux
            compiler: gcc
+            python: 3.6

            addons:
                apt:
@@ -27,13 +28,13 @@ jobs:

            before_install:
                - uname -s
-                - rvm install ruby-1.9.3-p551
-                - rvm use 1.9.3
                - rm src/{lexer,parser}.{c,h}
                - sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac

            install:
-                - bundle install --gemfile=docs/Gemfile
+                - pyenv global 3.6
+                - pip3 install pipenv
+                - pushd docs && pipenv sync && popd
                - wget http://ftp.debian.org/debian/pool/main/b/bison/bison_3.0.2.dfsg-2_amd64.deb
                - ar p bison_3.0.2.dfsg-2_amd64.deb data.tar.xz | tar xJ
                - if [ -n "$COVERAGE" ]; then pip install --user cpp-coveralls; fi
@@ -104,14 +105,12 @@ jobs:
                - uname -s
                - brew update
                - brew install flex bison
-                - rvm install ruby-1.9.3-p551
-                - rvm use 1.9.3
-                - gem install bundler
                - rm src/{lexer,parser}.{c,h}
                - sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac

            install:
-                - bundle install --gemfile=docs/Gemfile
+                - pip3 install pipenv
+                - pushd docs && pipenv sync && popd
                - if [ -n "$COVERAGE" ]; then pip install --user cpp-coveralls; fi


@@ -136,7 +135,9 @@ jobs:

            install:
                - sudo sh alpine-chroot-install -b v3.7 -a "$ARCH"
-                    -p 'build-base automake autoconf bison libtool oniguruma-dev'
+                    -p 'build-base automake autoconf bison git libtool oniguruma-dev python3 python3-dev libxml2-dev libxslt-dev'
+                - /alpine/enter-chroot pip3 install pipenv
+                - alpine sh -c 'cd docs && pipenv sync'

            before_script:
                - autoreconf -if
@@ -146,6 +147,9 @@ jobs:
                - alpine make
                - alpine make check

+            after_failure:
+                - cat test-suite.log
+                - cat tests/*.log

        # Build release binary statically linked with musl libc on Alpine Linux
        # (inside chroot). If building a tagged commit, then deploy release
@@ -13,9 +13,8 @@ RUN apt-get update && \
        git \
        bison \
        flex \
-        ruby \
-        wget \
-        ruby-dev && \
+        python3 \
+        wget && \
    wget https://github.com/kkos/oniguruma/archive/v5.9.6.tar.gz -O onig-5.9.6.tar.gz && \
    sha512sum onig-5.9.6.tar.gz | grep 6b048d345e148c9da38af7a8df76d4a358eb3d4db91fa7834076e511f526a63544284f212b0d56badbbd33112c8b458a5fff02bfbbda012ecfe478bc436ea679 && \
    tar zxvf onig-5.9.6.tar.gz && \
@@ -25,8 +24,8 @@ RUN apt-get update && \
        ./configure --prefix=/usr/local && \
        make && \
        make install ) && \
-    gem install bundler && \
-    (cd /app/docs && bundle install) && \
+    pip3 install pipenv && \
+    (cd /app/docs && pipenv sync) && \
    (cd /app && \
        autoreconf -i && \
        ./configure --disable-valgrind --enable-all-static --prefix=/usr/local && \
@@ -43,8 +42,7 @@ RUN apt-get update && \
        bison \
        git \
        flex \
-        ruby \
-        ruby-dev && \
+        python3 && \
    apt-get autoremove -y && \
    rm -rf oniguruma-5.9.6 && \
    rm -rf /var/lib/apt/lists/* /var/lib/gems
@@ -18,6 +18,10 @@ LIBJQ_SRC = src/builtin.c src/bytecode.c src/compile.c src/execute.c    \
 AM_CFLAGS = -Wextra -Wall -Wno-missing-field-initializers               \
        -Wno-unused-parameter -Wno-unused-function

+if WIN32
+AM_CFLAGS += -municode
+endif
+
 ACLOCAL_AMFLAGS = -I config/m4

 ### Generating the lexer and parser
@@ -52,6 +56,7 @@ libjq_la_LDFLAGS = $(onig_LDFLAGS) -export-symbols-regex '^j[qv]_' -version-info

 if WIN32
 libjq_la_LIBADD += -lshlwapi
+libjq_la_LDFLAGS += -no-undefined
 endif

 include_HEADERS = src/jv.h src/jq.h
@@ -60,6 +65,8 @@ if ENABLE_UBSAN
 AM_CFLAGS += -fsanitize=undefined
 endif

+AM_CPPFLAGS = -I$(srcdir)/src
+
 ### Running tests under Valgrind

 if ENABLE_ASAN
@@ -97,11 +104,13 @@ generate_ver = ver="`{ $(srcdir)/scripts/version || echo '$(VERSION)' ; } | sed
 .remake-version-h: .FORCE
 	@ $(generate_ver); test "x`cat src/version.h 2>/dev/null`" = "x$$ver" || touch .remake-version-h
 src/version.h: .remake-version-h
+	mkdir -p src
 	$(AM_V_GEN) $(generate_ver); echo "$$ver" > $@
 src/main.c: src/version.h

 src/builtin.inc: src/builtin.jq
-	$(AM_V_GEN) sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e 's/^/"/' -e 's/$$/\\n"/' $^ > $@
+	mkdir -p src
+	$(AM_V_GEN) sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e 's/^/"/' -e 's/$$/\\n"/' src/builtin.jq > $@
 src/builtin.o: src/builtin.inc

 bin_PROGRAMS = jq
@@ -127,13 +136,13 @@ TESTS_ENVIRONMENT = NO_VALGRIND=$(NO_VALGRIND)

 man_MANS = jq.1
 if ENABLE_DOCS
-jq.1: $(srcdir)/docs/content/3.manual/manual.yml
-	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
+jq.1: $(srcdir)/docs/content/manual/manual.yml
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; pipenv run python build_manpage.py ) > $@ || { rm -f $@; false; }
 jq.1.prebuilt: jq.1
-	$(AM_V_GEN) cp $^ $@ || { rm -f $@; false; }
+	$(AM_V_GEN) cp jq.1 $@ || { rm -f $@; false; }
 else
 jq.1: $(srcdir)/jq.1.prebuilt
-	$(AM_V_GEN) cp $^ $@
+	$(AM_V_GEN) cp $(srcdir)/jq.1.prebuilt $@
 endif


@@ -156,8 +165,8 @@ install-binaries: $(BUILT_SOURCES)
 	$(MAKE) $(AM_MAKEFLAGS) install-exec

 DOC_FILES = docs/content docs/public docs/templates docs/site.yml       \
-        docs/Gemfile docs/Gemfile.lock docs/Rakefile docs/README.md     \
-        jq.1.prebuilt
+        docs/Pipfile docs/Pipfile.lock docs/build_manpage.py            \
+        docs/build_manpage.py docs/README.md jq.1.prebuilt

 EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
        jq.1.prebuilt jq.spec src/lexer.c src/lexer.h src/parser.c      \
@@ -171,11 +180,11 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
        tests/modules/test_bind_order0.jq                               \
        tests/modules/test_bind_order1.jq                               \
        tests/modules/test_bind_order2.jq tests/onig.supp               \
-        tests/onig.test tests/setup tests/torture/input0.json		\
-        tests/optional.test tests/optionaltest				\
-	tests/utf8-truncate.jq tests/utf8test				\
-	tests/base64.test tests/base64test 				\
-	tests/jq-f-test.sh tests/shtest
+        tests/onig.test tests/setup tests/torture/input0.json           \
+        tests/optional.test tests/optionaltest                          \
+        tests/utf8-truncate.jq tests/utf8test                           \
+        tests/base64.test tests/base64test                              \
+        tests/jq-f-test.sh tests/shtest



@@ -1,6 +1,7 @@

 Release history

+ * jq version 1.6 was released on Fri Nov 2 2018
 * jq version 1.5 was released on Sat Aug 15 2015
 * jq version 1.4 was released on Mon Jun 9 2014
 * jq version 1.3 was released on Sun May 19 2013
@@ -8,6 +9,29 @@ Release history
 * jq version 1.1 was released on Sun Oct 21 2012
 * jq version 1.0 was released on Sun Oct 21 2012

+New features in 1.6 since 1.5:
+
+ - Destructuring Alternation
+
+ - New Builtins:
+   - builtins/0
+   - stderr/0
+   - halt/0, halt_error/1
+   - isempty/1
+   - walk/1
+   - utf8bytelength/1
+   - localtime/0, strflocaltime/1
+   - SQL-style builtins
+   - and more!
+
+ - Add support for ASAN and UBSAN
+
+ - Make it easier to use jq with shebangs (8f6f28c)
+
+ - Add $ENV builtin variable to access environment
+
+ - Add JQ_COLORS env var for configuring the output colors
+
 New features in 1.5 since 1.4:

 - regular expressions (with Oniguruma)
@@ -23,7 +23,7 @@ Source tarball and built executable releases can be found on the
 homepage and on the github release page, https://github.com/stedolan/jq/releases

 If you're building directly from the latest git, you'll need flex,
-bison (3.0 or newer), libtool, make, and autoconf installed.
+bison (3.0 or newer), libtool, make, automake, and autoconf installed.
 To get regexp support you'll also need to install Oniguruma or clone it as a
 git submodule as per the instructions below.
 (note that jq's tests require regexp support to pass).  To build, run:
@@ -1,26 +1,28 @@
 environment:
  matrix:
  - MSYSTEM: MINGW64
-    PATH: C:\msys64\usr\bin;C:\msys64\mingw64\bin;C:\Windows\System32;C:\Windows;%PATH%
+    PATH: C:\Python36;C:\Python36\Scripts;C:\msys64\usr\bin;C:\msys64\mingw64\bin;C:\Windows\System32;C:\Windows;%PATH%
  - MSYSTEM: MINGW32
-    PATH: C:\msys64\usr\bin;C:\msys64\mingw32\bin;C:\Windows\System32;C:\Windows;%PATH%
+    PATH: C:\Python36;C:\Python36\Scripts;C:\msys64\usr\bin;C:\msys64\mingw32\bin;C:\Windows\System32;C:\Windows;%PATH%

 clone_script:
-  - bash -lc "git clone -q --branch=$APPVEYOR_REPO_BRANCH https://github.com/${APPVEYOR_REPO_NAME}.git $APPVEYOR_BUILD_FOLDER"
-  - bash -lc "cd $APPVEYOR_BUILD_FOLDER && git checkout -qf $APPVEYOR_REPO_BRANCH"
+  - bash -lc "echo $APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME - $APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH"
+  - bash -lc "git clone -q --branch=${APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH:-$APPVEYOR_REPO_BRANCH} https://github.com/${APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME:-$APPVEYOR_REPO_NAME}.git $APPVEYOR_BUILD_FOLDER"
+  - bash -lc "cd $APPVEYOR_BUILD_FOLDER && git checkout -qf ${APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH:APPVEYOR_REPO_BRANCH}"
  - bash -lc "cd $APPVEYOR_BUILD_FOLDER && git submodule update --init --recursive"

 install:
-  # update mysy2
+  # update msys2
  - C:\msys64\usr\bin\bash -lc "pacman --needed --noconfirm -Sy pacman-mirrors"
-  - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -Sy"
-  - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S autoconf automake bison flex"
-  - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S git"
+  - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -Su"
+  - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S autoconf automake bison flex git"
+  - C:\msys64\usr\bin\bash -lc "pip3 install --upgrade pipenv"
+  - C:\msys64\usr\bin\bash -lc "cd $APPVEYOR_BUILD_FOLDER/docs && ls && pipenv sync"

 build_script:
  - bash -lc "exec 0</dev/null && cd $APPVEYOR_BUILD_FOLDER && autoreconf -i"
  - bash -lc "exec 0</dev/null && cd $APPVEYOR_BUILD_FOLDER && ./configure --with-oniguruma=builtin --disable-shared --enable-static --enable-all-static"
-  - bash -lc "exec 0</dev/null && cd $APPVEYOR_BUILD_FOLDER && make -j8"
+  - bash -lc "exec 0</dev/null && cd $APPVEYOR_BUILD_FOLDER && make"
  - bash -lc "exec 0</dev/null && cd $APPVEYOR_BUILD_FOLDER && cat jq.1  | groff -mandoc -Thtml > jq.html"
  - 7z a jq-package.zip jq.1 jq.html jq.exe
  - bash -lc "exec 0</dev/null && cd $APPVEYOR_BUILD_FOLDER && pwd && ls -la"
@@ -74,7 +74,7 @@ dnl Code coverage
 AC_ARG_ENABLE([gcov],
   AC_HELP_STRING([--enable-gcov], [enable gcov code coverage tool]))

-dnl Don't attempt to build docs if there's no Ruby lying around
+dnl Don't attempt to build docs if python deps aren't installed
 AC_ARG_ENABLE([docs],
   AC_HELP_STRING([--disable-docs], [don't build docs]))

@@ -87,25 +87,25 @@ AC_ARG_ENABLE([all-static],
   AC_HELP_STRING([--enable-all-static], [link jq with static libraries only]))

 AS_IF([test "x$enable_docs" != "xno"],[
-   AC_CHECK_PROGS(bundle_cmd, bundle)
+   AC_CHECK_PROGS(pipenv_cmd, pipenv)

-   AC_CACHE_CHECK([for Ruby dependencies], [jq_cv_ruby_deps],
-     [jq_cv_ruby_deps=yes;
-        AS_IF([test "x$bundle_cmd" = "x" || \
-        ! bmsg="`cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null`"],[
-           AC_MSG_WARN([$bmsg])
+   AC_CACHE_CHECK([for Python dependencies], [jq_cv_python_deps],
+     [jq_cv_python_deps=yes;
+        AS_IF([test "x$pipenv_cmd" = "x" || \
+        ! bmsg="`cd ${srcdir}/docs; LC_ALL=$LANG "$pipenv_cmd" check`"],[
+           AC_MSG_ERROR([$bmsg])
           cat <<EOF
 *****************************************************************
-*  Ruby dependencies for building jq documentation not found.   *
+*  Python dependencies for building jq documentation not found. *
 *  You can still build, install and hack on jq, but the manpage *
 *  will not be rebuilt and some of the tests will not run.      *
 *  See docs/README.md for how to install the docs dependencies. *
 *****************************************************************
 EOF
-           jq_cv_ruby_deps=no
+           jq_cv_python_deps=no
     ])])

-   if test "x$jq_cv_ruby_deps" != "xyes"; then
+   if test "x$jq_cv_python_deps" != "xyes"; then
     enable_docs=no
   fi
 ])
@@ -124,6 +124,7 @@ AC_FIND_FUNC([isatty], [c], [#include <unistd.h>], [0])
 AC_FIND_FUNC([_isatty], [c], [#include <io.h>], [0])
 AC_FIND_FUNC([strptime], [c], [#include <time.h>], [0, 0, 0])
 AC_FIND_FUNC([strftime], [c], [#include <time.h>], [0, 0, 0, 0])
+AC_FIND_FUNC([setenv], [c], [#include <stdlib.h>], [0, 0, 0])
 AC_FIND_FUNC([timegm], [c], [#include <time.h>], [0])
 AC_FIND_FUNC([gmtime_r], [c], [#include <time.h>], [0, 0])
 AC_FIND_FUNC([gmtime], [c], [#include <time.h>], [0])
@@ -1,6 +1,3 @@
 .DS_Store
 .sass-cache
 output/*
-
-# Autogenerated from public/css/base.scss
-public/css/base.css
@@ -1,7 +0,0 @@
-source "https://rubygems.org"
-
-gem "rake"
-gem "bonsai"
-gem "maruku"
-gem "ronn"
-gem "mustache", "<1.0" # to support ruby <2.0
@@ -1,63 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activesupport (4.2.3)
-      i18n (~> 0.7)
-      json (~> 1.7, >= 1.7.7)
-      minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
-      tzinfo (~> 1.1)
-    addressable (2.3.8)
-    bonsai (1.4.9)
-      activesupport (>= 3.0.3)
-      builder (>= 3.0.0)
-      i18n (>= 0.5.0)
-      launchy (>= 0.3.7)
-      liquid (>= 2.2.2)
-      maruku (>= 0.6.0)
-      rack
-      sass
-      sinatra (>= 1.0)
-      tilt (>= 1.3)
-      watch (>= 0.1.0)
-      yui-compressor
-    builder (3.2.2)
-    hpricot (0.8.6)
-    i18n (0.7.0)
-    json (1.8.3)
-    launchy (2.4.3)
-      addressable (~> 2.3)
-    liquid (3.0.6)
-    maruku (0.7.2)
-    minitest (5.8.0)
-    mustache (0.99.8)
-    rack (1.6.4)
-    rack-protection (1.5.3)
-      rack
-    rake (10.4.2)
-    rdiscount (2.1.8)
-    ronn (0.7.3)
-      hpricot (>= 0.8.2)
-      mustache (>= 0.7.0)
-      rdiscount (>= 1.5.8)
-    sass (3.4.16)
-    sinatra (1.4.6)
-      rack (~> 1.4)
-      rack-protection (~> 1.4)
-      tilt (>= 1.3, < 3)
-    thread_safe (0.3.5)
-    tilt (2.0.1)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
-    watch (0.1.0)
-    yui-compressor (0.12.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bonsai
-  maruku
-  mustache (< 1.0)
-  rake
-  ronn
@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+jinja2 = "*"
+pyyaml = "*"
+markdown = "*"
+lxml = "*"
@@ -0,0 +1,117 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "16a9ef323592a417142c30be17e58c3cf36dc2bfdbf4757106a796b22262a1ce"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "jinja2": {
+            "hashes": [
+                "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd",
+                "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"
+            ],
+            "index": "pypi",
+            "version": "==2.10"
+        },
+        "lxml": {
+            "hashes": [
+                "sha256:0537eee4902e8bf4f41bfee8133f7edf96533dd175930a12086d6a40d62376b2",
+                "sha256:0562ec748abd230ab87d73384e08fa784f9b9cee89e28696087d2d22c052cc27",
+                "sha256:09e91831e749fbf0f24608694e4573be0ef51430229450c39c83176cc2e2d353",
+                "sha256:1ae4c0722fc70c0d4fba43ae33c2885f705e96dce1db41f75ae14a2d2749b428",
+                "sha256:1c630c083d782cbaf1f7f37f6cac87bda9cff643cf2803a5f180f30d97955cef",
+                "sha256:2fe74e3836bd8c0fa7467ffae05545233c7f37de1eb765cacfda15ad20c6574a",
+                "sha256:37af783c2667ead34a811037bda56a0b142ac8438f7ed29ae93f82ddb812fbd6",
+                "sha256:3f2d9eafbb0b24a33f56acd16f39fc935756524dcb3172892721c54713964c70",
+                "sha256:47d8365a8ef14097aa4c65730689be51851b4ade677285a3b2daa03b37893e26",
+                "sha256:510e904079bc56ea784677348e151e1156040dbfb736f1d8ea4b9e6d0ab2d9f4",
+                "sha256:58d0851da422bba31c7f652a7e9335313cf94a641aa6d73b8f3c67602f75b593",
+                "sha256:7940d5c2185ffb989203dacbb28e6ae88b4f1bb25d04e17f94b0edd82232bcbd",
+                "sha256:7cf39bb3a905579836f7a8f3a45320d9eb22f16ab0c1e112efb940ced4d057a5",
+                "sha256:9563a23c1456c0ab550c087833bc13fcc61013a66c6420921d5b70550ea312bf",
+                "sha256:95b392952935947e0786a90b75cc33388549dcb19af716b525dae65b186138fc",
+                "sha256:983129f3fd3cef5c3cf067adcca56e30a169656c00fcc6c648629dbb850b27fa",
+                "sha256:a0b75b1f1854771844c647c464533def3e0a899dd094a85d1d4ed72ecaaee93d",
+                "sha256:b5db89cc0ef624f3a81214b7961a99f443b8c91e88188376b6b322fd10d5b118",
+                "sha256:c0a7751ba1a4bfbe7831920d98cee3ce748007eab8dfda74593d44079568219a",
+                "sha256:c0c5a7d4aafcc30c9b6d8613a362567e32e5f5b708dc41bc3a81dac56f8af8bb",
+                "sha256:d4d63d85eacc6cb37b459b16061e1f100d154bee89dc8d8f9a6128a5a538e92e",
+                "sha256:da5e7e941d6e71c9c9a717c93725cda0708c2474f532e3680ac5e39ec57d224d",
+                "sha256:dccad2b3c583f036f43f80ac99ee212c2fa9a45151358d55f13004d095e683b2",
+                "sha256:df46307d39f2aeaafa1d25309b8a8d11738b73e9861f72d4d0a092528f498baa",
+                "sha256:e70b5e1cb48828ddd2818f99b1662cb9226dc6f57d07fc75485405c77da17436",
+                "sha256:ea825562b8cd057cbc9810d496b8b5dec37a1e2fc7b27bc7c1e72ce94462a09a"
+            ],
+            "index": "pypi",
+            "version": "==4.3.1"
+        },
+        "markdown": {
+            "hashes": [
+                "sha256:c00429bd503a47ec88d5e30a751e147dcb4c6889663cd3e2ba0afe858e009baa",
+                "sha256:d02e0f9b04c500cde6637c11ad7c72671f359b87b9fe924b2383649d8841db7c"
+            ],
+            "index": "pypi",
+            "version": "==3.0.1"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432",
+                "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b",
+                "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9",
+                "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af",
+                "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834",
+                "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd",
+                "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d",
+                "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7",
+                "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b",
+                "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3",
+                "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c",
+                "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2",
+                "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7",
+                "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36",
+                "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1",
+                "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e",
+                "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1",
+                "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c",
+                "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856",
+                "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550",
+                "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492",
+                "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672",
+                "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401",
+                "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6",
+                "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6",
+                "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c",
+                "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd",
+                "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1"
+            ],
+            "version": "==1.1.0"
+        },
+        "pyyaml": {
+            "hashes": [
+                "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b",
+                "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf",
+                "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a",
+                "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3",
+                "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1",
+                "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1",
+                "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613",
+                "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04",
+                "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f",
+                "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537",
+                "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531"
+            ],
+            "index": "pypi",
+            "version": "==3.13"
+        }
+    },
+    "develop": {}
+}
@@ -2,24 +2,19 @@ Documentation
 =============

 The jq website, manpages and some of the tests are generated from this
-directory. The directory holds a [Bonsai](http://tinytree.info)
-website, and the manual is a YAML file in `content/3.manual`.
+directory. The manual is a YAML file in `content/manual`.

 To build the documentation (including building the jq manpage), you'll
-need a working Ruby setup. The easiest way to get one is to install
-RVM and Ruby 1.9.3 like so:
+need python3 and pipenv. You can install pipenv like so:

-    \curl -L https://get.rvm.io | bash -s stable --ruby=1.9.3
+    pip install pipenv

-After that finishes installing, you'll need to make sure RVM is on
-your path by doing `source $HOME/.rvm/scripts/rvm`, or just opening a
-new shell. See <https://rvm.io> for more info on RVM.
+Though, you may need to say pip3 instead, depending on your system. Once
+you have pipenv installed, you can install the dependencies by running
+`pipenv install` from the `docs` directory.

-Once RVM is installed, you can install all the dependencies for jq's
-documentation build by running this from the `docs` directory:
+Once this is done, rerun `./configure` in the jq root directory and then
+the Makefile will be able to generate the jq manpage.

-    bundle install
-
-When bundle manages to install the dependencies, rerun `./configure`
-in the jq root directory and then the Makefile will be able to
-generate the jq manpage.
+To build the website, run `pipenv run ./build_website.py` from inside
+the `docs` directory.
@@ -1,24 +0,0 @@
-current_dir = File.dirname(__FILE__)
-rakefile_manual = File.expand_path(File.join(current_dir, "Rakefile.manual"))
-rakefile_website = File.expand_path(File.join(current_dir, "Rakefile.website"))
-
-desc "Build the manpage from the bonsai source of the manual"
-task :manpage do
-    system %(#{$0} -f #{rakefile_manual} manpage)
-end
-
-desc "Collect jq unit test cases from the bonsai source of the manual"
-task :mantests do
-    system %(#{$0} -f #{rakefile_manual} mantests)
-end
-
-desc "Build the website from the bonsai sources"
-task :build do
-    system %(#{$0} -f #{rakefile_website} build)
-end
-
-desc "Serve a live view of the website on http://localhost:5000/jq/"
-task :serve do
-    system %(#{$0} -f #{rakefile_website} serve)
-end
-
@@ -1,49 +0,0 @@
-require 'ronn'
-require 'tempfile'
-require 'yaml'
-
-def load_manual
-  YAML::load(File.open("content/3.manual/manual.yml"))
-end
-
-task :manpage do
-  Tempfile.open "manpage" do |f|
-    manual = load_manual
-    f.puts manual['manpage_intro']
-    f.puts manual['body']
-    manual['sections'].each do |section|
-
-      f.puts "## #{section['title'].upcase}\n"
-      f.puts section['body']
-      f.puts ""
-      (section['entries'] || []).each do |entry|
-        f.puts "### #{entry['title']}\n"
-        f.puts entry['body']
-        f.puts ""
-        (entry['examples'] || []).each do |example|
-          f.puts "    jq '#{example['program']}'"
-          f.puts "       #{example['input']}"
-          f.puts "    => #{example['output'].join(", ")}"
-          f.puts
-        end
-      end
-      f.puts ""
-    end
-    f.puts manual['manpage_epilogue']
-    f.close
-    puts Ronn::Document.new(f.path).convert('roff').gsub(/<\/?code>/,"")
-  end
-end
-
-task :mantests do
-  load_manual['sections'].each do |section|
-    (section['entries'] || []).each do |entry|
-      (entry['examples'] || []).each do |example|
-        puts example['program'].gsub("\n", " ")
-        puts example['input']
-        example['output'].each do |s| puts s end
-        puts
-      end
-    end
-  end
-end
@@ -1,76 +0,0 @@
-require 'bonsai'
-require 'json'
-require 'liquid'
-require 'maruku'
-
-module ExtraFilters
-  def markdownify(input)
-    Maruku.new(input).to_html
-  end
-  def search_id(input)
-    input.gsub(/`/, '')
-  end
-  def section_id(input)
-    input.gsub(/[^a-zA-Z0-9_]/, '')
-  end
-  def entry_id(input)
-    input.gsub(/[ `]/, '')
-  end
-  def no_paragraph(input)
-    input.gsub('<p>', '').gsub('</p>', '')
-  end
-  def json(input)
-    input.to_json
-  end
-  def unique(input)
-    @n = (@n || 0) + 1
-    input + @n.to_s
-  end
-end
-
-Liquid::Template.register_filter(ExtraFilters)
-
-begin
-  `java 2>&1`
-rescue
-  class Bonsai::Exporter
-    def self.compress_assets
-      Bonsai.log "java not found! Not compressing javascript or stylesheets"
-    end
-  end
-end
-
-task :build do
-  Bonsai.root_dir = Dir.pwd
-  Bonsai::Exporter.publish!
-end
-
-task :serve do
-  begin
-    Bonsai.log "Press Control+C to quit"
-
-    require 'rack'
-    require 'sinatra'
-    require 'watch'
-    require 'launchy'
-
-    Bonsai.root_dir = Dir.pwd
-
-    server = fork {
-      app = Rack::Builder.app {
-        map "/jq" do
-          use Bonsai::StaticPassThrough, :root => Bonsai.root_dir + "/output", :urls => ["/"]
-        end
-        run Bonsai::DevelopmentServer
-      }
-      Rack::Handler.default.run(app, :Port => 5000) do
-        Launchy.open("http://localhost:5000/jq/")
-      end
-    }
-    Watch.new("{content,templates,public}/**/*") { Bonsai::Exporter.process! }
-  rescue Interrupt
-    Process.kill("QUIT", server)
-    Process.wait(server)
-    exit
-  end
-end
@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+from datetime import date
+from io import StringIO
+from lxml import etree
+import markdown
+from markdown.extensions import Extension
+import re
+import sys
+import yaml
+
+# Prevent our markdown parser from trying to help by interpreting things in angle brackets as HTML tags.
+class EscapeHtml(Extension):
+  def extendMarkdown(self, md, md_globals):
+    del md.preprocessors['html_block']
+    del md.inlinePatterns['html']
+
+class RoffWalker(object):
+  def __init__(self, tree, output=sys.stdout):
+    self.tree = tree
+    self.target = output
+    self.f = StringIO()
+
+  def walk(self):
+    self._walk(self.tree, parent_tag=None)
+    # We don't want to start lines with \. because that can confuse man
+    # For lines that start with \., we need to prefix them with \& so it
+    # knows not to treat that line as a directive
+    data = re.sub(r'^\\\.', r'\&.', self.f.getvalue(), flags=re.MULTILINE)
+    self.target.write(data)
+
+  def _ul_is_special(self, root):
+    if len(root) != 1:
+      return False
+    child = root[0]
+    if child.tag != 'li':
+      return False
+    msg = ''.join(child.itertext()).strip()
+    return msg.endswith(':')
+
+  def _walk_child(self, root):
+    if len(root) > 0:
+      self._walk(root[0], parent_tag=root.tag)
+
+  def _write_element(self, root, ensure_newline=True):
+    if root.text is not None:
+      text = self._sanitize(root.text)
+      self.__write_raw(text)
+    self._walk_child(root)
+    self._write_tail(root, ensure_newline=ensure_newline)
+
+  def _write_tail(self, root, ensure_newline=False, inline=False):
+    if root.tail is not None:
+      if inline or root.tail != '\n':
+        text = self._sanitize(root.tail)
+        if text.endswith('\n'):
+          ensure_newline = False
+        self.__write_raw(text)
+    if ensure_newline:
+      self.__write_raw('\n')
+
+  def _walk(self, root, parent_tag=None):
+    last_tag = None
+    while root is not None:
+      if root.tag == 'h1':
+        self.__write_cmd('.TH "JQ" "1" "{}" "" ""'.format(date.today().strftime('%B %Y')))
+        self.__write_cmd('.SH "NAME"')
+        # TODO: properly parse this
+        self.__write_raw(r'\fBjq\fR \- Command\-line JSON processor' + "\n")
+
+      elif root.tag == 'h2':
+        self.__write_cmd('.SH "{}"'.format(''.join(root.itertext()).strip()))
+
+      elif root.tag == 'h3':
+        text = ''.join(root.itertext()).strip()
+        self.__write_cmd('.SS "{}"'.format(self._h3_sanitize(text)))
+
+      elif root.tag == 'p':
+        if last_tag not in ['h2', 'h3'] and parent_tag not in ['li']:
+          self.__write_cmd('.P')
+        self._write_element(root, ensure_newline=(parent_tag != 'li'))
+
+      elif root.tag == 'ul':
+        if self._ul_is_special(root):
+          li = root[0]
+          self.__write_cmd('.TP')
+          self._write_element(li)
+          next = root.getnext()
+          while next is not None and next.tag == 'p':
+            if next.getnext() is not None and next.getnext().tag == 'pre':
+              # we don't want to .IP these, because it'll look funny with the code indent
+              break
+            self.__write_cmd('.IP')
+            self._write_element(next)
+            root = next
+            next = root.getnext()
+        else:
+          self._walk_child(root)
+          self._write_tail(root)
+          # A pre tag after the end of a list doesn't want two of the indentation commands
+          if root.getnext() is None or root.getnext().tag != 'pre':
+            self.__write_cmd('.IP "" 0')
+
+      elif root.tag == 'li':
+        self.__write_cmd(r'.IP "\(bu" 4')
+        if root.text is not None and root.text.strip() != '':
+          text = self._sanitize(root.text)
+          self.__write_raw(text)
+        self._walk_child(root)
+        self._write_tail(root, ensure_newline=True)
+
+      elif root.tag == 'strong':
+        if root.text is not None:
+          text = self._sanitize(root.text)
+          self.__write_raw('\\fB{}\\fR'.format(text))
+
+        self._write_tail(root, inline=True)
+
+      elif root.tag == 'em':
+        if root.text is not None:
+          text = self._sanitize(root.text)
+          self.__write_raw('\\fI{}\\fR'.format(text))
+        self._write_tail(root, inline=True)
+
+      elif root.tag == 'code':
+        if root.text is not None:
+          text = self._code_sanitize(root.text)
+          self.__write_raw('\\fB{}\\fR'.format(text))
+        self._write_tail(root, inline=True)
+
+      elif root.tag == 'pre':
+        self.__write_cmd('.IP "" 4')
+        self.__write_cmd('.nf\n') # extra newline for spacing reasons
+        next = root
+        first = True
+        while next is not None and next.tag == 'pre':
+          if not first:
+            self.__write_raw('\n')
+          text = ''.join(next.itertext(with_tail=False))
+          self.__write_raw(self._pre_sanitize(text))
+          first = False
+          root = next
+          next = next.getnext()
+        self.__write_cmd('.fi')
+        self.__write_cmd('.IP "" 0')
+
+      else:
+        self._walk_child(root)
+
+      last_tag = root.tag
+      root = root.getnext()
+
+  def _base_sanitize(self, text):
+    text = re.sub(r'\\', r'\\e', text)
+    text = re.sub(r'\.', r'\\.', text)
+    text = re.sub("'", r"\'", text)
+    text = re.sub('-', r'\-', text)
+    return text
+
+  def _pre_sanitize(self, text):
+    return self._base_sanitize(text)
+
+  def _code_sanitize(self, text):
+    text = self._base_sanitize(text)
+    text = re.sub(r'\s', ' ', text)
+    return text
+
+  def _h3_sanitize(self, text):
+    text = self._base_sanitize(text)
+    text = re.sub(' \n|\n ', ' ', text)
+    text = re.sub('\n', ' ', text)
+    return text
+
+  def _sanitize(self, text):
+    text = self._base_sanitize(text)
+    text = re.sub(r'<([^>]+)>', r'\\fI\1\\fR', text)
+    text = re.sub(r' +', ' ', text)
+    text = re.sub('\n', ' ', text)
+    return text
+
+  def __write_cmd(self, dat):
+    print('.', dat, sep='\n', file=self.f)
+    pass
+
+  def __write_raw(self, dat):
+    print(dat, sep='', end='', file=self.f)
+    pass
+
+def load_yml_file(fn):
+  with open(fn) as f:
+    return yaml.load(f)
+
+def dedent_body(body):
+  lines = [re.sub(r'^  (\S)', r'\1', l) for l in body.split('\n')]
+  return '\n'.join(lines)
+
+def convert_manual_to_markdown():
+  f = StringIO()
+  manual = load_yml_file("content/manual/manual.yml")
+  f.write(manual.get('manpage_intro', '\n'))
+  f.write(dedent_body(manual.get('body', '\n')))
+  for section in manual.get('sections', []):
+    f.write('## {}\n'.format(section.get('title', '').upper()))
+    f.write(dedent_body(section.get('body', '\n')))
+    f.write('\n')
+    for entry in section.get('entries', []):
+      f.write('### {}\n'.format(entry.get('title', '')))
+      f.write(dedent_body(entry.get('body', '\n')))
+      f.write('\n')
+      if entry.get('examples') is not None:
+        f.write("~~~~\n")
+        first = True
+        for example in entry.get('examples'):
+          if not first:
+            f.write('\n')
+          f.write("jq '{}'\n".format(example.get('program', '')))
+          f.write("   {}\n".format(example.get('input', '')))
+          output = [str(x) for x in example.get('output', [])]
+          f.write("=> {}\n".format(', '.join(output)))
+          first = False
+        f.write("~~~~\n")
+    f.write('\n')
+  f.write(manual.get('manpage_epilogue', ''))
+  return f.getvalue()
+
+# Convert manual.yml to our special markdown format
+markdown_data = convert_manual_to_markdown()
+
+# Convert markdown to html
+html_data = markdown.markdown(markdown_data, extensions=[EscapeHtml(), 'fenced_code'])
+
+# Parse the html into a tree so we can walk it
+tr = etree.HTML(html_data, etree.HTMLParser())
+
+# Convert the markdown to ROFF
+RoffWalker(tr).walk()
@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+import yaml
+
+with open("content/manual/manual.yml") as f:
+  manual = yaml.load(f)
+  for section in manual.get('sections', []):
+    for entry in section.get('entries', []):
+      for example in entry.get('examples', []):
+        print(example.get('program', '').replace('\n', ' '))
+        print(example.get('input', ''))
+        for s in example.get('output', []):
+          print(s)
+        print('')
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import glob
+import itertools
+from jinja2 import Environment, FileSystemLoader, Markup, select_autoescape, contextfunction
+from markdown import markdown
+import os
+import os.path
+import re
+import shutil
+import yaml
+
+env = Environment(
+  loader=FileSystemLoader('templates'),
+  autoescape=select_autoescape(['html.j2']),
+)
+
+def load_yml_file(fn):
+  with open(fn) as f:
+    return yaml.load(f)
+
+env.filters['search_id'] = lambda input: input.replace(r'`', '')
+env.filters['section_id'] = lambda input: re.sub(r"[^a-zA-Z0-9_]", '', input)
+env.filters['entry_id'] = lambda input: re.sub(r"[ `]", '', input)
+env.filters['markdownify'] = lambda input: Markup(markdown(input))
+env.filters['no_paragraph'] = lambda input: Markup(re.sub(r"</?p>", '', input))
+
+env.globals['unique_id'] = contextfunction(lambda ctx: str(next(ctx['unique_ctr'])))
+
+
+env.globals.update(load_yml_file('site.yml'))
+
+
+env.globals['navigation'] = ['tutorial', 'download', 'manual']
+
+def generate_file(env, fname='content/1.tutorial/default.yml'):
+  path, base = os.path.split(fname)
+  path = os.path.relpath(path, 'content')
+  output_dir = os.path.join('output', path)
+  output_path = os.path.join(output_dir, 'index.html')
+
+  template_name = re.sub(r".yml$", '.html.j2', base)
+
+  ctx = load_yml_file(fname)
+  ctx.update(unique_ctr=itertools.count(1), permalink=path)
+  os.makedirs(output_dir, exist_ok=True)
+  env.get_template(template_name).stream(ctx).dump(output_path, encoding='utf-8')
+
+
+def copy_public_files(root=''):
+  for f in os.scandir(os.path.join('public', root)):
+    src = os.path.join(root, f.name)
+    dst = os.path.join('output', src)
+    if f.is_dir():
+      os.makedirs(dst, exist_ok=True)
+      copy_public_files(src)
+    else:
+      shutil.copyfile(f.path, dst)
+
+copy_public_files()
+
+for fn in glob.glob('content/**/*.yml', recursive=True):
+  generate_file(env, fn)
@@ -180,14 +180,13 @@ body:

      #### Building the documentation

-      jq's documentation is compiled into static HTML using
-      [Bonsai](http://www.tinytree.info).  To view the documentation
-      locally, run `rake serve` (or `bundle exec rake serve`) from the
-      docs/ subdirectory.  To build the docs just `rake build` from the
-      docs subdirectory.  You'll need a few Ruby dependencies, which can
-      be installed by following the instructions in `docs/README.md`.
+      jq's documentation is compiled into static HTML using Python.
+      To build the docs, run `pipenv run python3 build_website.py` from
+      the docs/ subdirectory. To serve them locally, you can run
+      `python3 -m SimpleHTTPServer`. You'll need a few Python dependencies,
+      which can be installed by following the instructions in `docs/README.md`.

      The man page is built by `make jq.1`, or just `make`, also from
-      the YAML docs, and you'll still need the Ruby dependencies to
+      the YAML docs, and you'll still need the Python dependencies to
      build the manpage.

@@ -172,7 +172,7 @@ sections:
        ASCII output with every non-ASCII character replaced with the
        equivalent escape sequence.

-      * `--unbuffered`
+      * `--unbuffered`:

        Flush the output after each JSON object is printed (useful if
        you're piping a slow data source into jq and piping jq's
@@ -549,13 +549,16 @@ sections:
          dictionaries or hashes), as in: `{"a": 42, "b": 17}`.

          If the keys are "identifier-like", then the quotes can be left
-          off, as in `{a:42, b:17}`.  Keys generated by expressions need
-          to be parenthesized, e.g., `{("a"+"b"):59}`.
+          off, as in `{a:42, b:17}`.  Variable references as key
+          expressions use the value of the variable as the key.  Key
+          expressions other than constant literals, identifiers, or
+          variable references, need to be parenthesized, e.g.,
+          `{("a"+"b"):59}`.

-          The value can be any expression (although you may need to
-          wrap it in parentheses if it's a complicated one), which gets
-          applied to the {} expression's input (remember, all filters
-          have an input and an output).
+          The value can be any expression (although you may need to wrap
+          it in parentheses if, for example, it contains colons), which
+          gets applied to the {} expression's input (remember, all
+          filters have an input and an output).

              {foo: .bar}

@@ -593,6 +596,16 @@ sections:

              {"stedolan": ["JQ Primer", "More JQ"]}

+          Variable references as keys use the value of the variable as
+          the key.  Without a value then the variable's name becomes the
+          key and its value becomes the value,
+
+              "f o o" as $foo | "b a r" as $bar | {$foo, $bar:$foo}
+
+          produces
+
+              {"f o o":"f o o","b a r":"f o o"}
+
        examples:
          - program: '{user, title: .titles[]}'
            input: '{"user":"stedolan","titles":["JQ Primer", "More JQ"]}'
@@ -645,10 +658,10 @@ sections:
          - **Strings** are added by being joined into a larger string.

          - **Objects** are added by merging, that is, inserting all
-              the key-value pairs from both objects into a single
-              combined object. If both objects contain a value for the
-              same key, the object on the right of the `+` wins. (For
-              recursive merge use the `*` operator.)
+            the key-value pairs from both objects into a single
+            combined object. If both objects contain a value for the
+            same key, the object on the right of the `+` wins. (For
+            recursive merge use the `*` operator.)

          `null` can be added to any value, and returns the other
          value unchanged.
@@ -1962,40 +1975,40 @@ sections:

          jq provides a few SQL-style operators.

-           * INDEX(stream; index_expression):
+          * INDEX(stream; index_expression):

-             This builtin produces an object whose keys are computed by
-             the given index expression applied to each value from the
-             given stream.
+            This builtin produces an object whose keys are computed by
+            the given index expression applied to each value from the
+            given stream.

-           * JOIN($idx; stream; idx_expr; join_expr):
+          * JOIN($idx; stream; idx_expr; join_expr):

-             This builtin joins the values from the given stream to the
-             given index.  The index's keys are computed by applying the
-             given index expression to each value from the given stream.
-             An array of the value in the stream and the corresponding
-             value from the index is fed to the given join expression to
-             produce each result.
+            This builtin joins the values from the given stream to the
+            given index.  The index's keys are computed by applying the
+            given index expression to each value from the given stream.
+            An array of the value in the stream and the corresponding
+            value from the index is fed to the given join expression to
+            produce each result.

-           * JOIN($idx; stream; idx_expr):
+          * JOIN($idx; stream; idx_expr):

-             Same as `JOIN($idx; stream; idx_expr; .)`.
+            Same as `JOIN($idx; stream; idx_expr; .)`.

-           * JOIN($idx; idx_expr):
+          * JOIN($idx; idx_expr):

-             This builtin joins the input `.` to the given index, applying
-             the given index expression to `.` to compute the index key.
-             The join operation is as described above.
+            This builtin joins the input `.` to the given index, applying
+            the given index expression to `.` to compute the index key.
+            The join operation is as described above.

-           * IN(s):
+          * IN(s):

-             This builtin outputs `true` if `.` appears in the given
-             stream, otherwise it outputs `false`.
+            This builtin outputs `true` if `.` appears in the given
+            stream, otherwise it outputs `false`.

-           * IN(source; s):
+          * IN(source; s):

-             This builtin outputs `true` if any value in the source stream
-             appears in the second stream, otherwise it outputs `false`.
+            This builtin outputs `true` if any value in the source stream
+            appears in the second stream, otherwise it outputs `false`.

      - title: "`builtins`"
        body: |
@@ -2031,13 +2044,16 @@ sections:
          produces a value other than false or null, but act the same
          as `C` otherwise.

+          `if A then B end` is the same as `if A then B else .  end`.
+          That is, the `else` branch is optional, and if absent is the
+          same as `.`.
+
          Checking for false or null is a simpler notion of
          "truthiness" than is found in Javascript or Python, but it
          means that you'll sometimes have to be more explicit about
-          the condition you want: you can't test whether, e.g. a
+          the condition you want.  You can't test whether, e.g. a
          string is empty using `if .name then A else B end`, you'll
-          need something more like `if (.name | length) > 0 then A else
-          B end` instead.
+          need something more like `if .name then A else B end` instead.

          If the condition `A` produces multiple results, then `B` is evaluated
          once for each result that is not false or null, and `C` is evaluated
@@ -2046,14 +2062,13 @@ sections:
          More cases can be added to an if using `elif A then B` syntax.

        examples:
-          - program: |-
-              if . == 0 then
+          - program: 'if . == 0 then
                "zero"
              elif . == 1 then
                "one"
              else
                "many"
-              end
+              end'
            input: 2
            output: ['"many"']

@@ -2215,6 +2230,7 @@ sections:
          STRING | FILTER( [REGEX, FLAGS] )

      where:
+
      * STRING, REGEX and FLAGS are jq strings and subject to jq string interpolation;
      * REGEX, after string interpolation, should be a valid PCRE regex;
      * FILTER is one of `test`, `match`, or `capture`, as described below.
@@ -2627,9 +2643,9 @@ sections:

          For example, in the following expression there is a binding
          which is visible "to the right" of it, `... | .*3 as
-          $times_three | [.  + $times_three] | ...`, but not "to the
+          $times_three | [. + $times_three] | ...`, but not "to the
          left".  Consider this expression now, `... | (.*3 as
-          $times_three | [.+ $times_three]) | ...`: here the binding
+          $times_three | [. + $times_three]) | ...`: here the binding
          `$times_three` is _not_ visible past the closing parenthesis.

      - title: Reduce
@@ -2852,7 +2868,7 @@ sections:

      Two builtins provide minimal output capabilities, `debug`, and
      `stderr`.  (Recall that a jq program's output values are always
-      output as JSON texts on `stdout`.)  The `debug` builtin can have
+      output as JSON texts on `stdout`.) The `debug` builtin can have
      application-specific behavior, such as for executables that use
      the libjq C API but aren't the jq executable itself.  The `stderr`
      builtin outputs its input in raw mode to stder with no additional
@@ -441,7 +441,7 @@ sections:
            input: 5
            output: [6]

-      - title: `length`
+      - title: '`length`'
        body: |

          The builtin function `length` gets the length of various
@@ -462,7 +462,7 @@ sections:
            input: '[[1,2], "string", {"a":2}, null]'
            output: [2, 6, 1, 0]

-      - title: `keys`
+      - title: '`keys`'
        body: |

          The builtin function `keys`, when given an object, returns
@@ -485,7 +485,7 @@ sections:
            input: '[42,3,35]'
            output: ['[0,1,2]']

-      - title: `has`
+      - title: '`has`'
        body: |

          The builtin function `has` returns whether the input object
@@ -504,7 +504,7 @@ sections:
            input: '[[0,1], ["a","b","c"]]'
            output: ['[false, true]']

-      - title: `to_entries`, `from_entries`, `with_entries`
+      - title: '`to_entries`, `from_entries`, `with_entries`'
        body: |

          These functions convert between an object and an array of
@@ -529,14 +529,14 @@ sections:
            output: ['{"KEY_a": 1, "KEY_b": 2}']


-      - title: `select`
+      - title: '`select`'
        body: |

          The function `select(foo)` produces its input unchanged if
          `foo` returns true for that input, and produces no output
          otherwise.

-          It's useful for filtering lists: `[1,2,3] | map(select(. >= 2))`
+          It's useful for filtering lists: '`[1,2,3] | map(select(. >= 2))`'
          will give you `[3]`.

        examples:
@@ -544,7 +544,7 @@ sections:
            input: '[1,5,3,0,7]'
            output: ['[5,3,7]']

-      - title: `empty`
+      - title: '`empty`'
        body: |

          `empty` returns no results. None at all. Not even `null`.
@@ -559,7 +559,7 @@ sections:
            input: 'null'
            output: ['[1,2,3]']

-      - title: `map(x)`
+      - title: '`map(x)`'
        body: |

          For any filter `x`, `map(x)` will run that filter for each
@@ -574,7 +574,7 @@ sections:
            input: '[1,2,3]'
            output: ['[2,3,4]']

-      - title: `add`
+      - title: '`add`'
        body: |

          The filter `add` takes as input an array, and produces as
@@ -596,7 +596,7 @@ sections:
            input: '[]'
            output: ["null"]

-      - title: `range`
+      - title: '`range`'
        body: |

          The `range` function produces a range of numbers. `range(4;10)`
@@ -612,7 +612,7 @@ sections:
            input: 'null'
            output: ['[2,3]']

-      - title: `tonumber`
+      - title: '`tonumber`'
        body: |

          The `tonumber` function parses its input as a number. It
@@ -624,7 +624,7 @@ sections:
            input: '[1, "1"]'
            output: [1, 1]

-      - title: `tostring`
+      - title: '`tostring`'
        body: |

          The `tostring` function prints its input as a
@@ -636,7 +636,7 @@ sections:
            input: '[1, "1", [1]]'
            output: ['"1"', '"1"', '"[1]"']

-      - title: `type`
+      - title: '`type`'
        body: |

          The `type` function returns the type of its argument as a
@@ -648,7 +648,7 @@ sections:
            input: '[0, false, [], {}, null, "hello"]'
            output: ['["number", "boolean", "array", "object", "null", "string"]']

-      - title: `sort, sort_by`
+      - title: '`sort, sort_by`'
        body: |

          The `sort` functions sorts its input, which must be an
@@ -680,7 +680,7 @@ sections:
            input: '[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}]'
            output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}]']

-      - title: `group_by`
+      - title: '`group_by`'
        body: |

          `group_by(.foo)` takes as input an array, groups the
@@ -697,7 +697,7 @@ sections:
            input: '[{"foo":1, "bar":10}, {"foo":3, "bar":100}, {"foo":1, "bar":1}]'
            output: ['[[{"foo":1, "bar":10}, {"foo":1, "bar":1}], [{"foo":3, "bar":100}]]']

-      - title: `min`, `max`, `min_by`, `max_by`
+      - title: '`min`, `max`, `min_by`, `max_by`'
        body: |

          Find the minimum or maximum element of the input array. The
@@ -713,7 +713,7 @@ sections:
            input: '[{"foo":1, "bar":14}, {"foo":2, "bar":3}]'
            output: ['{"foo":2, "bar":3}']

-      - title: `unique`
+      - title: '`unique`'
        body: |

          The `unique` function takes as input an array and produces
@@ -725,7 +725,7 @@ sections:
            input: '[1,2,5,3,5,3,1,3]'
            output: ['[1,2,3,5]']

-      - title: `reverse`
+      - title: '`reverse`'
        body: |

          This function reverses an array.
@@ -735,7 +735,7 @@ sections:
            input: '[1,2,3,4]'
            output: ['[4,3,2,1]']

-      - title: `contains`
+      - title: '`contains`'
        body: |

          The filter `contains(b)` will produce true if b is
@@ -764,7 +764,7 @@ sections:
            input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
            output: ['false']

-      - title: `recurse`
+      - title: '`recurse`'
        body: |

          The `recurse` function allows you to search through a
@@ -796,7 +796,7 @@ sections:
              - '{"foo":[]}'


-      - title: "String interpolation - `\(foo)`"
+      - title: String interpolation - `\(foo)`
        body: |

          Inside a string, you can put an expression inside parens
@@ -882,7 +882,7 @@ sections:

  - title: Conditionals and Comparisons
    entries:
-      - title: `==`, `!=`
+      - title: '`==`, `!=`'
        body: |

          The expression 'a == b' will produce 'true' if the result of a and b
@@ -931,7 +931,7 @@ sections:
            input: 2
            output: ['"many"']

-      - title: `>, >=, <=, <`
+      - title: '`>, >=, <=, <`'
        body: |

          The comparison operators `>`, `>=`, `<=`, `<` return whether
@@ -1,10 +1,10 @@
 ---
-headline: jq Manual (development version)
+headline: jq 1.6 Manual

 history: |

-  *For released versions, see [jq 1.5](/jq/manual/v1.5),
-  [jq 1.4](/jq/manual/v1.4) or [jq 1.3](/jq/manual/v1.3).*
+  *The manual for the development version of jq can be found
+  [here](/jq/manual).*

 body: |

@@ -623,7 +623,7 @@ sections:
  - title: Builtin operators and functions
    body: |

-      Some jq operator (for instance, `+`) do different things
+      Some jq operators (for instance, `+`) do different things
      depending on the type of their arguments (arrays, numbers,
      etc.). However, jq never does implicit type conversions. If you
      try to add a string to an object you'll get an error message and
@@ -0,0 +1,173 @@
+body {
+    padding-top: 80px;
+}
+
+.container {
+    max-width: 970px;
+}
+
+/* index.liquid *******************************************/
+#blurb {
+    padding-top: 40px;
+}
+#blurb p {
+    font-size: 1.9em;
+}
+#blurb .btn-group {
+    margin: 4px;
+}
+
+#multiblurb {
+    line-height: 1.7;
+    text-align: center;
+    font-size: 12pt;
+}
+#multiblurb code {
+    border: 0;
+    font-size: 12pt;
+}
+
+#news {
+    font-size: 12pt;
+}
+#news .date {
+    font-style: italic;
+}
+
+/* default.liquid *****************************************/
+.tutorial-example {
+    position: relative;
+    margin-bottom: 10px;
+}
+.tutorial-example pre {
+    margin-bottom: 0px;
+}
+.tutorial-example a {
+    position: absolute;
+    top: 0px;
+    right: 0px;
+    padding: 15px 8px;
+    color: #777777;
+    font-weight: bold;
+    line-height: 10px;
+    font-size: 12px;
+    border-left: 1px solid #DDDDDD;
+    display: block;
+}
+.tutorial-example .accordion-body pre {
+    margin: 0 4px;
+    border-top: 0;
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+}
+
+@media print {
+    .tutorial-example a {
+        display: none;
+    }
+}
+/* manual.liquid ******************************************/
+section {
+    padding-top: 24px;
+}
+
+h3 code {
+    border: 0;
+    font-size: 20px;
+}
+
+@media (max-width: 991px) {
+    #navcolumn {
+        /* Put nav column above manual content */
+        position: relative !important;
+        margin-bottom: 60px;
+    }
+}
+@media (min-width: 992px) {
+    #manualcontent {
+        /* Put nav column left of manual content */
+        padding-left: 280px;
+    }
+}
+.nav-pills {
+    margin-bottom: 20px;
+}
+    .nav-pills li a {
+        padding: 8px 12px;
+}
+
+.manual-example table {
+    border-top: 1px solid #E5E5E5;
+}
+.manual-example table td {
+    white-space: pre-wrap;
+    font-family: Monaco, Menlo, Consolas, "Courier New", monospace;
+}
+.manual-example table td.jqprogram {
+    font-weight: bold;
+}
+.manual-example table th {
+    text-align: right;
+    padding-right: 10px;
+}
+
+@media print {
+    #navcolumn {
+        display: none !important;
+    }
+
+    .manual-example {
+        display: block !important;
+        height: auto !important;
+    }
+
+    .jqplay-btn {
+        display: none !important;
+    }
+}
+/* shared/_footer.liquid **********************************/
+footer {
+    background-color: #F5F5F5;
+    padding: 20px 0;
+    margin-top: 40px;
+    color: #999999;
+    text-align: center;
+}
+footer p {
+    margin: 8px 0;
+}
+
+/* typeahead **********************************************/
+.twitter-typeahead {
+    width: 100%;
+}
+
+.tt-menu {
+    width: 100%;
+    background-color: #fff;
+    padding: 8px 0;
+    border: 1px solid #ccc;
+    border: 1px solid rgba(0, 0, 0, 0.2);
+    -webkit-border-radius: 8px;
+    -moz-border-radius: 8px;
+    border-radius: 8px;
+    -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+    -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+    box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+}
+
+.tt-suggestion {
+    padding: 3px 20px;
+}
+.tt-suggestion:hover {
+    cursor: pointer;
+    color: #fff;
+    background-color: #446e9b;
+}
+.tt-suggestion.tt-cursor {
+    color: #fff;
+    background-color: #446e9b;
+}
+.tt-suggestion p {
+    margin: 0;
+}
@@ -44,7 +44,7 @@ $(function() {
  $.each($('.manual-example table'), function(index, value) {
    $value = $(value)
    var j = $value.find('tr:nth-child(2) td:first').text();
-    var q = $value.find('.jqprogram').text().replace(/^jq /, '').replace(/^'(.+)'$/, '$1');
+    var q = $value.find('.jqprogram').text().replace(/^jq /, '').replace(/(\r\n|\n|\r)/gm," ").replace(/^'(.+)'$/, '$1');
    var url = 'https://jqplay.org/jq?q=' + encodeURIComponent(q) +'&j=' + encodeURIComponent(j)
    var $last_tr = $value.find('tr:last');
    $last_tr.after('<tr class="jqplay-btn"><th><a href="' + url + '" class="btn btn-primary btn-sm">Run</a></th><th></th></tr><tr><th></th><th></th></tr>');
@@ -1,26 +1,26 @@
 <!DOCTYPE html>
 <html lang="en">
-{% include "shared/head" %}
+{% include "shared/_head.html.j2" %}

  <body id="{{slug}}">
-{% include "shared/navbar" %}
+{% include "shared/_navbar.html.j2" %}

    <div class="container">
      <div class="row">
        <h1>{{headline}}</h1>
        {% for item in body %}
          {% if item.text %}
-            {{ item.text | replace: '$JQ_VERSION', jq_version | markdownify }}
+            {{ item.text | replace('$JQ_VERSION', jq_version) | markdownify }}
          {% endif %}

          {% if item.command %}
-            {% capture resultID %}{{ "result" | unique}}{% endcapture %}
+            {% set resultID = unique_id() %}
            <div class="tutorial-example">
              <div class="accordion-heading">
                <pre>{{item.command}}</pre>
-                <a data-toggle="collapse" href="#{{resultID}}">Show result</a>
+                <a data-toggle="collapse" href="#result{{resultID}}">Show result</a>
              </div>
-              <div id="{{resultID}}" class="accordion-body collapse">
+              <div id="result{{resultID}}" class="accordion-body collapse">
                <pre>{{item.result}}</pre>
              </div>
            </div>
@@ -29,6 +29,6 @@
      </div>
    </div>

-{% include "shared/footer" %}
+{% include "shared/_footer.html.j2" %}
  </body>
 </html>
@@ -1,9 +1,9 @@
 <!DOCTYPE html>
 <html lang="en">
-{% include "shared/head" %}
+{% include "shared/_head.html.j2" %}

-  <body id="{{slug}}">
-{% include "shared/navbar" %}
+  <body id="{{ slug }}">
+{% include "shared/_navbar.html.j2" %}

    <div class="container">
      <div class="row">
@@ -20,14 +20,14 @@
                <div class="btn-toolbar" role="toolbar">
                  <div class="btn-group" role="group">
                    <a class="btn btn-primary dropdown-toggle" data-toggle="dropdown" href="#">
-                      Download jq 1.5
+                      Download jq 1.6
                      <span class="caret"></span>
                    </a>
                    <ul class="dropdown-menu">
-                      <li><a href="https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64">Linux (64-bit)</a></li>
-                      <li><a href="https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64">OS X (64-bit)</a></li>
-                      <li><a href="https://github.com/stedolan/jq/releases/download/jq-1.5/jq-win64.exe">Windows (64-bit)</a></li>
-                      <li><a href="/jq/download/">Other platforms, older versions, and source</a></li>
+                      <li><a href="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64">Linux (64-bit)</a></li>
+                      <li><a href="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64">OS X (64-bit)</a></li>
+                      <li><a href="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-win64.exe">Windows (64-bit)</a></li>
+                      <li><a href="{{root}}/download/">Other platforms, older versions, and source</a></li>
                    </ul>
                  </div>

@@ -68,6 +68,6 @@
      </div>
    </div>

-{% include "shared/footer" %}
+{% include "shared/_footer.html.j2" %}
  </body>
 </html>
@@ -1,9 +1,9 @@
 <!DOCTYPE html>
 <html lang="en">
-{% include "shared/head" %}
+{% include "shared/_head.html.j2" %}

  <body id="{{slug}}" data-spy="scroll" data-target="#navcolumn" data-offset="100">
-{% include "shared/navbar" %}
+{% include "shared/_navbar.html.j2" %}

    <div class="container">
      <div class="row">
@@ -32,41 +32,41 @@
          {% for section in sections %}
            <section id="{{section.title | section_id}}">
              <h2>{{section.title}}</h2>
-              {{section.body | markdownify}}
+              {{ (section.body | markdownify) if section.body }}
              {% for entry in section.entries %}
                <section id="{{entry.title | entry_id}}">
                  <h3>
-                    {{entry.title | markdownify | no_paragraph}}
+                    {{entry.title | markdownify | no_paragraph }}
                    {% if entry.subtitle %}<small>{{entry.subtitle}}</small>{% endif %}
                  </h3>
                  {{entry.body | markdownify}}

                  {% if entry.examples %}
                    <div>
-                      {% capture exampleID %}{{ "" | unique }}{% endcapture %}
+                      {% set exampleID = unique_id() %}
                      <a data-toggle="collapse" href="#example{{exampleID}}">
                        <i class="glyphicon glyphicon-chevron-right"></i>
-                        {% if entry.examples[1] %}Examples{%else%}Example{%endif%}
+                        {% if entry.examples | length > 1 %}Examples{%else%}Example{%endif%}
                      </a>
                      <div id="example{{exampleID}}" class="manual-example collapse">
                        {% for example in entry.examples %}
                          <table>
-                            <tr><th></th><td class="jqprogram">jq '{{example.program | escape}}'</td></tr>
-                            <tr><th>Input</th><td>{{example.input | escape}}</td></tr>
-                            {% unless example.output[0] %}
+                            <tr><th></th><td class="jqprogram">jq '{{ example.program }}'</td></tr>
+                            <tr><th>Input</th><td>{{ example.input }}</td></tr>
+                            {% if not example.output[0] %}
                              <tr>
                                <th>Output</th>
                                <td><i>none</i></td>
                              </tr>
-                            {% endunless %}
+                            {% endif %}
                            {% for output in example.output %}
                              <tr>
-                                {% if forloop.index == 1 %}
+                                {% if loop.first %}
                                  <th>Output</th>
                                {% else %}
                                  <th></th>
                                {% endif %}
-                                <td>{{output | escape}}</td>
+                                <td>{{ output }}</td>
                              </tr>
                            {% endfor %}
                          </table>
@@ -82,15 +82,15 @@
      </div>
    </div>

-{% include "shared/footer" %}
+{% include "shared/_footer.html.j2" %}
    <script>
      var section_map = {
        {% for section in sections %}
          {% for entry in section.entries %}
-            {{entry.title | search_id | json}} : {{entry.title | entry_id | json}},
+            {{entry.title | search_id | tojson }} : {{entry.title | entry_id | tojson }},
          {% endfor %}
-          {{section.title | search_id | json}} : {{section.title | section_id | json}}
-          {% unless forloop.last %},{% endunless %}
+          {{section.title | search_id | tojson }} : {{section.title | section_id | tojson }}
+          {% if not loop.last %},{% endif %}
        {% endfor %}
      };
    </script>
@@ -1,6 +1,6 @@
    <footer>
      <div class="container">
-        <p>This website is made with <a href="http://www.tinytree.info">Bonsai</a> and <a href="http://getbootstrap.com">Bootstrap</a>, themed with <a href="https://bootswatch.com">Bootswatch</a>.</p>
+        <p>This website is made with <a href="http://getbootstrap.com">Bootstrap</a>, themed with <a href="https://bootswatch.com">Bootswatch</a>.</p>
        <p>jq is licensed under the MIT license (code) and the <a href="https://creativecommons.org/licenses/by/3.0/">CC-BY-3.0</a> license (docs).</p>
      </div>
    </footer>
@@ -7,9 +7,9 @@
    <title>{{headline}}</title>
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
-    <link rel="canonical" href="{{url}}{{permalink | remove:'/index'}}" />
+    <link rel="canonical" href="{{ url }}{{ permalink | replace('/index','') }}" />
    <link href="https://maxcdn.bootstrapcdn.com/bootswatch/3.3.5/spacelab/bootstrap.min.css" rel="stylesheet" integrity="sha256-j7Dtnd7ZjexEiPNbscbopFn9+Cs0b3TLipKsWAPHZIM= sha512-RFhfi6P8zWMAJrEGU+CPjuxPh3r/UUBGqQ+/o6WKPIVZmQqeOipGotH2ihRULuQ8wsMBoK15TSZqc/7VYWyuIw==" crossorigin="anonymous">
-    <link rel="stylesheet" href="{{root}}/css/base.css" type="text/css">
+    <link rel="stylesheet" href="{{ root }}/css/base.css" type="text/css">
    <!--[if lt IE 9]>
      <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
@@ -13,8 +13,8 @@
        <div class="navbar-collapse collapse" id="nav-collapse">
          <ul class="nav navbar-nav">
            {% for item in navigation %}
-              <li {% if item.permalink == permalink %} class="active" {% endif %}>
-                <a href="{{root}}{{item.permalink}}">{{item.name}}</a>
+              <li {% if item == permalink %} class="active" {% endif %}>
+                <a href="{{root}}/{{item}}">{{item | capitalize}}</a>
              </li>
            {% endfor %}
            <li><a href="https://github.com/stedolan/jq/issues">Issues</a></li>
@@ -11,7 +11,7 @@ set -o xtrace
 scriptdir=`dirname "$0"`
 cd "$scriptdir"/../docs
 rm -rf output
-rake build
+pipenv run python3 build_website.py
 cd ..

 # copy to /tmp
@@ -34,6 +34,9 @@ void *alloca (size_t);
 #endif
 #include <string.h>
 #include <time.h>
+#ifdef WIN32
+#include <windows.h>
+#endif
 #include "builtin.h"
 #include "compile.h"
 #include "jq_parser.h"
@@ -862,7 +865,7 @@ static jv f_match(jq_state *jq, jv input, jv regex, jv modifiers, jv testmode) {
      if (region->end[0] == region->beg[0]) {
        unsigned long idx;
        const char *fr = (const char*)input_string;
-        for (idx = 0; fr != input_string+region->beg[0]; idx++) {
+        for (idx = 0; fr < input_string+region->beg[0]; idx++) {
          fr += jvp_utf8_decode_length(*fr);
        }
        jv match = jv_object_set(jv_object(), jv_string("offset"), jv_number(idx));
@@ -899,7 +902,7 @@ static jv f_match(jq_state *jq, jv input, jv regex, jv modifiers, jv testmode) {
            cap = jv_object_set(cap, jv_string("string"), jv_null());
          } else {
            fr = input_string;
-            for (idx = 0; fr != input_string+region->beg[i]; idx++) {
+            for (idx = 0; fr < input_string+region->beg[i]; idx++) {
              fr += jvp_utf8_decode_length(*fr);
            }
            cap = jv_object_set(jv_object(), jv_string("offset"), jv_number(idx));
@@ -911,7 +914,7 @@ static jv f_match(jq_state *jq, jv input, jv regex, jv modifiers, jv testmode) {
          continue;
        }
        fr = input_string;
-        for (idx = len = 0; fr != input_string+region->end[i]; len++) {
+        for (idx = len = 0; fr < input_string+region->end[i]; len++) {
          if (fr == input_string+region->beg[i]) idx = len, len=0;
          fr += jvp_utf8_decode_length(*fr);
        }
@@ -932,7 +935,7 @@ static jv f_match(jq_state *jq, jv input, jv regex, jv modifiers, jv testmode) {
      break;
    } else { /* Error */
      UChar ebuf[ONIG_MAX_ERROR_MESSAGE_LEN];
-      onig_error_code_to_str(ebuf, onigret, einfo);
+      onig_error_code_to_str(ebuf, onigret, &einfo);
      jv_free(result);
      result = jv_invalid_with_msg(jv_string_concat(jv_string("Regex failure: "),
            jv_string((char*)ebuf)));
@@ -1053,9 +1056,8 @@ static jv f_nan(jq_state *jq, jv input) {
  return jv_number(NAN);
 }

-static jv f_error(jq_state *jq, jv input, jv msg) {
-  jv_free(input);
-  return jv_invalid_with_msg(msg);
+static jv f_error(jq_state *jq, jv input) {
+  return jv_invalid_with_msg(input);
 }

 // FIXME Should autoconf check for this!
@@ -1186,6 +1188,27 @@ static jv tm2jv(struct tm *tm) {
                  jv_number(tm->tm_yday));
 }

+#if defined(WIN32) && !defined(HAVE_SETENV)
+static int setenv(const char *var, const char *val, int ovr)
+{
+  BOOL b;
+  char c[2];
+  if (!ovr)
+  {
+    DWORD d;
+    d = GetEnvironmentVariableA (var, c, 2);
+    if (0 != d && GetLastError () != ERROR_ENVVAR_NOT_FOUND) {
+      return d;
+    }
+  }
+  b = SetEnvironmentVariableA (var, val);
+  if (b) {
+    return 0;
+  }
+  return 1;
+}
+#endif
+
 /*
 * mktime() has side-effects and anyways, returns time in the local
 * timezone, not UTC.  We want timegm(), which isn't standard.
@@ -1201,10 +1224,23 @@ static jv tm2jv(struct tm *tm) {
 *
 * Returns (time_t)-2 if mktime()'s side-effects cannot be corrected.
 */
-static time_t my_timegm(struct tm *tm) {
+static time_t my_mktime(struct tm *tm) {
 #ifdef HAVE_TIMEGM
  return timegm(tm);
-#else /* HAVE_TIMEGM */
+#elif HAVE_TM_TM_GMT_OFF
+
+  time_t t = mktime(tm);
+  if (t == (time_t)-1)
+    return t;
+  return t + tm->tm_gmtoff;
+#elif HAVE_TM___TM_GMT_OFF
+  time_t t = mktime(tm);
+  if (t == (time_t)-1)
+    return t;
+  return t + tm->__tm_gmtoff;
+#elif WIN32
+  return _mkgmtime(tm);
+#else
  char *tz;

  tz = (tz = getenv("TZ")) != NULL ? strdup(tz) : NULL;
@@ -1214,18 +1250,6 @@ static time_t my_timegm(struct tm *tm) {
  if (tz != NULL)
    setenv("TZ", tz, 1);
  return t;
-#endif /* !HAVE_TIMEGM */
-}
-static time_t my_mktime(struct tm *tm) {
-  time_t t = mktime(tm);
-  if (t == (time_t)-1)
-    return t;
-#ifdef HAVE_TM_TM_GMT_OFF
-  return t + tm->tm_gmtoff;
-#elif HAVE_TM___TM_GMT_OFF
-  return t + tm->__tm_gmtoff;
-#else
-  return (time_t)-2; /* Not supported */
 #endif
 }

@@ -1286,7 +1310,25 @@ static void set_tm_yday(struct tm *tm) {
  tm->tm_yday = yday;
 }

-#ifdef HAVE_STRPTIME
+#ifndef HAVE_STRPTIME
+static char *strptime(const char *s, const char *format, struct tm *tm) {
+  if (strcmp(format, "%Y-%m-%dT%H:%M:%SZ"))
+    return NULL;
+
+  int count, end;
+  count = sscanf(s, "%d-%d-%dT%d:%d:%d%n",
+                    &tm->tm_year, &tm->tm_mon, &tm->tm_mday,
+                    &tm->tm_hour, &tm->tm_min, &tm->tm_sec,
+                    &end );
+  if (count == 6 && s[end] == 'Z') {
+    tm->tm_year -= 1900;
+    tm->tm_mon--;
+    return (char*)s + end + 1;
+  }
+  return NULL;
+}
+#endif
+
 static jv f_strptime(jq_state *jq, jv a, jv b) {
  if (jv_get_kind(a) != JV_KIND_STRING || jv_get_kind(b) != JV_KIND_STRING) {
    return ret_error2(a, b, jv_string("strptime/1 requires string inputs and arguments"));
@@ -1298,8 +1340,13 @@ static jv f_strptime(jq_state *jq, jv a, jv b) {
  tm.tm_yday = 367; // sentinel
  const char *input = jv_string_value(a);
  const char *fmt = jv_string_value(b);
-  const char *end = strptime(input, fmt, &tm);

+#ifndef HAVE_STRPTIME
+  if (strcmp(fmt, "%Y-%m-%dT%H:%M:%SZ")) {
+    return ret_error2(a, b, jv_string("strptime/1 only supports ISO 8601 on this platform"));
+  }
+#endif
+  const char *end = strptime(input, fmt, &tm);
  if (end == NULL || (*end != '\0' && !isspace(*end))) {
    return ret_error2(a, b, jv_string_fmt("date \"%s\" does not match format \"%s\"", input, fmt));
  }
@@ -1336,13 +1383,6 @@ static jv f_strptime(jq_state *jq, jv a, jv b) {
  jv_free(a); // must come after `*end` because `end` is a pointer into `a`'s string
  return r;
 }
-#else
-static jv f_strptime(jq_state *jq, jv a, jv b) {
-  jv_free(a);
-  jv_free(b);
-  return jv_invalid_with_msg(jv_string("strptime/1 not implemented on this platform"));
-}
-#endif

 #define TO_TM_FIELD(t, j, i)                    \
    do {                                        \
@@ -1630,7 +1670,7 @@ static const struct cfunction function_list[] = {
  {(cfunction_ptr)f_max, "max", 1},
  {(cfunction_ptr)f_min_by_impl, "_min_by_impl", 2},
  {(cfunction_ptr)f_max_by_impl, "_max_by_impl", 2},
-  {(cfunction_ptr)f_error, "error", 2},
+  {(cfunction_ptr)f_error, "error", 1},
  {(cfunction_ptr)f_format, "format", 2},
  {(cfunction_ptr)f_env, "env", 1},
  {(cfunction_ptr)f_halt, "halt", 1},
@@ -1640,7 +1680,7 @@ static const struct cfunction function_list[] = {
  {(cfunction_ptr)f_get_jq_origin, "get_jq_origin", 1},
  {(cfunction_ptr)f_match, "_match_impl", 4},
  {(cfunction_ptr)f_modulemeta, "modulemeta", 1},
-  {(cfunction_ptr)f_input, "_input", 1},
+  {(cfunction_ptr)f_input, "input", 1},
  {(cfunction_ptr)f_debug, "debug", 1},
  {(cfunction_ptr)f_stderr, "stderr", 1},
  {(cfunction_ptr)f_strptime, "strptime", 2},
@@ -1703,12 +1743,10 @@ static block bind_bytecoded_builtins(block b) {
                                            BLOCK(gen_param("start"), gen_param("end")),
                                            range));
  }
-  return block_bind(builtins, b, OP_IS_CALL_PSEUDO);
+  return BLOCK(builtins, b);
 }

-
-
-static const char* const jq_builtins =
+static const char jq_builtins[] =
 /* Include jq-coded builtins */
 #include "src/builtin.inc"

@@ -1744,46 +1782,18 @@ static block gen_builtin_list(block builtins) {
  return BLOCK(builtins, gen_function("builtins", gen_noop(), gen_const(list)));
 }

-static int builtins_bind_one(jq_state *jq, block* bb, const char* code) {
-  struct locfile* src;
-  src = locfile_init(jq, "<builtin>", code, strlen(code));
-  block funcs;
-  int nerrors = jq_parse_library(src, &funcs);
-  if (nerrors == 0) {
-    *bb = block_bind(funcs, *bb, OP_IS_CALL_PSEUDO);
-  }
-  locfile_free(src);
-  return nerrors;
-}
-
-static int slurp_lib(jq_state *jq, block* bb) {
-  int nerrors = 0;
-  char* home = getenv("HOME");
-  if (home) {    // silently ignore no $HOME
-    jv filename = jv_string_append_str(jv_string(home), "/.jq");
-    jv data = jv_load_file(jv_string_value(filename), 1);
-    if (jv_is_valid(data)) {
-      nerrors = builtins_bind_one(jq, bb, jv_string_value(data) );
-    }
-    jv_free(filename);
-    jv_free(data);
-  }
-  return nerrors;
-}
-
 int builtins_bind(jq_state *jq, block* bb) {
-  block builtins = gen_noop();
-  int nerrors = slurp_lib(jq, bb);
-  if (nerrors) {
-    block_free(*bb);
-    return nerrors;
-  }
-  nerrors = builtins_bind_one(jq, &builtins, jq_builtins);
+  block builtins;
+  struct locfile* src = locfile_init(jq, "<builtin>", jq_builtins, sizeof(jq_builtins)-1);
+  int nerrors = jq_parse_library(src, &builtins);
  assert(!nerrors);
+  locfile_free(src);
+
  builtins = bind_bytecoded_builtins(builtins);
  builtins = gen_cbinding(function_list, sizeof(function_list)/sizeof(function_list[0]), builtins);
  builtins = gen_builtin_list(builtins);
-  *bb = block_bind(builtins, *bb, OP_IS_CALL_PSEUDO);
+
+  *bb = block_bind_incremental(builtins, *bb, OP_IS_CALL_PSEUDO);
  *bb = block_drop_unreferenced(*bb);
  return nerrors;
 }
@@ -1,5 +1,5 @@
 def halt_error: halt_error(5);
-def error: error(.);
+def error(msg): msg|error;
 def map(f): [.[] | f];
 def select(f): if f then . else empty end;
 def sort_by(f): _sort_by_impl(map([f]));
@@ -10,7 +10,7 @@ def max_by(f): _max_by_impl(map([f]));
 def min_by(f): _min_by_impl(map([f]));
 def add: reduce .[] as $x (null; . + $x);
 def del(f): delpaths([path(f)]);
-def _assign(paths; value): value as $v | reduce path(paths) as $p (.; setpath($p; $v));
+def _assign(paths; $value): reduce path(paths) as $p (.; setpath($p; $value));
 def _modify(paths; update): reduce path(paths) as $p (.; label $out | (setpath($p; getpath($p) | update) | ., break $out), delpaths([$p]));
 def map_values(f): .[] |= f;

@@ -168,9 +168,9 @@ def until(cond; next):
         if cond then . else (next|_until) end;
     _until;
 def limit($n; exp):
-  if $n < 0 then exp
-  else label $out | foreach exp as $item ($n; .-1; $item, if . <= 0 then break $out else empty end)
-  end;
+    if $n > 0 then label $out | foreach exp as $item ($n; .-1; $item, if . <= 0 then break $out else empty end)
+    elif $n == 0 then empty
+    else exp end;
 def isempty(g): 0 == ((label $go | g | (1, break $go)) // 0);
 def first(g): label $out | g | ., break $out;
 def last(g): reduce g as $item (null; $item);
@@ -200,12 +200,11 @@ def transpose:
 	        end;
 def in(xs): . as $x | xs | has($x);
 def inside(xs): . as $x | xs | contains($x);
-def input: _input;
 def repeat(exp):
     def _repeat:
         exp, _repeat;
     _repeat;
-def inputs: try repeat(_input) catch if .=="break" then empty else .|error end;
+def inputs: try repeat(input) catch if .=="break" then empty else .|error end;
 # like ruby's downcase - only characters A to Z are affected
 def ascii_downcase:
  explode | map( if 65 <= . and . <= 90 then . + 32  else . end) | implode;
@@ -217,22 +216,25 @@ def ascii_upcase:
 def truncate_stream(stream):
  . as $n | null | stream | . as $input | if (.[0]|length) > $n then setpath([0];$input[0][$n:]) else empty end;
 def fromstream(i):
-  foreach i as $item (
-    [null,false,null,false];
-    if ($item[0]|length) == 0 then [null,false,.[2],.[3]]
-    elif ($item|length) == 1 and ($item[0]|length) < 2 then [null,false,.[0],.[1]]
-    else . end |
-    . as $state |
-    if ($item|length) > 1 and ($item[0]|length) > 0 then
-      [.[0]|setpath(($item|.[0]); ($item|.[1])),
-      true,
-      $state[2],
-      $state[3]]
+  foreach i as $i (
+    [null, null];
+
+    if ($i | length) == 2 then
+      if ($i[0] | length) == 0 then .
+      else [ ( .[0] | setpath($i[0]; $i[1]) ), .[1] ]
+      end
+    elif ($i[0] | length) == 1 then [ null, .[0] ]
    else .
    end;
-    if ($item[0]|length) == 1 and ($item|length == 1) and .[3] then .[2] else empty end,
-    if ($item[0]|length) == 0 then $item[1] else empty end
-    );
+
+    if ($i | length) == 1 then
+      if ($i[0] | length) == 1 then .[1]
+      else empty
+      end
+    elif ($i[0] | length) == 0 then $i[1]
+    else empty
+    end
+  );
 def tostream:
  {string:true,number:true,boolean:true,null:true} as $leaf_types |
  . as $dot |
@@ -49,9 +49,11 @@ struct inst {
  // Unbound instructions (references to other things that may or may not
  // exist) are created by "gen_foo_unbound", and bindings are created by
  // block_bind(definition, body), which binds all instructions in
-  // body which are unboudn and refer to "definition" by name.
+  // body which are unbound and refer to "definition" by name.
  struct inst* bound_by;
  char* symbol;
+  int any_unbound;
+  int referenced;

  int nformals;
  int nactuals;
@@ -73,6 +75,8 @@ static inst* inst_new(opcode op) {
  i->bytecode_pos = -1;
  i->bound_by = 0;
  i->symbol = 0;
+  i->any_unbound = 0;
+  i->referenced = 0;
  i->nformals = -1;
  i->nactuals = -1;
  i->subfn = gen_noop();
@@ -156,6 +160,7 @@ block gen_const_global(jv constant, const char *name) {
  inst* i = inst_new(STORE_GLOBAL);
  i->imm.constant = constant;
  i->symbol = strdup(name);
+  i->any_unbound = 0;
  return inst_block(i);
 }

@@ -211,19 +216,22 @@ block gen_op_unbound(opcode op, const char* name) {
  assert(opcode_describe(op)->flags & OP_HAS_BINDING);
  inst* i = inst_new(op);
  i->symbol = strdup(name);
+  i->any_unbound = 1;
  return inst_block(i);
 }

 block gen_op_var_fresh(opcode op, const char* name) {
  assert(opcode_describe(op)->flags & OP_HAS_VARIABLE);
-  return block_bind(gen_op_unbound(op, name),
-                    gen_noop(), OP_HAS_VARIABLE);
+  block b = gen_op_unbound(op, name);
+  b.first->bound_by = b.first;
+  return b;
 }

 block gen_op_bound(opcode op, block binder) {
  assert(block_is_single(binder));
  block b = gen_op_unbound(op, binder.first->symbol);
  b.first->bound_by = binder.first;
+  b.first->any_unbound = 0;
  return b;
 }

@@ -310,21 +318,7 @@ static int block_count_actuals(block b) {
  return args;
 }

-static int block_count_refs(block binder, block body) {
-  int nrefs = 0;
-  for (inst* i = body.first; i; i = i->next) {
-    if (i != binder.first && i->bound_by == binder.first) {
-      nrefs++;
-    }
-    // counting recurses into closures
-    nrefs += block_count_refs(binder, i->subfn);
-    // counting recurses into argument list
-    nrefs += block_count_refs(binder, i->arglist);
-  }
-  return nrefs;
-}
-
-static int block_bind_subblock(block binder, block body, int bindflags, int break_distance) {
+static int block_bind_subblock_inner(int* any_unbound, block binder, block body, int bindflags, int break_distance) {
  assert(block_is_single(binder));
  assert((opcode_describe(binder.first->op)->flags & bindflags) == (bindflags & ~OP_BIND_WILDCARD));
  assert(binder.first->symbol);
@@ -336,6 +330,9 @@ static int block_bind_subblock(block binder, block body, int bindflags, int brea
    binder.first->nformals = block_count_formals(binder);
  int nrefs = 0;
  for (inst* i = body.first; i; i = i->next) {
+    if (i->any_unbound == 0)
+      continue;
+
    int flags = opcode_describe(i->op)->flags;
    if ((flags & bindflags) == (bindflags & ~OP_BIND_WILDCARD) && i->bound_by == 0 &&
        (!strcmp(i->symbol, binder.first->symbol) ||
@@ -357,14 +354,25 @@ static int block_bind_subblock(block binder, block body, int bindflags, int brea
      // a break whenever we come across a STOREV of *anonlabel...
      break_distance++;
    }
+
+    i->any_unbound = (i->symbol && !i->bound_by);
+
    // binding recurses into closures
-    nrefs += block_bind_subblock(binder, i->subfn, bindflags, break_distance);
+    nrefs += block_bind_subblock_inner(&i->any_unbound, binder, i->subfn, bindflags, break_distance);
    // binding recurses into argument list
-    nrefs += block_bind_subblock(binder, i->arglist, bindflags, break_distance);
+    nrefs += block_bind_subblock_inner(&i->any_unbound, binder, i->arglist, bindflags, break_distance);
+
+    if (i->any_unbound)
+      *any_unbound = 1;
  }
  return nrefs;
 }

+static int block_bind_subblock(block binder, block body, int bindflags, int break_distance) {
+  int any_unbound;
+  return block_bind_subblock_inner(&any_unbound, binder, body, bindflags, break_distance);
+}
+
 static int block_bind_each(block binder, block body, int bindflags) {
  assert(block_has_only_binders(binder, bindflags));
  bindflags |= OP_HAS_BINDING;
@@ -375,7 +383,7 @@ static int block_bind_each(block binder, block body, int bindflags) {
  return nrefs;
 }

-block block_bind(block binder, block body, int bindflags) {
+static block block_bind(block binder, block body, int bindflags) {
  block_bind_each(binder, body, bindflags);
  return block_join(binder, body);
 }
@@ -413,72 +421,101 @@ block block_bind_library(block binder, block body, int bindflags, const char *li
  return body; // We don't return a join because we don't want those sticking around...
 }

-// Bind binder to body and throw away any defs in binder not referenced
-// (directly or indirectly) from body.
+// Bind binder to body, then throw it away if not referenced.
 block block_bind_referenced(block binder, block body, int bindflags) {
+  assert(block_is_single(binder));
  assert(block_has_only_binders(binder, bindflags));
  bindflags |= OP_HAS_BINDING;
-  block refd = gen_noop();
-  block unrefd = gen_noop();
-  int nrefs;
-  for (int last_kept = 0, kept = 0; ; ) {
-    for (inst* curr; (curr = block_take(&binder));) {
-      block b = inst_block(curr);
-      nrefs = block_bind_each(b, body, bindflags);
-      // Check if this binder is referenced from any of the ones we
-      // already know are referenced by body.
-      nrefs += block_count_refs(b, refd);
-      nrefs += block_count_refs(b, body);
-      if (nrefs) {
-        refd = BLOCK(refd, b);
-        kept++;
-      } else {
-        unrefd = BLOCK(unrefd, b);
-      }
+
+  if (block_bind_subblock(binder, body, bindflags, 0) == 0) {
+    block_free(binder);
+  } else {
+    body = BLOCK(binder, body);
+  }
+  return body;
+}
+
+static inst* block_take_last(block* b) {
+  inst* i = b->last;
+  if (i == 0)
+    return 0;
+  if (i->prev) {
+    i->prev->next = i->next;
+    b->last = i->prev;
+    i->prev = 0;
+  } else {
+    b->first = 0;
+    b->last = 0;
+  }
+  return i;
+}
+
+// Binds a sequence of binders, which *must not* alrady be bound to each other,
+// to body, throwing away unreferenced defs
+block block_bind_incremental(block binder, block body, int bindflags) {
+  assert(block_has_only_binders(binder, bindflags));
+  bindflags |= OP_HAS_BINDING;
+
+  inst* curr;
+  while ((curr = block_take_last(&binder))) {
+    body = block_bind_referenced(inst_block(curr), body, bindflags);
+  }
+  return body;
+}
+
+block block_bind_self(block binder, int bindflags) {
+  assert(block_has_only_binders(binder, bindflags));
+  bindflags |= OP_HAS_BINDING;
+  block body = gen_noop();
+
+  inst* curr;
+  while ((curr = block_take_last(&binder))) {
+    block b = inst_block(curr);
+    block_bind_subblock(b, body, bindflags, 0);
+    body = BLOCK(b, body);
+  }
+  return body;
+}
+
+static void block_mark_referenced(block body) {
+  int saw_top = 0;
+  for (inst* i = body.last; i; i = i->prev) {
+    if (saw_top && i->bound_by == i && !i->referenced)
+      continue;
+    if (i->op == TOP) {
+      saw_top = 1;
    }
-    if (kept == last_kept)
-      break;
-    last_kept = kept;
-    binder = unrefd;
-    unrefd = gen_noop();
+    if (i->bound_by) {
+      i->bound_by->referenced = 1;
+    }
+
+    block_mark_referenced(i->arglist);
+    block_mark_referenced(i->subfn);
  }
-  block_free(unrefd);
-  return block_join(refd, body);
 }

 block block_drop_unreferenced(block body) {
-  inst* curr;
+  block_mark_referenced(body);
+
  block refd = gen_noop();
-  block unrefd = gen_noop();
-  int drop;
-  do {
-    drop = 0;
-    while ((curr = block_take(&body)) && curr->op != TOP) {
-      block b = inst_block(curr);
-      if (block_count_refs(b,refd) + block_count_refs(b,body) == 0) {
-        unrefd = BLOCK(unrefd, b);
-        drop++;
-      } else {
-        refd = BLOCK(refd, b);
-      }
+  inst* curr;
+  while ((curr = block_take(&body))) {
+    if (curr->bound_by == curr && !curr->referenced) {
+      inst_free(curr);
+    } else {
+      refd = BLOCK(inst_block(curr), refd);
    }
-    if (curr && curr->op == TOP) {
-      body = BLOCK(inst_block(curr),body);
-    }
-    body = BLOCK(refd, body);
-    refd = gen_noop();
-  } while (drop != 0);
-  block_free(unrefd);
-  return body;
+  }
+  return refd;
 }

 jv block_take_imports(block* body) {
  jv imports = jv_array();

-  inst* top = NULL;
-  if (body->first && body->first->op == TOP) {
-    top = block_take(body);
-  }
+  /* Parser should never generate TOP before imports */
+  assert(!(body->first && body->first->op == TOP && body->first->next &&
+        (body->first->next->op == MODULEMETA || body->first->next->op == DEPS)));
+
  while (body->first && (body->first->op == MODULEMETA || body->first->op == DEPS)) {
    inst* dep = block_take(body);
    if (dep->op == DEPS) {
@@ -486,9 +523,6 @@ jv block_take_imports(block* body) {
    }
    inst_free(dep);
  }
-  if (top) {
-    *body = block_join(inst_block(top),*body);
-  }
  return imports;
 }

@@ -550,6 +584,7 @@ block gen_function(const char* name, block formals, block body) {
  }
  i->subfn = body;
  i->symbol = strdup(name);
+  i->any_unbound = -1;
  i->arglist = formals;
  block b = inst_block(i);
  block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING, 0);
@@ -1081,7 +1116,8 @@ block gen_cbinding(const struct cfunction* cfunctions, int ncfunctions, block co
    inst* i = inst_new(CLOSURE_CREATE_C);
    i->imm.cfunc = &cfunctions[cfunc];
    i->symbol = strdup(i->imm.cfunc->name);
-    code = block_bind(inst_block(i), code, OP_IS_CALL_PSEUDO);
+    i->any_unbound = 0;
+    code = BLOCK(inst_block(i), code);
  }
  return code;
 }
@@ -72,9 +72,10 @@ int block_has_only_binders(block, int bindflags);
 int block_has_main(block);
 int block_is_funcdef(block b);
 int block_is_single(block b);
-block block_bind(block binder, block body, int bindflags);
 block block_bind_library(block binder, block body, int bindflags, const char* libname);
 block block_bind_referenced(block binder, block body, int bindflags);
+block block_bind_incremental(block binder, block body, int bindflags);
+block block_bind_self(block binder, int bindflags);
 block block_drop_unreferenced(block body);

 jv block_take_imports(block* body);
@@ -658,10 +658,12 @@ jv jv_string_indexes(jv j, jv k) {
  int idxlen = jv_string_length_bytes(jv_copy(k));
  jv a = jv_array();

-  p = jstr;
-  while ((p = _jq_memmem(p, (jstr + jlen) - p, idxstr, idxlen)) != NULL) {
-    a = jv_array_append(a, jv_number(p - jstr));
-    p += idxlen;
+  if (idxlen != 0) {
+    p = jstr;
+    while ((p = _jq_memmem(p, (jstr + jlen) - p, idxstr, idxlen)) != NULL) {
+      a = jv_array_append(a, jv_number(p - jstr));
+      p += idxlen;
+    }
  }
  jv_free(j);
  jv_free(k);
@@ -1342,7 +1344,13 @@ int jv_contains(jv a, jv b) {
  } else if (jv_get_kind(a) == JV_KIND_ARRAY) {
    r = jv_array_contains(jv_copy(a), jv_copy(b));
  } else if (jv_get_kind(a) == JV_KIND_STRING) {
-    r = strstr(jv_string_value(a), jv_string_value(b)) != 0;
+    int b_len = jv_string_length_bytes(jv_copy(b));
+    if (b_len != 0) {
+      r = _jq_memmem(jv_string_value(a), jv_string_length_bytes(jv_copy(a)),
+                     jv_string_value(b), b_len) != 0;
+    } else {
+      r = 1;
+    }
  } else {
    r = jv_equal(jv_copy(a), jv_copy(b));
  }
@@ -98,12 +98,9 @@ static jv validate_relpath(jv name) {
    return res;
  }
  jv components = jv_string_split(jv_copy(name), jv_string("/"));
-  jv rp = jv_array_get(jv_copy(components), 0);
-  components = jv_array_slice(components, 1, jv_array_length(jv_copy(components)));
  jv_array_foreach(components, i, x) {
    if (!strcmp(jv_string_value(x), "..")) {
      jv_free(x);
-      jv_free(rp);
      jv_free(components);
      jv res = jv_invalid_with_msg(jv_string_fmt("Relative paths to modules may not traverse to parent directories (%s)", s));
      jv_free(name);
@@ -111,18 +108,16 @@ static jv validate_relpath(jv name) {
    }
    if (i > 0 && jv_equal(jv_copy(x), jv_array_get(jv_copy(components), i - 1))) {
      jv_free(x);
-      jv_free(rp);
      jv_free(components);
      jv res = jv_invalid_with_msg(jv_string_fmt("module names must not have equal consecutive components: %s",
                                                 jv_string_value(name)));
      jv_free(name);
      return res;
    }
-    rp = jv_string_concat(rp, jv_string_concat(jv_string("/"), x));
+    jv_free(x);
  }
  jv_free(components);
-  jv_free(name);
-  return rp;
+  return name;
 }

 // Assumes name has been validated
@@ -138,10 +133,20 @@ static jv jv_basename(jv name) {

 // Asummes validated relative path to module
 static jv find_lib(jq_state *jq, jv rel_path, jv search, const char *suffix, jv jq_origin, jv lib_origin) {
-  if (jv_get_kind(search) != JV_KIND_ARRAY)
-    return jv_invalid_with_msg(jv_string_fmt("Module search path must be an array"));
-  if (jv_get_kind(rel_path) != JV_KIND_STRING)
+  if (!jv_is_valid(rel_path)) {
+    jv_free(search);
+    return rel_path;
+  }
+  if (jv_get_kind(rel_path) != JV_KIND_STRING) {
+    jv_free(rel_path);
+    jv_free(search);
    return jv_invalid_with_msg(jv_string_fmt("Module path must be a string"));
+  }
+  if (jv_get_kind(search) != JV_KIND_ARRAY) {
+    jv_free(rel_path);
+    jv_free(search);
+    return jv_invalid_with_msg(jv_string_fmt("Module search path must be an array"));
+  }

  struct stat st;
  int ret;
@@ -233,14 +238,17 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block
  jv deps = block_take_imports(src_block);
  block bk = *src_block;
  int nerrors = 0;
-  const char *as_str = NULL;

  jv_array_foreach(deps, i, dep) {
+    const char *as_str = NULL;
    int is_data = jv_get_kind(jv_object_get(jv_copy(dep), jv_string("is_data"))) == JV_KIND_TRUE;
    int raw = 0;
    jv v = jv_object_get(jv_copy(dep), jv_string("raw"));
    if (jv_get_kind(v) == JV_KIND_TRUE)
      raw = 1;
+    int optional = 0;
+    if (jv_get_kind(jv_object_get(jv_copy(dep), jv_string("optional"))) == JV_KIND_TRUE)
+      optional = 1;
    jv_free(v);
    jv relpath = validate_relpath(jv_object_get(jv_copy(dep), jv_string("relpath")));
    jv as = jv_object_get(jv_copy(dep), jv_string("as"));
@@ -254,35 +262,51 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block
    jv resolved = find_lib(jq, relpath, search, is_data ? ".json" : ".jq", jv_copy(jq_origin), jv_copy(lib_origin));
    // XXX ...move the rest of this into a callback.
    if (!jv_is_valid(resolved)) {
+      jv_free(as);
+      if (optional) {
+        jv_free(resolved);
+        continue;
+      }
      jv emsg = jv_invalid_get_msg(resolved);
      jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg)));
      jv_free(emsg);
-      jv_free(as);
      jv_free(deps);
      jv_free(jq_origin);
      jv_free(lib_origin);
      return 1;
    }
-    uint64_t state_idx = 0;
-    for (; state_idx < lib_state->ct; ++state_idx) {
-      if (strcmp(lib_state->names[state_idx],jv_string_value(resolved)) == 0)
-        break;
-    }
-    if (state_idx < lib_state->ct) { // Found
-      jv_free(resolved);
-      // Bind the library to the program
-      bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, as_str);
-    } else { // Not found.   Add it to the table before binding.
-      block dep_def_block = gen_noop();
+
+    if (is_data) {
+      // Can't reuse data libs because the wrong name is bound
+      block dep_def_block;
      nerrors += load_library(jq, resolved, is_data, raw, as_str, &dep_def_block, lib_state);
-      // resolved has been freed
      if (nerrors == 0) {
-        // Bind the library to the program
+        // Bind as both $data::data and $data for backward compatibility vs common sense
        bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, as_str);
-        if (is_data)
-          bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, NULL);
+        bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, NULL);
+      }
+    } else {
+      uint64_t state_idx = 0;
+      for (; state_idx < lib_state->ct; ++state_idx) {
+        if (strcmp(lib_state->names[state_idx],jv_string_value(resolved)) == 0)
+          break;
+      }
+
+      if (state_idx < lib_state->ct) { // Found
+        jv_free(resolved);
+        // Bind the library to the program
+        bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, as_str);
+      } else { // Not found.   Add it to the table before binding.
+        block dep_def_block = gen_noop();
+        nerrors += load_library(jq, resolved, is_data, raw, as_str, &dep_def_block, lib_state);
+        // resolved has been freed
+        if (nerrors == 0) {
+          // Bind the library to the program
+          bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, as_str);
+        }
      }
    }
+
    jv_free(as);
  }
  jv_free(lib_origin);
@@ -324,6 +348,7 @@ static int load_library(jq_state *jq, jv lib_path, int is_data, int raw, const c
                                      jv_string(dirname(lib_origin)),
                                      &program, lib_state);
      free(lib_origin);
+      program = block_bind_self(program, OP_IS_CALL_PSEUDO);
    }
  }
  state_idx = lib_state->ct++;
@@ -375,6 +400,16 @@ int load_program(jq_state *jq, struct locfile* src, block *out_block) {
  if (nerrors)
    return nerrors;

+  char* home = getenv("HOME");
+  if (home) {    // silently ignore no $HOME
+    /* Import ~/.jq as a library named "" found in $HOME */
+    block import = gen_import_meta(gen_import("", NULL, 0),
+        gen_const(JV_OBJECT(
+            jv_string("optional"), jv_true(),
+            jv_string("search"), jv_string(home))));
+    program = BLOCK(import, program);
+  }
+
  nerrors = process_dependencies(jq, jq_get_jq_origin(jq), jq_get_prog_origin(jq), &program, &lib_state);
  block libs = gen_noop();
  for (uint64_t i = 0; i < lib_state.ct; ++i) {
@@ -146,6 +146,17 @@ enum {
 };
 static int options = 0;

+enum {
+    JQ_OK              =  0,
+    JQ_OK_NULL_KIND    = -1, /* exit 0 if --exit-status is not set*/
+    JQ_ERROR_SYSTEM    =  2,
+    JQ_ERROR_COMPILE   =  3,
+    JQ_OK_NO_OUTPUT    = -4, /* exit 0 if --exit-status is not set*/
+    JQ_ERROR_UNKNOWN   =  5,
+};
+#define jq_exit_with_status(r)  exit(abs(r))
+#define jq_exit(r)              exit( r > 0 ? r : 0 )
+
 static const char *skip_shebang(const char *p) {
  if (strncmp(p, "#!", sizeof("#!") - 1) != 0)
    return p;
@@ -162,23 +173,23 @@ static const char *skip_shebang(const char *p) {
 }

 static int process(jq_state *jq, jv value, int flags, int dumpopts) {
-  int ret = 14; // No valid results && -e -> exit(4)
+  int ret = JQ_OK_NO_OUTPUT; // No valid results && -e -> exit(4)
  jq_start(jq, value, flags);
  jv result;
  while (jv_is_valid(result = jq_next(jq))) {
    if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
      if (options & ASCII_OUTPUT) {
-        jv_dumpf(result, stdout, JV_PRINT_ASCII);
+        jv_dumpf(jv_copy(result), stdout, JV_PRINT_ASCII);
      } else {
        fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
      }
-      ret = 0;
+      ret = JQ_OK;
      jv_free(result);
    } else {
      if (jv_get_kind(result) == JV_KIND_FALSE || jv_get_kind(result) == JV_KIND_NULL)
-        ret = 11;
+        ret = JQ_OK_NULL_KIND;
      else
-        ret = 0;
+        ret = JQ_OK;
      if (options & SEQ)
        priv_fwrite("\036", 1, stdout, dumpopts & JV_PRINT_ISATTY);
      jv_dump(result, dumpopts);
@@ -193,11 +204,11 @@ static int process(jq_state *jq, jv value, int flags, int dumpopts) {
    options |= EXIT_STATUS_EXACT;
    jv exit_code = jq_get_exit_code(jq);
    if (!jv_is_valid(exit_code))
-      ret = 0;
+      ret = JQ_OK;
    else if (jv_get_kind(exit_code) == JV_KIND_NUMBER)
      ret = jv_number_value(exit_code);
    else
-      ret = 5;
+      ret = JQ_ERROR_UNKNOWN;
    jv_free(exit_code);
    jv error_message = jq_get_error_message(jq);
    if (jv_get_kind(error_message) == JV_KIND_STRING) {
@@ -222,7 +233,7 @@ static int process(jq_state *jq, jv value, int flags, int dumpopts) {
      fprintf(stderr, "jq: error (at %s) (not a string): %s\n",
              jv_string_value(input_pos), jv_string_value(msg));
    }
-    ret = 5;
+    ret = JQ_ERROR_UNKNOWN;
    jv_free(input_pos);
    jv_free(msg);
  }
@@ -236,12 +247,32 @@ static void debug_cb(void *data, jv input) {
  fprintf(stderr, "\n");
 }

+#ifdef WIN32
+int umain(int argc, char* argv[]);
+
+int wmain(int argc, wchar_t* wargv[]) {
+  size_t arg_sz;
+  char **argv = alloca(argc * sizeof(wchar_t*));
+  for (int i = 0; i < argc; i++) {
+    argv[i] = alloca((arg_sz = WideCharToMultiByte(CP_UTF8,
+                                                   0,
+                                                   wargv[i],
+                                                   -1, 0, 0, 0, 0)));
+    WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i], arg_sz, 0, 0);
+  }
+  return umain(argc, argv);
+}
+
+int umain(int argc, char* argv[]) {
+#else /*}*/
 int main(int argc, char* argv[]) {
+#endif
  jq_state *jq = NULL;
-  int ret = 0;
+  int ret = JQ_OK_NO_OUTPUT;
  int compiled = 0;
  int parser_flags = 0;
  int nfiles = 0;
+  int last_result = -1; /* -1 = no result, 0=null or false, 1=true */
  int badwrite;
  jv ARGS = jv_array(); /* positional arguments */
  jv program_arguments = jv_object(); /* named arguments */
@@ -251,17 +282,6 @@ int main(int argc, char* argv[]) {
  fflush(stderr);
  _setmode(fileno(stdout), _O_TEXT | _O_U8TEXT);
  _setmode(fileno(stderr), _O_TEXT | _O_U8TEXT);
-  int wargc;
-  wchar_t **wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
-  assert(wargc == argc);
-  size_t arg_sz;
-  for (int i = 0; i < argc; i++) {
-    argv[i] = alloca((arg_sz = WideCharToMultiByte(CP_UTF8,
-                                                   0,
-                                                   wargv[i],
-                                                   -1, 0, 0, 0, 0)));
-    WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i], arg_sz, 0, 0);
-  }
 #endif

  if (argc) progname = argv[0];
@@ -269,7 +289,7 @@ int main(int argc, char* argv[]) {
  jq = jq_init();
  if (jq == NULL) {
    perror("malloc");
-    ret = 2;
+    ret = JQ_ERROR_SYSTEM;
    goto out;
  }

@@ -469,7 +489,7 @@ int main(int argc, char* argv[]) {
            fprintf(stderr, "%s: Bad JSON in --%s %s %s: %s\n", progname, which,
                    argv[i+1], argv[i+2], jv_string_value(data));
            jv_free(data);
-            ret = 2;
+            ret = JQ_ERROR_SYSTEM;
            goto out;
          }
          if (strcmp(which, "argfile") == 0 &&
@@ -498,7 +518,7 @@ int main(int argc, char* argv[]) {
      }
      if (isoption(argv[i], 'V', "version", &short_opts)) {
        printf("jq-%s\n", JQ_VERSION);
-        ret = 0;
+        ret = JQ_OK;
        goto out;
      }
      if (isoption(argv[i], 0, "run-tests", &short_opts)) {
@@ -521,11 +541,20 @@ int main(int argc, char* argv[]) {

 #ifdef USE_ISATTY
  if (isatty(STDOUT_FILENO)) {
-    dumpopts |= JV_PRINT_ISATTY;
 #ifndef WIN32
-  /* Disable color by default on Windows builds as Windows
-     terminals tend not to display it correctly */
-    dumpopts |= JV_PRINT_COLOR;
+    dumpopts |= JV_PRINT_ISATTY | JV_PRINT_COLOR;
+#else
+  /* Verify we actually have the console, as the NUL device is also regarded as
+     tty.  Windows can handle color if ANSICON (or ConEmu) is installed, or
+     Windows 10 supports the virtual terminal */
+    DWORD mode;
+    HANDLE con = GetStdHandle(STD_OUTPUT_HANDLE);
+    if (GetConsoleMode(con, &mode)) {
+      dumpopts |= JV_PRINT_ISATTY;
+      if (getenv("ANSICON") != NULL ||
+          SetConsoleMode(con, mode | 4/*ENABLE_VIRTUAL_TERMINAL_PROCESSING*/))
+        dumpopts |= JV_PRINT_COLOR;
+    }
 #endif
  }
 #endif
@@ -577,7 +606,7 @@ int main(int argc, char* argv[]) {
      data = jv_invalid_get_msg(data);
      fprintf(stderr, "%s: %s\n", progname, jv_string_value(data));
      jv_free(data);
-      ret = 2;
+      ret = JQ_ERROR_SYSTEM;
      goto out;
    }
    jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string(dirname(program_origin))));
@@ -595,7 +624,7 @@ int main(int argc, char* argv[]) {
    compiled = jq_compile_args(jq, program, jv_copy(program_arguments));
  }
  if (!compiled){
-    ret = 3;
+    ret = JQ_ERROR_COMPILE;
    goto out;
  }

@@ -629,6 +658,8 @@ int main(int argc, char* argv[]) {
           (jv_is_valid((value = jq_util_input_next_input(input_state))) || jv_invalid_has_msg(jv_copy(value)))) {
      if (jv_is_valid(value)) {
        ret = process(jq, value, jq_flags, dumpopts);
+        if (ret <= 0 && ret != JQ_OK_NO_OUTPUT)
+          last_result = (ret != JQ_OK_NULL_KIND);
        continue;
      }

@@ -636,7 +667,7 @@ int main(int argc, char* argv[]) {
      jv msg = jv_invalid_get_msg(value);
      if (!(options & SEQ)) {
        // --seq -> errors are not fatal
-        ret = 4;
+        ret = JQ_OK_NO_OUTPUT;
        fprintf(stderr, "parse error: %s\n", jv_string_value(msg));
        jv_free(msg);
        break;
@@ -647,22 +678,29 @@ int main(int argc, char* argv[]) {
  }

  if (jq_util_input_errors(input_state) != 0)
-    ret = 2;
+    ret = JQ_ERROR_SYSTEM;

 out:
  badwrite = ferror(stdout);
  if (fclose(stdout)!=0 || badwrite) {
    fprintf(stderr,"Error: writing output failed: %s\n", strerror(errno));
-    ret = 2;
+    ret = JQ_ERROR_SYSTEM;
  }

  jv_free(ARGS);
  jv_free(program_arguments);
  jq_util_input_free(&input_state);
  jq_teardown(&jq);
-  if (ret >= 10 && (options & EXIT_STATUS))
-    return ret - 10;
-  if (ret >= 10 && !(options & EXIT_STATUS_EXACT))
-    return 0;
-  return ret;
+
+  if (options & (EXIT_STATUS|EXIT_STATUS_EXACT)) {
+    if (ret != JQ_OK_NO_OUTPUT)
+      jq_exit_with_status(ret);
+    else
+      switch (last_result) {
+        case -1: jq_exit_with_status(JQ_OK_NO_OUTPUT);
+        case  0: jq_exit_with_status(JQ_OK_NULL_KIND);
+        default: jq_exit_with_status(JQ_OK);
+      }
+  } else
+    jq_exit(ret);
 }
@@ -1,8 +1,8 @@
-/* A Bison parser, made by GNU Bison 3.0.4.  */
+/* A Bison parser, made by GNU Bison 3.0.5.  */

 /* Bison interface for Yacc-like parsers in C

-   Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
+   Copyright (C) 1984, 1989-1990, 2000-2015, 2018 Free Software Foundation, Inc.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -40,7 +40,7 @@
 extern int yydebug;
 #endif
 /* "%code requires" blocks.  */
-#line 11 "src/parser.y" /* yacc.c:1909  */
+#line 11 "src/parser.y" /* yacc.c:1910  */

 #include "locfile.h"
 struct lexer_param;
@@ -57,7 +57,7 @@ struct lexer_param;
    }                                           \
  } while (0)

-#line 61 "src/parser.h" /* yacc.c:1909  */
+#line 61 "src/parser.h" /* yacc.c:1910  */

 /* Token type.  */
 #ifndef YYTOKENTYPE
@@ -163,12 +163,12 @@ struct lexer_param;

 union YYSTYPE
 {
-#line 31 "src/parser.y" /* yacc.c:1909  */
+#line 31 "src/parser.y" /* yacc.c:1910  */

  jv literal;
  block blk;

-#line 172 "src/parser.h" /* yacc.c:1909  */
+#line 172 "src/parser.h" /* yacc.c:1910  */
 };

 typedef union YYSTYPE YYSTYPE;
@@ -331,7 +331,7 @@ FuncDefs:
  $$ = gen_noop();
 } |
 FuncDef FuncDefs {
-  $$ = block_bind($1, $2, OP_IS_CALL_PSEUDO);
+  $$ = block_join($1, $2);
 }

 Exp:
@@ -357,6 +357,9 @@ Term "as" Patterns '|' Exp {
 "if" Exp "then" Exp ElseBody {
  $$ = gen_cond($2, $4, $5);
 } |
+"if" Exp "then" Exp "end" {
+  $$ = gen_cond($2, $4, gen_noop());
+} |
 "if" Exp "then" error {
  FAIL(@$, "Possibly unterminated 'if' statement");
  $$ = $2;
@@ -928,6 +931,10 @@ IDENT ':' ExpD {
  $$ = gen_dictpair($1, BLOCK(gen_op_simple(POP), gen_op_simple(DUP2),
                              gen_op_simple(DUP2), gen_op_simple(INDEX)));
  }
+| '$' IDENT ':' ExpD {
+  $$ = gen_dictpair(gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($2))),
+                    $4);
+  }
 | '$' IDENT {
  $$ = gen_dictpair(gen_const($2),
                    gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($2))));
@@ -309,7 +309,7 @@ static int jq_util_input_read_more(jq_util_input_state *state) {
  state->buf_valid_len = 0;
  if (state->current_input) {
    char *res;
-    memset(state->buf, 0, sizeof(state->buf));
+    memset(state->buf, 0xff, sizeof(state->buf));

    while (!(res = fgets(state->buf, sizeof(state->buf), state->current_input)) &&
           ferror(state->current_input) && errno == EINTR)
@@ -334,20 +334,22 @@ static int jq_util_input_read_more(jq_util_input_state *state) {
        size_t i;

        /*
-         * XXX We can't know how many bytes we've read!
+         * XXX We don't know how many bytes we've read!
         *
         * We can't use getline() because there need not be any newlines
         * in the input.  The only entirely correct choices are: use
         * fgetc() or fread().  Using fread() will complicate buffer
         * management here.
         *
-         * For now we guess how much fgets() read.
+         * For now we check how much fgets() read by scanning backwards for the
+         * terminating '\0'. This only works because we previously memset our
+         * buffer with something nonzero.
         */
-        for (p = state->buf, i = 0; i < sizeof(state->buf); i++) {
-          if (state->buf[i] != '\0')
-            p = &state->buf[i];
+        for (p = state->buf, i = sizeof(state->buf) - 1; i > 0; i--) {
+          if (state->buf[i] == '\0')
+            break;
        }
-        state->buf_valid_len = p - state->buf + 1;
+        state->buf_valid_len = i;
      } else if (p == NULL) {
        state->buf_valid_len = sizeof(state->buf) - 1;
      } else {
@@ -304,6 +304,14 @@ null
 [11,22,33,44,55,66,77,88,99]
 [11,22,33]

+[limit(0; error)]
+"badness"
+[]
+
+[limit(1; 1, error)]
+"badness"
+[1]
+
 [first(range(.)), last(range(.)), nth(0; range(.)), nth(5; range(.)), try nth(-1; range(.)) catch .]
 10
 [0,9,0,5,"nth doesn't support negative indices"]
@@ -1036,6 +1044,19 @@ null
 null
 [5,6]

+[if true then 3 end]
+7
+[3]
+
+[if false then 3 end]
+7
+[7]
+
+[if false then 3 else . end]
+7
+[7]
+
+
 [.[] | [.foo[] // .bar]]
 [{"foo":[1,2], "bar": 42}, {"foo":[1], "bar": null}, {"foo":[null,false,3], "bar": 18}, {"foo":[], "bar":42}, {"foo": [null,false,null], "bar": 41}]
 [[1,2], [1], [3], [42], [41]]
@@ -1091,6 +1112,28 @@ null
 {}
 [true, true, false]

+# containment operator (embedded NULs!)
+[contains(""), contains("\u0000")]
+"\u0000"
+[true, true]
+
+[contains(""), contains("a"), contains("ab"), contains("c"), contains("d")]
+"ab\u0000cd"
+[true, true, true, true, true]
+
+[contains("cd"), contains("b\u0000"), contains("ab\u0000")]
+"ab\u0000cd"
+[true, true, true]
+
+[contains("b\u0000c"), contains("b\u0000cd"), contains("b\u0000cd")]
+"ab\u0000cd"
+[true, true, true]
+
+[contains("@"), contains("\u0000@"), contains("\u0000what")]
+"ab\u0000cd"
+[false, false, false]
+
+
 # Try/catch and general `?` operator
 [.[]|try if . == 0 then error("foo") elif . == 1 then .a elif . == 2 then empty else . end catch .]
 [0,1,2,3]
@@ -1370,6 +1413,10 @@ include "c"; [a, c]
 null
 [0,"acmehbah"]

+import "data" as $e; import "data" as $d; [$d[].this,$e[].that,$d::d[].this,$e::e[].that]|join(";")
+null
+"is a test;is too;is a test;is too"
+
 %%FAIL
 module (.+1); 0
 jq: error: Module metadata must be constant at <top-level>, line 1:
@@ -1516,4 +1563,9 @@ isempty(1,error("foo"))
 null
 false

+# Regression test for #1815
+index("")
+""
+null
+

@@ -3,5 +3,5 @@
 . "${0%/*}/setup" "$@"

 # We set PAGER because there's a mantest for `env` that uses it.
-(cd $JQBASEDIR/docs && rake mantests) |
+(cd $JQBASEDIR/docs && pipenv run python3 build_mantests.py) |
    env PAGER=less $VALGRIND $Q $JQ -L "$mods" --run-tests
@@ -96,14 +96,20 @@ cmp $d/out $d/expected
 cat > $d/expected <<EOF
 ignoring parse error: Unfinished abandoned text at EOF at line 1, column 4
 EOF
-printf '"foo' | $JQ -ce --seq . > $d/out 2>&1
+printf '"foo' | $JQ -c --seq . > $d/out 2>&1
+cmp $d/out $d/expected
+
+# with -e option should give 4 here as there's no valid output after
+# ignoring parse errors with --seq.
+printf '"foo' | $JQ -ce --seq . > $d/out 2>&1 || ret=$?
+[ $ret -eq 4 ]
 cmp $d/out $d/expected

 # Numeric values truncated by EOF are ignored
 cat > $d/expected <<EOF
 ignoring parse error: Unfinished abandoned text at EOF at line 1, column 1
 EOF
-printf '1' | $JQ -ce --seq . > $d/out 2>&1
+printf '1' | $JQ -c --seq . > $d/out 2>&1
 cmp $d/out $d/expected

 cat > $d/expected <<EOF
@@ -115,6 +121,21 @@ if printf '1\n' | $JQ -cen --seq '[inputs] == []' >/dev/null 2> $d/out; then
 fi
 cmp $d/out $d/expected

+
+## Test --exit-status
+data='{"i": 1}\n{"i": 2}\n{"i": 3}\n'
+echo "$data" | $JQ --exit-status 'select(.i==1)' > /dev/null 2>&1
+echo "$data" | $JQ --exit-status 'select(.i==2)' > /dev/null 2>&1
+echo "$data" | $JQ --exit-status 'select(.i==3)' > /dev/null 2>&1
+ret=0
+echo "$data" | $JQ --exit-status 'select(.i==4)' > /dev/null 2>&1 || ret=$?
+[ $ret -eq 4 ]
+ret=0
+echo "$data" | $JQ --exit-status 'select(.i==2) | false' > /dev/null 2>&1 || ret=$?
+[ $ret -eq 1 ]
+echo "$data" | $JQ --exit-status 'select(.i==2) | true' > /dev/null 2>&1
+
+
 # Regression test for #951
 printf '"a\n' > $d/input
 if $VALGRIND $Q $JQ -e . $d/input; then
Author	SHA1	Message	Date
William Langford	51fc9e02a5	fixup! Configure appveyor build to use python	2019-02-22 20:05:09 -05:00
William Langford	0d31bdc4cf	fixup! Configure appveyor build to use python	2019-02-22 20:02:44 -05:00
William Langford	0960bfc06d	fixup! Configure appveyor build to use python	2019-02-22 19:53:14 -05:00
William Langford	22e6f9e966	Change contains to return true for empty string needles The behavior of memmem for an empty needle is inconsistent between implementations of libc. Our tests imply that we want an empty string needle to be true, so check for an empty needle before calling memmem.	2019-02-22 19:50:26 -05:00
William Langford	df8346a935	Configure appveyor build to use python	2019-02-22 19:50:26 -05:00
William Langford	e1fb525796	Cleanup leftover references to ruby	2019-02-22 19:32:31 -05:00
William Langford	da170faafe	Configure travis build to use python	2019-02-22 19:32:31 -05:00
William Langford	03f746bf24	Remove ruby dependency from mantests	2019-02-22 19:32:31 -05:00
William Langford	5989388c9c	Remove ruby dependency from manpage build	2019-02-22 19:32:31 -05:00
William Langford	208c668818	Remove ruby dependency from website build	2019-02-22 19:32:31 -05:00
Nico Williams	9a0d5be790	Merge pull request #1837 from muhmuhten/limit_0	2019-02-21 08:17:41 -06:00
Muh Muhten	130152a2ac	Ensure limit(0; ...) is empty	2019-02-21 04:11:41 -05:00
Muh Muhten	730c730940	Rewrite `_assign/2` using `$argument` syntax	2019-02-20 20:12:52 -06:00
Muh Muhten	85caac4473	Work around fgets()'s lack of length reporting Stop losing \0s from the end of files because we don't know how much fgets() read into our buffer. If we prefill the buffer with anything other than zeroes, we can actually tell the difference between the terminator and unused space in the buffer. This is possibly more expensive, but at least it should be correct. Fixes #1504	2019-02-20 20:12:52 -06:00
Nicolas Williams	048acee39b	Fix if-then-else example formatting	2019-02-20 20:03:26 -06:00
Nicolas Williams	8a02b2c9c1	Document optional `else` clause of `if`	2019-02-20 20:03:26 -06:00
Chance Zibolski	4f6045a94e	Support "if" without "else" clause Defaults the else clause to the identity filter "." if an else clause is missing. The following two jq programs are considered equivalent: ``` if .foo then .foo = "bar" else . end ``` ``` if .foo then .foo = "bar" end ```	2019-02-20 20:03:26 -06:00
Muh Muhten	1ef90acb38	Bind data imports properly when already resolved	2019-02-20 20:02:41 -06:00
Muh Muhten	e843a4fdf9	Make builtin binding fast again by binding only referenced symbols Avoid doing the internal binding of top-level symbols in the parser, leaving that work to be done in a post-processing step. For builtins, this lets us do a reference-aware bind step (block_bind_incremental) after generating builtins/0. Libraries are a bit trickier since they may be bound multiple times, so instead of thinking through the implications I added (block_bind_self) to resolve all internal symbols immediately.	2019-02-20 19:43:13 -06:00
Muh Muhten	b2b0bd37a1	Load ~/.jq as a library instead of with builtins Remove the special code which loads ~/.jq in builtin.c, and instead glue an optional include which points to the same file onto the main program in linker.c. Fixes a minor bug where errors in ~/.jq would be labelled <builtin>.	2019-02-20 19:16:18 -06:00
Muh Muhten	450ade4818	Add import metadata key "optional" A library marked is imported if found, but silently skipped if missing. This is the desired semantic for the auto-include at ~/.jq	2019-02-20 19:16:18 -06:00
Muh Muhten	c814c4e001	Replace TOP-before-imports special case with assert The case isn't actually possible afaict.	2019-02-20 19:16:18 -06:00
Muh Muhten	4f4494c770	Catch .. as the first component of a module path Only the second and subsequent path components were being checked, which I guess is theoretically security-relevant. There's no apparent point to reconstructing the path after splitting it by adding /s back in, either.	2019-02-20 19:16:18 -06:00
Muh Muhten	ff014824bc	Pass on the error message when rel_path is invalid "Module path must be a string" is not a useful error message when the reason the module path isn't a string is because the string it was got replaced with an invalid with an error message for some other reason. Also fixes a few memory leaks on early exits.	2019-02-20 19:16:18 -06:00
Muh Muhten	8edf58a95f	Simplify definition of block_bind_referenced	2019-02-20 19:16:18 -06:00
Jason Hood	88e1f59ec1	Allow globbing on Windows The mingw-w64 runtime expands wildcards, which causes a discrepancy between `main`'s `argv` (multiple file names) and Windows' `wargv` (a single wildcard). Use `wmain` to retrieve the wide character args.	2019-02-17 23:58:23 -06:00
Jason Hood	80b064f97b	Make UTC time on Windows Windows has `_mkgmtime` for the UTC equivalent of `mktime`.	2019-02-16 20:42:25 -06:00
Jason Hood	0fc8200ab4	Add ISO 8601-specific strptime Implement `strptime` for systems lacking it, but only supporting the ISO 8601 format string.	2019-02-16 20:41:41 -06:00
Jason Hood	87594f81d6	Prevent redirecting to NUL crash Windows regards the `NUL` device as tty, which causes `WriteFile` to fail (the bytes written pointer cannot be `NULL` in this case). Tweak the color test to ensure tty is accurate.	2019-02-16 20:40:41 -06:00
Jason Hood	def42a31ca	Possibly enable color output on Windows (#1494 ) Windows 10 supports color output, but it must be explicitly enabled; ANSICON provides support for other versions. Enable color if either of those are available. Resolves #1494.	2019-02-16 20:40:41 -06:00
Muh Muhten	605bfb3577	Reimplement block_drop_unreferenced in linear time	2019-02-16 20:30:20 -06:00
Nico Williams	9fa2e51099	Merge pull request #1818 from muhmuhten/free_in_index	2019-02-09 16:32:20 -06:00
Muh Muhten	3f59bee3e0	Make sure to free in jv_string_indexes	2019-02-09 17:17:59 -05:00
Muh Muhten	88613f5469	Improve linking time by marking subtrees with unbound symbols	2019-02-08 22:41:31 -06:00
Nicolas Williams	2660b04a73	Fix infinite loop in index("") (fix #1815 )	2019-02-08 20:40:55 -06:00
Matt Clegg	18ac789d6d	FIX if-then-else example "(.+)" doesn't match new line characters so the link generated for [if-then-else](https://github.com/stedolan/jq/blob/90bc29c1b544c0436ec44246e180fdbb6d6384df/docs/content/3.manual/v1.5/manual.yml#L1804) doesn't work	2019-02-07 11:08:44 -06:00
Muh Muhten	956d40d9dd	Define error/1 in terms of error/0 The parser generates a call to error/0 on break. Having that exported from C directly removes a language dependency on builtin.jq. In any case, error/0 seems to be the more fundamental operation, seeing as the old C impementation of error/1 did nothing useful with its input.	2019-02-07 11:05:06 -06:00
Muh Muhten	2fd725209e	Export input/0 directly from C, removing _input/0 The distinction between input/0 and _input/0 has been unnecessary since a9c613..7dc34b, which moved end-of-input handling into C and replaced the jq definition of input with `def input: _input;`	2019-02-06 12:06:03 -06:00
Derrick Pallas	21be3857b5	builtin/f_match: prevent overruns of input_string If these values do not match exactly, it is because the UTF-8 is invalid anyway and we counted codepoints differently than oniguruma did. Perhaps it would be better to error out here, but at least one similar loop already uses < vs != and since we're off the rails anyway this might be OK. It is certainly better than overruning the buffer. Resolves https://github.com/stedolan/jq/issues/1804	2019-02-04 14:12:03 -06:00
Derrick Pallas	d29bf5b4fc	builtin/f_match: call onig_error_code_to_str with einfo correctly	2019-02-02 16:17:12 -08:00
Muh Muhten	3c5b1419a2	Unify timegm fallbacks into my_mktime my_timegm was introduced in 1900c7 to add a mktime fallback used in 4a6241, the subsequently removed in c53823. As a result of this code shuffling, the jq mktime function (f_mktime) wound up using a fallback even on systems where timegm is available, with incorrect results in some time zones with DST, e.g. % TZ=America/New_York jq -n '"2018-08-31T00:00:00Z"\|fromdate\|todate' "2018-08-31T01:00:00Z" This undoes 1900c7 and moves the workaround it added into the #else of my_mktime.	2019-01-30 22:05:51 -06:00
Muh Muhten	012af2186b	Fix build on BSD make Replace uses of the nonstandard $^ variable by explicitly specifying the source filenames. This is the only obvious thing preventing the build from running correctly on non-GNU make.	2019-01-30 09:48:04 -06:00
Muh Muhten	12a628abdc	Fix include/import namespace confusion	2019-01-30 09:47:24 -06:00
David Haguenauer	63ba2b1637	Add tests for built-in function `contains`	2019-01-22 11:00:08 -06:00
Nicolas Williams	61cd6dbb3b	`contains` should handle embedded NULs (fix #1732 )	2019-01-18 22:33:46 -06:00
William Langford	4b4fefa254	Merge pull request #1774 from dooglus/patch-1 Fix grammar typo.	2019-01-05 16:04:30 -05:00
Nicolas Williams	0bc77083ba	Out-of-srcdir builds: AM_CPPFLAGS=-I${srcdir}/src	2018-12-26 12:53:48 -06:00
Ricardo Constantino	b10878fea7	builtin: add missing windows.h include Follow-up fix to `b436156f5` because BOOL and DWORD are not built-in to mingw64.	2018-12-26 12:53:48 -06:00
Ricardo Constantino	6893284ee5	Makefile.am: add no-undefined for win32 "Undefined symbols are not allowed in x86_64-w64-mingw32 shared libraries" This was also applied in the official MINGW-packages repo: Alexpux/MINGW-packages@b5566c58a	2018-12-26 02:45:54 +00:00
HE, Tao	b436156f5b	Mingw-w64 on windows doesn't have `setenv`, fix that. Signed-off-by: HE, Tao <sighingnow@gmail.com>	2018-12-19 09:07:14 -06:00
Eugen	fb1c635af6	Make fromstream/1 more efficient	2018-12-18 14:20:35 -06:00
Ryoichi KATO	6d3d2750ec	fix --exit-code issues #1142 and #1139 * Set default error code to -4 in main(), Fixes #1142 * fix --exit-code with more than one object in input, Fixes #1139 - Return code 1 or 4 based on last output, not last input.	2018-12-18 14:17:47 -06:00
mail6543210	712330940b	Ensure that `${builddir}/src` exists	2018-12-18 14:15:04 -06:00
William Langford	9c029f7077	Merge pull request #1779 from gvangool/update-NEWS Adds the release info for 1.6 to NEWS	2018-12-15 15:20:23 -05:00
Gert Van Gool	2760fd365b	Adds the release info for 1.6 to NEWS	2018-12-15 19:51:29 +01:00
Nicolas Williams	8ea4a5584e	Allow variable refs as object keys {$key:value} Users are often surprised by the requirement to parenthesize any non-trivial object key expressions in object constructors. E.g., {"a"+"b":1}. This commit adds one more kind of key expression besides literals and idents: variable references. A common use case for this is jq programs as JSON templates to fill in with variables computed from inputs or passed in on the command-line. E.g., {some_key:$value}. Now users can also use, e.g., {$key:$value}. This and the restrictions on key and value expressions in object constructors are now clarified a bit in the documentation.	2018-12-10 21:54:59 -06:00
Chris Moore	9caa1ba146	Fix grammar typo. 'operator' needs to be plural here.	2018-11-29 05:44:05 -08:00
William Langford	bb87e6e30f	Merge pull request #1768 from haochenx/readme-add-automake-as-build-dependency-in add automake as build dependency to README.md	2018-11-20 10:13:47 -05:00
William Langford	a1f1231a73	Add missing jv_copy when printing with -ar	2018-11-20 09:58:25 -05:00
Haochen Xie	0ab71031a6	add automake as build dependency to README.md	2018-11-12 16:00:53 -05:00
William Langford	2f2d05b087	Minor website fixes for 1.6	2018-11-02 08:15:18 -04:00