Add lib.h to dist file list

Thanks @shym!!

.gitignore

@@ -1,5 +1,13 @@
 *.o
+*.a
+*.lo
+*.la
+*.lai
+*.so
+*.so.*
 *~
+.*.sw[a-p]
+tags
 
 jq
 jq.1
@@ -14,15 +22,30 @@ parser.c
 parser.h
 
 # Autotools junk
+.libs
 .deps
+libtool
 *.log
 stamp-h1
 config.log
 config.status
 autom4te.cache
-config.h
+INSTALL
 Makefile
 jq-*.tar.gz
 configure
 aclocal.m4
 Makefile.in
+version.h
+.remake-version-h
+config.cache
+*.rpm
+m4/libtool.m4
+m4/ltoptions.m4
+m4/ltsugar.m4
+m4/ltversion.m4
+m4/lt~obsolete.m4
+
+cscope.in.out
+cscope.out
+cscope.po.out

.travis.yml

@@ -0,0 +1,17 @@
+language: c
+
+before_script:
+    - sudo apt-get install -qq valgrind
+    - bundle install --gemfile=docs/Gemfile
+    - autoreconf -i
+
+script:
+    - ./setup.sh
+    - ./configure && make
+
+compiler:
+    - gcc
+    - clang
+
+notifications:
+    email: false

AUTHORS

@@ -4,3 +4,4 @@ Stephen Dolan        <mu@netsoc.tcd.ie>
 Contributions by
 Anthony Shortland    <anthony.shortland@me.com>      - rpmbuild target
 Lee Thompson         <stagr.lee@gmail.com>           - autoconf stuff
+Nicolas Williams     <nico@cryptonector.com>         - library-fication, autoconf stuff, misc

Makefile.am

@@ -1,13 +1,12 @@
 ### C source files to be built and distributed.
 
-JQ_INCS = jq_parser.h builtin.h bytecode.h compile.h execute.h		\
-  forkable_stack.h frame_layout.h jv.h jv_alloc.h jv_aux.h jv_dtoa.h	\
-  jv_parse.h jv_unicode.h locfile.h opcode.h opcode_list.h parser.y	\
-  jv_utf8_tables.h lexer.l
+LIBJQ_INCS = jq_parser.h builtin.h bytecode.h compile.h exec_stack.h	\
+   libm.h jv_alloc.h jv_dtoa.h jv_unicode.h locfile.h		 	\
+   opcode_list.h parser.y jv_utf8_tables.h lexer.l
 
-JQ_SRC = opcode.c bytecode.c compile.c execute.c builtin.c jv.c		\
-  jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_alloc.c	\
-  jq_test.c ${JQ_INCS}
+LIBJQ_SRC = locfile.c bytecode.c compile.c execute.c builtin.c jv.c	\
+  jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_file.c	\
+  jv_alloc.c jq_test.c ${LIBJQ_INCS}
 
 
 ### C build options
@@ -15,14 +14,15 @@ JQ_SRC = opcode.c bytecode.c compile.c execute.c builtin.c jv.c		\
 AM_CFLAGS = -Wextra -Wall -Wno-missing-field-initializers	\
   -Wno-unused-parameter -Wno-unused-function
 
+ACLOCAL_AMFLAGS = -I config/m4
 
 ### Generating the lexer and parser
 
 # While there is some autoconf macro support for lex/flex, it doesn't support
 # header file creation so we'll use good old make
-BUILT_SOURCES = lexer.h lexer.c parser.h parser.c
+BUILT_SOURCES = lexer.h lexer.c parser.h parser.c version.h
 lexer.c: lexer.l
-	flex -o lexer.c --header-file=lexer.h $<
+	$(AM_V_LEX) flex -o lexer.c --header-file=lexer.h $<
 lexer.h: lexer.c
 
 # Tell YACC (bison) autoconf macros that you want a header file created.
@@ -30,14 +30,31 @@ lexer.h: lexer.c
 # OSX ships an old bison, so update with homebrew or macports
 AM_YFLAGS = --warnings=all -d
 
+### libjq
+
+lib_LTLIBRARIES = libjq.la
+libjq_la_SOURCES = ${LIBJQ_SRC}
+libjq_la_LIBADD = -lm
+libjq_la_LDFLAGS = -export-symbols-regex '^j[qv]_' -version-info 1:4:0
+
+include_HEADERS = jv.h jq.h
+
 
 ### Building the jq binary
 
-main.c: config.h
+# Remake the version.h header file if, and only if, the git ID has changed
+.PHONY: .FORCE
+.FORCE:
+generate_ver = ver="`{ $(srcdir)/scripts/version || echo '$(VERSION)' ; } | sed 's/.*/\#define JQ_VERSION \"&\"/'`"
+.remake-version-h: .FORCE
+	@ $(generate_ver); test "x`cat version.h 2>/dev/null`" = "x$$ver" || touch .remake-version-h
+version.h: .remake-version-h
+	$(AM_V_GEN) $(generate_ver); echo "$$ver" > $@
 
 bin_PROGRAMS = jq
-jq_SOURCES = ${JQ_SRC} main.c
-
+jq_SOURCES = main.c
+jq_LDFLAGS = -static-libtool-libs
+jq_LDADD = libjq.la -lm
 
 ### Tests (make check)
 
@@ -56,16 +73,16 @@ man_MANS = jq.1
 .PHONY: real_docs
 if ENABLE_DOCS
 jq.1: $(srcdir)/docs/content/3.manual/manual.yml
-	( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
 jq.1.default: $(srcdir)/docs/default_manpage.md
-	( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
 real_docs: jq.1
 	if cmp jq.1 $(srcdir)/jq.1.default > /dev/null; then\
 	  rm -f jq.1; $(MAKE) $(AM_MAKEFLAGS) jq.1;\
 	fi
 else
 jq.1: $(srcdir)/jq.1.default
-	cp $^ $@
+	$(AM_V_GEN) cp $^ $@
 real_docs:
 	@echo "Ruby dependencies not found, cannot build manpage." > /dev/stderr
 	@echo "Follow the instructions in docs/README.md to install them" > /dev/stderr
@@ -93,24 +110,21 @@ DOC_FILES = docs/content docs/public docs/templates docs/site.yml	\
 # setup is only used by distribution developers, not package developers.
 # Still, as a matter of allowing patching, its not a bad idea to distribute 
 # the developer setup script in the tarball.
-EXTRA_DIST = config.h.in $(BUILT_SOURCES) $(man_MANS) $(TESTS)	\
-	$(TEST_LOG_COMPILER) gen_utf8_tables.py jq.spec \
-	$(DOC_FILES)
+EXTRA_DIST = $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER) jq.spec	\
+	 $(DOC_FILES) scripts/version parser.h parser.c lexer.h	\
+	 lexer.c
 
 # README.md is expected in Github projects, good stuff in it, so we'll 
 # distribute it and install it with the package in the doc directory.
 docdir = ${datadir}/doc/${PACKAGE}
 dist_doc_DATA = README.md COPYING AUTHORS README
 
-releasetag:
-	git tag -s "jq-$$(cat VERSION)" -m "jq release $$(cat VERSION)"
-
 RELEASE ?= 1
-rpm: jq
+rpm: dist jq.spec
 	@echo "Packaging jq as an RPM ..."
 	mkdir -p rpm/SOURCES rpm/BUILD rpm/BUILDROOT rpm/RPMS rpm/SPECS
-	cp jq-$$(cat VERSION).tar.gz rpm/SOURCES/
-	rpmbuild -tb --define "_topdir ${PWD}/rpm" --define "_prefix /usr" --define "myver $$(cat VERSION)" --define "myrel ${RELEASE}" rpm/SOURCES/jq-$$(cat VERSION).tar.gz
+	cp jq-$(VERSION).tar.gz rpm/SOURCES/
+	rpmbuild -tb --define "_topdir ${PWD}/rpm" --define "_prefix /usr" --define "myver $(VERSION)" --define "myrel ${RELEASE}" rpm/SOURCES/jq-$(VERSION).tar.gz
 	find rpm/RPMS/ -name "*.rpm" -exec mv {} ./ \;
 	rm -rf rpm
 
@@ -119,5 +133,12 @@ dist-clean-local:
 
 # Not sure why this doesn't get cleaned up automatically, guess
 # automake used to man pages which are hand coded?
-clean-local:
+# 'make clean' doesn't delete the manpage if it can't be rebuilt
+.PHONY: clean-local-docs
+clean-local-docs:
+if ENABLE_DOCS
 	rm -f jq.1
+endif 
+
+clean-local: clean-local-docs
+	rm -f version.h .remake-version-h

README.md

@@ -6,18 +6,30 @@ jq is a command-line JSON processor.
 If you want to learn to use jq, read the documentation at
 [http://stedolan.github.io/jq](http://stedolan.github.io/jq). This
 documentation is generated from the docs/ folder of this repository.
+You can also try it online at [http://jqplay.herokuapp.com](http://jqplay.herokuapp.com).
 
 If you want to hack on jq, feel free, but be warned that its internals
 are not well-documented at the moment. Bring a hard hat and a
-shovel. Also, read the wiki: https://github.com/stedolan/jq/wiki
+shovel. Also, read the wiki: http://github.com/stedolan/jq/wiki
 
-To build jq, run
+If you're building directly from the latest git, you'll need flex and
+bison installed. To build, run:
 
+    autoreconf -i
     ./configure
-    make
-    sudo make install (optionally)
+    make -j8
+    make check
 
-If you've just checked out the latest version from git (rather than
-using a released source tarball) then you'll need to run this first:
+After make finishes, you'll be able to use `./jq`. You can also
+install it using:
+
+    sudo make install
+
+If you're not using the latest git version but instead building a
+released tarball (available on the website), then you won't need to
+run `autoreconf` (and shouldn't), and you won't need flex or bison.
+
+To cross-compile for OS X and Windows, see docs/Rakefile's build task
+and scripts/crosscompile.  You'll need a cross-compilation environment,
+such as Mingw for cross-compiling for Windows.
 
-    autoreconf

builtin.c

@@ -1,29 +1,15 @@
+#include <math.h>
+#include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 #include "builtin.h"
 #include "compile.h"
 #include "jq_parser.h"
+#include "bytecode.h"
 #include "locfile.h"
-#include "jv_aux.h"
 #include "jv_unicode.h"
 
 
-
-typedef jv (*func_1)(jv);
-typedef jv (*func_2)(jv,jv);
-typedef jv (*func_3)(jv,jv,jv);
-typedef jv (*func_4)(jv,jv,jv,jv);
-typedef jv (*func_5)(jv,jv,jv,jv,jv);
-jv cfunction_invoke(struct cfunction* function, jv input[]) {
-  switch (function->nargs) {
-  case 1: return ((func_1)function->fptr)(input[0]);
-  case 2: return ((func_2)function->fptr)(input[0], input[1]);
-  case 3: return ((func_3)function->fptr)(input[0], input[1], input[2]);
-  case 4: return ((func_4)function->fptr)(input[0], input[1], input[2], input[3]);
-  case 5: return ((func_5)function->fptr)(input[0], input[1], input[2], input[3], input[4]);
-  default: return jv_invalid_with_msg(jv_string("Function takes too many arguments"));
-  }
-}
-
 static jv type_error(jv bad, const char* msg) {
   jv err = jv_invalid_with_msg(jv_string_fmt("%s %s",
                                              jv_kind_name(jv_get_kind(bad)),
@@ -64,6 +50,18 @@ static jv f_plus(jv input, jv a, jv b) {
   }
 }
 
+#define LIBM_DD(name) \
+static jv f_ ## name(jv input) { \
+  if (jv_get_kind(input) != JV_KIND_NUMBER) { \
+    return type_error(input, "number required"); \
+  } \
+  jv ret = jv_number(name(jv_number_value(input))); \
+  jv_free(input); \
+  return ret; \
+}
+#include "libm.h"
+#undef LIBM_DD
+
 static jv f_negate(jv input) {
   if (jv_get_kind(input) != JV_KIND_NUMBER) {
     return type_error(input, "cannot be negated");
@@ -73,6 +71,64 @@ static jv f_negate(jv input) {
   return ret;
 }
 
+static jv f_startswith(jv a, jv b) {
+  int alen = jv_string_length_bytes(jv_copy(a));
+  int blen = jv_string_length_bytes(jv_copy(b));
+  jv ret;
+
+  if (blen <= alen && memcmp(jv_string_value(a), jv_string_value(b), blen) == 0)
+    ret = jv_true();
+  else
+    ret = jv_false();
+  jv_free(a);
+  jv_free(b);
+  return ret;
+}
+
+static jv f_endswith(jv a, jv b) {
+  const char *astr = jv_string_value(a);
+  const char *bstr = jv_string_value(b);
+  size_t alen = jv_string_length_bytes(jv_copy(a));
+  size_t blen = jv_string_length_bytes(jv_copy(b));
+  jv ret;;
+
+  if (alen < blen ||
+     memcmp(astr + (alen - blen), bstr, blen) != 0)
+    ret = jv_false();
+  else
+    ret = jv_true();
+  jv_free(a);
+  jv_free(b);
+  return ret;
+}
+
+static jv f_ltrimstr(jv input, jv left) {
+  if (jv_get_kind(f_startswith(jv_copy(input), jv_copy(left))) != JV_KIND_TRUE) {
+    jv_free(left);
+    return input;
+  }
+  /*
+   * FIXME It'd be better to share the suffix with the original input --
+   * that we could do, we just can't share prefixes.
+   */
+  int prefixlen = jv_string_length_bytes(left);
+  jv res = jv_string_sized(jv_string_value(input) + prefixlen,
+                           jv_string_length_bytes(jv_copy(input)) - prefixlen);
+  jv_free(input);
+  return res;
+}
+
+static jv f_rtrimstr(jv input, jv right) {
+  if (jv_get_kind(f_endswith(jv_copy(input), jv_copy(right))) == JV_KIND_TRUE) {
+    jv res = jv_string_sized(jv_string_value(input),
+                             jv_string_length_bytes(jv_copy(input)) - jv_string_length_bytes(right));
+    jv_free(input);
+    return res;
+  }
+  jv_free(right);
+  return input;
+}
+
 static jv f_minus(jv input, jv a, jv b) {
   jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
@@ -100,9 +156,34 @@ static jv f_minus(jv input, jv a, jv b) {
 }
 
 static jv f_multiply(jv input, jv a, jv b) {
+  jv_kind ak = jv_get_kind(a);
+  jv_kind bk = jv_get_kind(b);
   jv_free(input);
-  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
+  if (ak == JV_KIND_NUMBER && bk == JV_KIND_NUMBER) {
     return jv_number(jv_number_value(a) * jv_number_value(b));
+  } else if ((ak == JV_KIND_STRING && bk == JV_KIND_NUMBER) ||
+             (ak == JV_KIND_NUMBER && bk == JV_KIND_STRING)) {
+    jv str = a;
+    jv num = b;
+    if (ak == JV_KIND_NUMBER) {
+      str = b;
+      num = a;
+    }
+    int n;
+    size_t alen = jv_string_length_bytes(jv_copy(str));
+    jv res = str;
+
+    for (n = jv_number_value(num) - 1; n > 0; n--)
+      res = jv_string_append_buf(res, jv_string_value(str), alen);
+
+    jv_free(num);
+    if (n < 0) {
+      jv_free(str);
+      return jv_null();
+    }
+    return res;
+  } else if (ak == JV_KIND_OBJECT && bk == JV_KIND_OBJECT) {
+    return jv_object_merge_recursive(a, b);
   } else {
     return type_error2(a, b, "cannot be multiplied");
   }  
@@ -112,6 +193,17 @@ static jv f_divide(jv input, jv a, jv b) {
   jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
     return jv_number(jv_number_value(a) / jv_number_value(b));
+  } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
+    return jv_string_split(a, b);
+  } else {
+    return type_error2(a, b, "cannot be divided");
+  }  
+}
+
+static jv f_mod(jv input, jv a, jv b) {
+  jv_free(input);
+  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
+    return jv_number((intmax_t)jv_number_value(a) % (intmax_t)jv_number_value(b));
   } else {
     return type_error2(a, b, "cannot be divided");
   }  
@@ -167,6 +259,19 @@ static jv f_contains(jv a, jv b) {
   }
 }
 
+static jv f_dump(jv input) {
+  return jv_dump_string(input, 0);
+}
+
+static jv f_json_parse(jv input) {
+  if (jv_get_kind(input) != JV_KIND_STRING)
+    return type_error(input, "only strings can be parsed");
+  jv res = jv_parse_sized(jv_string_value(input),
+                          jv_string_length_bytes(jv_copy(input)));
+  jv_free(input);
+  return res;
+}
+
 static jv f_tonumber(jv input) {
   if (jv_get_kind(input) == JV_KIND_NUMBER) {
     return input;
@@ -188,6 +293,8 @@ static jv f_length(jv input) {
     return jv_number(jv_object_length(input));
   } else if (jv_get_kind(input) == JV_KIND_STRING) {
     return jv_number(jv_string_length_codepoints(input));
+  } else if (jv_get_kind(input) == JV_KIND_NUMBER) {
+    return jv_number(fabs(jv_number_value(input)));
   } else if (jv_get_kind(input) == JV_KIND_NULL) {
     jv_free(input);
     return jv_number(0);
@@ -468,15 +575,29 @@ static jv f_error(jv input, jv msg) {
   return jv_invalid_with_msg(msg);
 }
 
+#define LIBM_DD(name) \
+  {(cfunction_ptr)f_ ## name, "_" #name, 1},
+   
 static const struct cfunction function_list[] = {
+#include "libm.h"
   {(cfunction_ptr)f_plus, "_plus", 3},
   {(cfunction_ptr)f_negate, "_negate", 1},
   {(cfunction_ptr)f_minus, "_minus", 3},
   {(cfunction_ptr)f_multiply, "_multiply", 3},
   {(cfunction_ptr)f_divide, "_divide", 3},
+  {(cfunction_ptr)f_mod, "_mod", 3},
+  {(cfunction_ptr)f_dump, "tojson", 1},
+  {(cfunction_ptr)f_json_parse, "fromjson", 1},
   {(cfunction_ptr)f_tonumber, "tonumber", 1},
   {(cfunction_ptr)f_tostring, "tostring", 1},
   {(cfunction_ptr)f_keys, "keys", 1},
+  {(cfunction_ptr)f_startswith, "startswith", 2},
+  {(cfunction_ptr)f_endswith, "endswith", 2},
+  {(cfunction_ptr)f_ltrimstr, "ltrimstr", 2},
+  {(cfunction_ptr)f_rtrimstr, "rtrimstr", 2},
+  {(cfunction_ptr)jv_string_split, "split", 2},
+  {(cfunction_ptr)jv_string_explode, "explode", 1},
+  {(cfunction_ptr)jv_string_implode, "implode", 1},
   {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking
   {(cfunction_ptr)jv_getpath, "getpath", 2},
   {(cfunction_ptr)jv_delpaths, "delpaths", 2},
@@ -500,6 +621,7 @@ static const struct cfunction function_list[] = {
   {(cfunction_ptr)f_error, "error", 2},
   {(cfunction_ptr)f_format, "format", 2},
 };
+#undef LIBM_DD
 
 struct bytecoded_builtin { const char* name; block code; };
 static block bind_bytecoded_builtins(block b) {
@@ -526,56 +648,105 @@ static block bind_bytecoded_builtins(block b) {
     };
     for (unsigned i=0; i<sizeof(builtin_def_1arg)/sizeof(builtin_def_1arg[0]); i++) {
       builtins = BLOCK(builtins, gen_function(builtin_def_1arg[i].name,
-                                              gen_op_block_unbound(CLOSURE_PARAM, "arg"),
+                                              gen_param("arg"),
                                               builtin_def_1arg[i].code));
     }
   }
   {
-    block rangevar = block_bind(gen_op_var_unbound(STOREV, "rangevar"),
-                                gen_noop(), OP_HAS_VARIABLE);
+    block rangevar = gen_op_var_fresh(STOREV, "rangevar");
     block init = BLOCK(gen_op_simple(DUP), gen_call("start", gen_noop()), rangevar);
     block range = BLOCK(init, 
                         gen_call("end", gen_noop()),
-                        gen_op_var_bound(RANGE, rangevar));
+                        gen_op_bound(RANGE, rangevar));
     builtins = BLOCK(builtins, gen_function("range",
-                                            BLOCK(gen_op_block_unbound(CLOSURE_PARAM, "start"),
-                                                  gen_op_block_unbound(CLOSURE_PARAM, "end")),
+                                            BLOCK(gen_param("start"), gen_param("end")),
                                             range));
   }
   
   return block_bind_referenced(builtins, b, OP_IS_CALL_PSEUDO);
 }
 
+#define LIBM_DD(name) "def " #name ": _" #name ";",
+
 static const char* const jq_builtins[] = {
   "def map(f): [.[] | f];",
   "def select(f): if f then . else empty end;",
   "def sort_by(f): _sort_by_impl(map([f]));",
   "def group_by(f): _group_by_impl(map([f]));",
   "def unique: group_by(.) | map(.[0]);",
+  "def unique_by(f): group_by(f) | map(.[0]);",
   "def max_by(f): _max_by_impl(map([f]));",
   "def min_by(f): _min_by_impl(map([f]));",
+#include "libm.h"
   "def add: reduce .[] as $x (null; . + $x);",
   "def del(f): delpaths([path(f)]);",
   "def _assign(paths; value): value as $v | reduce path(paths) as $p (.; setpath($p; $v));",
   "def _modify(paths; update): reduce path(paths) as $p (.; setpath($p; getpath($p) | update));",
   "def recurse(f): ., (f | select(. != null) | recurse(f));",
+  "def recurse_down: recurse(.[]?);",
   "def to_entries: [keys[] as $k | {key: $k, value: .[$k]}];",
   "def from_entries: map({(.key): .value}) | add;",
   "def with_entries(f): to_entries | map(f) | from_entries;",
   "def reverse: [.[length - 1 - range(0;length)]];",
+  "def indices(i): if type == \"array\" and (i|type) == \"array\" then .[i] elif type == \"array\" then .[[i]] else .[i] end;",
+  "def index(i):   if type == \"array\" and (i|type) == \"array\" then .[i] elif type == \"array\" then .[[i]] else .[i] end | .[0];",
+  "def rindex(i):  if type == \"array\" and (i|type) == \"array\" then .[i] elif type == \"array\" then .[[i]] else .[i] end | .[-1:][0];",
+  "def paths: path(recurse(if (type|. == \"array\" or . == \"object\") then .[] else empty end))|select(length > 0);",
+  "def leaf_paths: . as $dot|paths|select(. as $p|$dot|getpath($p)|type|. != \"array\" and . != \"object\");",
+  "def any: reduce .[] as $i (false; . or $i);",
+  "def all: reduce .[] as $i (true; . and $i);",
+  "def arrays: select(type == \"array\");",
+  "def objects: select(type == \"object\");",
+  "def iterables: arrays, objects;",
+  "def booleans: select(type == \"boolean\");",
+  "def numbers: select(type == \"number\");",
+  "def strings: select(type == \"string\");",
+  "def nulls: select(type == \"null\");",
+  "def values: arrays, objects, booleans, numbers, strings;",
+  "def scalars: select(. == null or . == true or . == false or type == \"number\" or type == \"string\");",
+  "def join(x): reduce .[] as $i (\"\"; . + (if . == \"\" then $i else x + $i end));",
 };
+#undef LIBM_DD
 
 
-block builtins_bind(block b) {
-  for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) {
-    struct locfile src;
-    locfile_init(&src, jq_builtins[i], strlen(jq_builtins[i]));
-    block funcs;
-    int nerrors = jq_parse_library(&src, &funcs);
-    assert(!nerrors);
-    b = block_bind_referenced(funcs, b, OP_IS_CALL_PSEUDO);
-    locfile_free(&src);
+static int builtins_bind_one(jq_state *jq, block* bb, const char* code) {
+  struct locfile src;
+  locfile_init(&src, jq, code, strlen(code));
+  block funcs;
+  int nerrors = jq_parse_library(&src, &funcs);
+  if (nerrors == 0) {
+    *bb = block_bind_referenced(funcs, *bb, OP_IS_CALL_PSEUDO);
   }
-  b = bind_bytecoded_builtins(b);
-  return gen_cbinding(function_list, sizeof(function_list)/sizeof(function_list[0]), b);
+  locfile_free(&src);
+  return nerrors;
+}
+
+static int slurp_lib(jq_state *jq, block* bb) {
+  int nerrors = 0;
+  char* home = getenv("HOME");
+  if (home) {    // silently ignore no $HOME
+    jv filename = jv_string_append_str(jv_string(home), "/.jq");
+    jv data = jv_load_file(jv_string_value(filename), 1);
+    if (jv_is_valid(data)) {
+      nerrors = builtins_bind_one(jq, bb, jv_string_value(data) );
+    }
+    jv_free(filename);
+    jv_free(data);
+  }
+  return nerrors;
+}
+
+int builtins_bind(jq_state *jq, block* bb) {
+  int nerrors = slurp_lib(jq, bb);
+  if (nerrors) {
+    block_free(*bb);
+    return nerrors;
+  }
+  for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) {
+    nerrors = builtins_bind_one(jq, bb, jq_builtins[i]);
+    assert(!nerrors);
+  }
+  *bb = bind_bytecoded_builtins(*bb);
+  *bb = gen_cbinding(function_list, sizeof(function_list)/sizeof(function_list[0]), *bb);
+  return nerrors;
 }

builtin.h

@@ -1,21 +1,10 @@
 #ifndef BUILTIN_H
 #define BUILTIN_H
 
+#include "jq.h"
+#include "bytecode.h"
 #include "compile.h"
 
-block builtins_bind(block);
-
-
-typedef void (*cfunction_ptr)(void);
-
-struct cfunction {
-  cfunction_ptr fptr;
-  const char* name;
-  int nargs;
-};
-
-
-jv cfunction_invoke(struct cfunction* function, jv input[]);
-
+int builtins_bind(jq_state *, block*);
 
 #endif

bytecode.c

@@ -3,9 +3,39 @@
 #include <stdlib.h>
 
 #include "bytecode.h"
-#include "opcode.h"
 #include "jv_alloc.h"
 
+// flags, length
+#define NONE 0, 1
+#define CONSTANT OP_HAS_CONSTANT, 2
+#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
+#define BRANCH OP_HAS_BRANCH, 2
+#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
+#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
+#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
+#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
+
+#define OP(name, imm, in, out) \
+  {name, #name, imm, in, out},
+
+static const struct opcode_description opcode_descriptions[] = {
+#include "opcode_list.h"
+};
+
+static const struct opcode_description invalid_opcode_description = {
+  -1, "#INVALID", 0, 0, 0, 0
+};
+
+
+const struct opcode_description* opcode_describe(opcode op) {
+  if ((int)op >= 0 && (int)op < NUM_OPCODES) {
+    return &opcode_descriptions[op];
+  } else {
+    return &invalid_opcode_description;
+  }
+}
+
+
 static int bytecode_operation_length(uint16_t* codeptr) {
   int length = opcode_describe(*codeptr)->length;
   if (*codeptr == CALL_JQ) {
@@ -14,6 +44,22 @@ static int bytecode_operation_length(uint16_t* codeptr) {
   return length;
 }
 
+static void dump_code(int indent, struct bytecode* bc) {
+  int pc = 0;
+  while (pc < bc->codelen) {
+    printf("%*s", indent, "");
+    dump_operation(bc, bc->code + pc);
+    printf("\n");
+    pc += bytecode_operation_length(bc->code + pc);
+  }
+}
+
+static void symbol_table_free(struct symbol_table* syms) {
+  jv_mem_free(syms->cfunctions);
+  jv_free(syms->cfunc_names);
+  jv_mem_free(syms);
+}
+
 void dump_disassembly(int indent, struct bytecode* bc) {
   if (bc->nclosures > 0) {
     printf("%*s[params: ", indent, "");
@@ -37,16 +83,6 @@ void dump_disassembly(int indent, struct bytecode* bc) {
   }
 }
 
-void dump_code(int indent, struct bytecode* bc) {
-  int pc = 0;
-  while (pc < bc->codelen) {
-    printf("%*s", indent, "");
-    dump_operation(bc, bc->code + pc);
-    printf("\n");
-    pc += bytecode_operation_length(bc->code + pc);
-  }
-}
-
 static struct bytecode* getlevel(struct bytecode* bc, int level) {
   while (level > 0) {
     bc = bc->parent;
@@ -109,13 +145,9 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
   }  
 }
 
-void symbol_table_free(struct symbol_table* syms) {
-  jv_mem_free(syms->cfunctions);
-  jv_free(syms->cfunc_names);
-  jv_mem_free(syms);
-}
-
 void bytecode_free(struct bytecode* bc) {
+  if (!bc)
+    return;
   jv_mem_free(bc->code);
   jv_free(bc->constants);
   for (int i=0; i<bc->nsubfunctions; i++)

bytecode.h

@@ -3,10 +3,52 @@
 #include <stdint.h>
 
 #include "jv.h"
-#include "opcode.h"
-#include "builtin.h"
+
+typedef enum {
+#define OP(name, imm, in, out) name,
+#include "opcode_list.h"
+#undef OP
+} opcode;
+
+enum {
+  NUM_OPCODES = 
+#define OP(name, imm, in, out) +1
+#include "opcode_list.h"
+#undef OP
+};
+
+enum {
+  OP_HAS_CONSTANT = 2,
+  OP_HAS_VARIABLE = 4,
+  OP_HAS_BRANCH = 8,
+  OP_HAS_CFUNC = 32,
+  OP_HAS_UFUNC = 64,
+  OP_IS_CALL_PSEUDO = 128,
+  OP_HAS_BINDING = 1024,
+};
+struct opcode_description {
+  opcode op;
+  const char* name;
+
+  int flags;
+
+  // length in 16-bit units
+  int length;
+
+  int stack_in, stack_out;
+};
+
+const struct opcode_description* opcode_describe(opcode op);
+
 
 #define MAX_CFUNCTION_ARGS 10
+typedef void (*cfunction_ptr)();
+struct cfunction {
+  cfunction_ptr fptr;
+  const char* name;
+  int nargs;
+};
+
 struct symbol_table {
   struct cfunction* cfunctions;
   int ncfunctions;
@@ -39,10 +81,8 @@ struct bytecode {
 };
 
 void dump_disassembly(int, struct bytecode* code);
-void dump_code(int, struct bytecode* code);
 void dump_operation(struct bytecode* bc, uint16_t* op);
 
-void symbol_table_free(struct symbol_table* syms);
 void bytecode_free(struct bytecode* bc);
 
 #endif

compile.c

@@ -1,8 +1,9 @@
+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE // for strdup
+#endif
 #include <assert.h>
 #include <string.h>
 #include <stdlib.h>
-#include "opcode.h"
 #include "compile.h"
 #include "bytecode.h"
 #include "locfile.h"
@@ -153,33 +154,26 @@ void inst_set_target(block b, block target) {
   b.first->imm.target = target.last;
 }
 
-block gen_op_var_unbound(opcode op, const char* name) {
+block gen_op_unbound(opcode op, const char* name) {
+  assert(opcode_describe(op)->flags & OP_HAS_BINDING);
+  inst* i = inst_new(op);
+  i->symbol = strdup(name);
+  return inst_block(i);
+}
+
+block gen_op_var_fresh(opcode op, const char* name) {
   assert(opcode_describe(op)->flags & OP_HAS_VARIABLE);
-  inst* i = inst_new(op);
-  i->symbol = strdup(name);
-  return inst_block(i);
+  return block_bind(gen_op_unbound(op, name),
+                    gen_noop(), OP_HAS_VARIABLE);
 }
 
-block gen_op_var_bound(opcode op, block binder) {
+block gen_op_bound(opcode op, block binder) {
   assert(block_is_single(binder));
-  block b = gen_op_var_unbound(op, binder.first->symbol);
+  block b = gen_op_unbound(op, binder.first->symbol);
   b.first->bound_by = binder.first;
   return b;
 }
 
-block gen_op_block_unbound(opcode op, const char* name) {
-  assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO);
-  inst* i = inst_new(op);
-  i->symbol = strdup(name);
-  return inst_block(i);
-}
-
-block gen_op_block_bound(opcode op, block binder) {
-  assert(block_is_single(binder));
-  block b = gen_op_block_unbound(op, binder.first->symbol);
-  b.first->bound_by = binder.first;
-  return b;
-}
 
 static void inst_join(inst* a, inst* b) {
   assert(a && b);
@@ -280,12 +274,16 @@ block gen_function(const char* name, block formals, block body) {
   return b;
 }
 
+block gen_param(const char* name) {
+  return gen_op_unbound(CLOSURE_PARAM, name);
+}
+
 block gen_lambda(block body) {
   return gen_function("@lambda", gen_noop(), body);
 }
 
 block gen_call(const char* name, block args) {
-  block b = gen_op_block_unbound(CALL_JQ, name);
+  block b = gen_op_unbound(CALL_JQ, name);
   b.first->arglist = args;
   return b;
 }
@@ -306,30 +304,27 @@ block gen_both(block a, block b) {
 
 
 block gen_collect(block expr) {
-  block array_var = block_bind(gen_op_var_unbound(STOREV, "collect"),
-                               gen_noop(), OP_HAS_VARIABLE);
+  block array_var = gen_op_var_fresh(STOREV, "collect");
   block c = BLOCK(gen_op_simple(DUP), gen_const(jv_array()), array_var);
 
-  block tail = BLOCK(gen_op_var_bound(APPEND, array_var),
+  block tail = BLOCK(gen_op_bound(APPEND, array_var),
                      gen_op_simple(BACKTRACK));
 
   return BLOCK(c,
                gen_op_target(FORK, tail),
                expr, 
                tail,
-               gen_op_var_bound(LOADVN, array_var));
+               gen_op_bound(LOADVN, array_var));
 }
 
 block gen_reduce(const char* varname, block source, block init, block body) {
-  block res_var = block_bind(gen_op_var_unbound(STOREV, "reduce"),
-                             gen_noop(), OP_HAS_VARIABLE);
-
+  block res_var = gen_op_var_fresh(STOREV, "reduce");
   block loop = BLOCK(gen_op_simple(DUP),
                      source,
-                     block_bind(gen_op_var_unbound(STOREV, varname),
-                                BLOCK(gen_op_var_bound(LOADVN, res_var),
+                     block_bind(gen_op_unbound(STOREV, varname),
+                                BLOCK(gen_op_bound(LOADVN, res_var),
                                       body,
-                                      gen_op_var_bound(STOREV, res_var)),
+                                      gen_op_bound(STOREV, res_var)),
                                 OP_HAS_VARIABLE),
                      gen_op_simple(BACKTRACK));
   return BLOCK(gen_op_simple(DUP),
@@ -337,19 +332,18 @@ block gen_reduce(const char* varname, block source, block init, block body) {
                res_var,
                gen_op_target(FORK, loop),
                loop,
-               gen_op_var_bound(LOADVN, res_var));
+               gen_op_bound(LOADVN, res_var));
 }
 
 block gen_definedor(block a, block b) {
   // var found := false
-  block found_var = block_bind(gen_op_var_unbound(STOREV, "found"),
-                               gen_noop(), OP_HAS_VARIABLE);
+  block found_var = gen_op_var_fresh(STOREV, "found");
   block init = BLOCK(gen_op_simple(DUP), gen_const(jv_false()), found_var);
 
   // if found, backtrack. Otherwise execute b
   block backtrack = gen_op_simple(BACKTRACK);
   block tail = BLOCK(gen_op_simple(DUP),
-                     gen_op_var_bound(LOADV, found_var),
+                     gen_op_bound(LOADV, found_var),
                      gen_op_target(JUMP_F, backtrack),
                      backtrack,
                      gen_op_simple(POP),
@@ -361,7 +355,7 @@ block gen_definedor(block a, block b) {
   // found := true, produce result
   block if_found = BLOCK(gen_op_simple(DUP),
                          gen_const(jv_true()),
-                         gen_op_var_bound(STOREV, found_var),
+                         gen_op_bound(STOREV, found_var),
                          gen_op_target(JUMP, tail));
 
   return BLOCK(init,
@@ -400,7 +394,7 @@ block gen_or(block a, block b) {
 
 block gen_var_binding(block var, const char* name, block body) {
   return BLOCK(gen_op_simple(DUP), var,
-               block_bind(gen_op_var_unbound(STOREV, name),
+               block_bind(gen_op_unbound(STOREV, name),
                           body, OP_HAS_VARIABLE));
 }
 
@@ -475,7 +469,7 @@ static int expand_call_arglist(struct locfile* locations, block* b) {
             break;
           case CLOSURE_CREATE:
             block_append(&prelude, b);
-            block_append(&callargs, gen_op_block_bound(CLOSURE_REF, b));
+            block_append(&callargs, gen_op_bound(CLOSURE_REF, b));
             break;
           }
           actual_args++;

compile.h

@@ -2,13 +2,9 @@
 #define COMPILE_H
 #include <stdint.h>
 #include "jv.h"
-#include "opcode.h"
+#include "bytecode.h"
 #include "locfile.h"
 
-struct bytecode;
-struct symbol_table;
-struct cfunction;
-
 struct inst;
 typedef struct inst inst;
 
@@ -24,11 +20,12 @@ block gen_noop();
 block gen_op_simple(opcode op);
 block gen_const(jv constant);
 block gen_op_target(opcode op, block target);
-block gen_op_var_unbound(opcode op, const char* name);
-block gen_op_var_bound(opcode op, block binder);
-block gen_op_block_unbound(opcode op, const char* name);
+block gen_op_unbound(opcode op, const char* name);
+block gen_op_bound(opcode op, block binder);
+block gen_op_var_fresh(opcode op, const char* name);
 
 block gen_function(const char* name, block formals, block body);
+block gen_param(const char* name);
 block gen_lambda(block body);
 block gen_call(const char* name, block body);
 block gen_subexp(block a);

config.h.in

@@ -1,32 +0,0 @@
-/* config.h.in.  Generated from configure.ac by autoheader.  */
-
-/* Define to 1 if your C compiler doesn't accept -c and -o together. */
-#undef NO_MINUS_C_MINUS_O
-
-/* Name of package */
-#undef PACKAGE
-
-/* Define to the address where bug reports for this package should be sent. */
-#undef PACKAGE_BUGREPORT
-
-/* Define to the full name of this package. */
-#undef PACKAGE_NAME
-
-/* Define to the full name and version of this package. */
-#undef PACKAGE_STRING
-
-/* Define to the one symbol short name of this package. */
-#undef PACKAGE_TARNAME
-
-/* Define to the home page for this package. */
-#undef PACKAGE_URL
-
-/* Define to the version of this package. */
-#undef PACKAGE_VERSION
-
-/* Version number of package */
-#undef VERSION
-
-/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
-   `char[]'. */
-#undef YYTEXT_POINTER

config/.gitignore

@@ -0,0 +1,10 @@
+config.guess
+config.sub
+compile
+depcomp
+install-sh
+ltmain.sh
+missing
+ylwrap
+test-driver
+ar-lib

config/compile

@@ -1,343 +0,0 @@
-#! /bin/sh
-# Wrapper for compilers which do not understand '-c -o'.
-
-scriptversion=2012-03-05.13; # UTC
-
-# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2009, 2010, 2012 Free
-# Software Foundation, Inc.
-# Written by Tom Tromey <tromey@cygnus.com>.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# As a special exception to the GNU General Public License, if you
-# distribute this file as part of a program that contains a
-# configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-# This file is maintained in Automake, please report
-# bugs to <bug-automake@gnu.org> or send patches to
-# <automake-patches@gnu.org>.
-
-nl='
-'
-
-# We need space, tab and new line, in precisely that order.  Quoting is
-# there to prevent tools from complaining about whitespace usage.
-IFS=" ""	$nl"
-
-file_conv=
-
-# func_file_conv build_file lazy
-# Convert a $build file to $host form and store it in $file
-# Currently only supports Windows hosts. If the determined conversion
-# type is listed in (the comma separated) LAZY, no conversion will
-# take place.
-func_file_conv ()
-{
-  file=$1
-  case $file in
-    / | /[!/]*) # absolute file, and not a UNC file
-      if test -z "$file_conv"; then
-	# lazily determine how to convert abs files
-	case `uname -s` in
-	  MINGW*)
-	    file_conv=mingw
-	    ;;
-	  CYGWIN*)
-	    file_conv=cygwin
-	    ;;
-	  *)
-	    file_conv=wine
-	    ;;
-	esac
-      fi
-      case $file_conv/,$2, in
-	*,$file_conv,*)
-	  ;;
-	mingw/*)
-	  file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'`
-	  ;;
-	cygwin/*)
-	  file=`cygpath -m "$file" || echo "$file"`
-	  ;;
-	wine/*)
-	  file=`winepath -w "$file" || echo "$file"`
-	  ;;
-      esac
-      ;;
-  esac
-}
-
-# func_cl_dashL linkdir
-# Make cl look for libraries in LINKDIR
-func_cl_dashL ()
-{
-  func_file_conv "$1"
-  if test -z "$lib_path"; then
-    lib_path=$file
-  else
-    lib_path="$lib_path;$file"
-  fi
-  linker_opts="$linker_opts -LIBPATH:$file"
-}
-
-# func_cl_dashl library
-# Do a library search-path lookup for cl
-func_cl_dashl ()
-{
-  lib=$1
-  found=no
-  save_IFS=$IFS
-  IFS=';'
-  for dir in $lib_path $LIB
-  do
-    IFS=$save_IFS
-    if $shared && test -f "$dir/$lib.dll.lib"; then
-      found=yes
-      lib=$dir/$lib.dll.lib
-      break
-    fi
-    if test -f "$dir/$lib.lib"; then
-      found=yes
-      lib=$dir/$lib.lib
-      break
-    fi
-  done
-  IFS=$save_IFS
-
-  if test "$found" != yes; then
-    lib=$lib.lib
-  fi
-}
-
-# func_cl_wrapper cl arg...
-# Adjust compile command to suit cl
-func_cl_wrapper ()
-{
-  # Assume a capable shell
-  lib_path=
-  shared=:
-  linker_opts=
-  for arg
-  do
-    if test -n "$eat"; then
-      eat=
-    else
-      case $1 in
-	-o)
-	  # configure might choose to run compile as 'compile cc -o foo foo.c'.
-	  eat=1
-	  case $2 in
-	    *.o | *.[oO][bB][jJ])
-	      func_file_conv "$2"
-	      set x "$@" -Fo"$file"
-	      shift
-	      ;;
-	    *)
-	      func_file_conv "$2"
-	      set x "$@" -Fe"$file"
-	      shift
-	      ;;
-	  esac
-	  ;;
-	-I)
-	  eat=1
-	  func_file_conv "$2" mingw
-	  set x "$@" -I"$file"
-	  shift
-	  ;;
-	-I*)
-	  func_file_conv "${1#-I}" mingw
-	  set x "$@" -I"$file"
-	  shift
-	  ;;
-	-l)
-	  eat=1
-	  func_cl_dashl "$2"
-	  set x "$@" "$lib"
-	  shift
-	  ;;
-	-l*)
-	  func_cl_dashl "${1#-l}"
-	  set x "$@" "$lib"
-	  shift
-	  ;;
-	-L)
-	  eat=1
-	  func_cl_dashL "$2"
-	  ;;
-	-L*)
-	  func_cl_dashL "${1#-L}"
-	  ;;
-	-static)
-	  shared=false
-	  ;;
-	-Wl,*)
-	  arg=${1#-Wl,}
-	  save_ifs="$IFS"; IFS=','
-	  for flag in $arg; do
-	    IFS="$save_ifs"
-	    linker_opts="$linker_opts $flag"
-	  done
-	  IFS="$save_ifs"
-	  ;;
-	-Xlinker)
-	  eat=1
-	  linker_opts="$linker_opts $2"
-	  ;;
-	-*)
-	  set x "$@" "$1"
-	  shift
-	  ;;
-	*.cc | *.CC | *.cxx | *.CXX | *.[cC]++)
-	  func_file_conv "$1"
-	  set x "$@" -Tp"$file"
-	  shift
-	  ;;
-	*.c | *.cpp | *.CPP | *.lib | *.LIB | *.Lib | *.OBJ | *.obj | *.[oO])
-	  func_file_conv "$1" mingw
-	  set x "$@" "$file"
-	  shift
-	  ;;
-	*)
-	  set x "$@" "$1"
-	  shift
-	  ;;
-      esac
-    fi
-    shift
-  done
-  if test -n "$linker_opts"; then
-    linker_opts="-link$linker_opts"
-  fi
-  exec "$@" $linker_opts
-  exit 1
-}
-
-eat=
-
-case $1 in
-  '')
-     echo "$0: No command.  Try '$0 --help' for more information." 1>&2
-     exit 1;
-     ;;
-  -h | --h*)
-    cat <<\EOF
-Usage: compile [--help] [--version] PROGRAM [ARGS]
-
-Wrapper for compilers which do not understand '-c -o'.
-Remove '-o dest.o' from ARGS, run PROGRAM with the remaining
-arguments, and rename the output as expected.
-
-If you are trying to build a whole package this is not the
-right script to run: please start by reading the file 'INSTALL'.
-
-Report bugs to <bug-automake@gnu.org>.
-EOF
-    exit $?
-    ;;
-  -v | --v*)
-    echo "compile $scriptversion"
-    exit $?
-    ;;
-  cl | *[/\\]cl | cl.exe | *[/\\]cl.exe )
-    func_cl_wrapper "$@"      # Doesn't return...
-    ;;
-esac
-
-ofile=
-cfile=
-
-for arg
-do
-  if test -n "$eat"; then
-    eat=
-  else
-    case $1 in
-      -o)
-	# configure might choose to run compile as 'compile cc -o foo foo.c'.
-	# So we strip '-o arg' only if arg is an object.
-	eat=1
-	case $2 in
-	  *.o | *.obj)
-	    ofile=$2
-	    ;;
-	  *)
-	    set x "$@" -o "$2"
-	    shift
-	    ;;
-	esac
-	;;
-      *.c)
-	cfile=$1
-	set x "$@" "$1"
-	shift
-	;;
-      *)
-	set x "$@" "$1"
-	shift
-	;;
-    esac
-  fi
-  shift
-done
-
-if test -z "$ofile" || test -z "$cfile"; then
-  # If no '-o' option was seen then we might have been invoked from a
-  # pattern rule where we don't need one.  That is ok -- this is a
-  # normal compilation that the losing compiler can handle.  If no
-  # '.c' file was seen then we are probably linking.  That is also
-  # ok.
-  exec "$@"
-fi
-
-# Name of file we expect compiler to create.
-cofile=`echo "$cfile" | sed 's|^.*[\\/]||; s|^[a-zA-Z]:||; s/\.c$/.o/'`
-
-# Create the lock directory.
-# Note: use '[/\\:.-]' here to ensure that we don't use the same name
-# that we are using for the .o file.  Also, base the name on the expected
-# object file name, since that is what matters with a parallel build.
-lockdir=`echo "$cofile" | sed -e 's|[/\\:.-]|_|g'`.d
-while true; do
-  if mkdir "$lockdir" >/dev/null 2>&1; then
-    break
-  fi
-  sleep 1
-done
-# FIXME: race condition here if user kills between mkdir and trap.
-trap "rmdir '$lockdir'; exit 1" 1 2 15
-
-# Run the compile.
-"$@"
-ret=$?
-
-if test -f "$cofile"; then
-  test "$cofile" = "$ofile" || mv "$cofile" "$ofile"
-elif test -f "${cofile}bj"; then
-  test "${cofile}bj" = "$ofile" || mv "${cofile}bj" "$ofile"
-fi
-
-rmdir "$lockdir"
-exit $ret
-
-# Local Variables:
-# mode: shell-script
-# sh-indentation: 2
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:

config/depcomp

@@ -1,708 +0,0 @@
-#! /bin/sh
-# depcomp - compile a program generating dependencies as side-effects
-
-scriptversion=2012-03-27.16; # UTC
-
-# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2006, 2007, 2009, 2010,
-# 2011, 2012 Free Software Foundation, Inc.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# As a special exception to the GNU General Public License, if you
-# distribute this file as part of a program that contains a
-# configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-# Originally written by Alexandre Oliva <oliva@dcc.unicamp.br>.
-
-case $1 in
-  '')
-     echo "$0: No command.  Try '$0 --help' for more information." 1>&2
-     exit 1;
-     ;;
-  -h | --h*)
-    cat <<\EOF
-Usage: depcomp [--help] [--version] PROGRAM [ARGS]
-
-Run PROGRAMS ARGS to compile a file, generating dependencies
-as side-effects.
-
-Environment variables:
-  depmode     Dependency tracking mode.
-  source      Source file read by 'PROGRAMS ARGS'.
-  object      Object file output by 'PROGRAMS ARGS'.
-  DEPDIR      directory where to store dependencies.
-  depfile     Dependency file to output.
-  tmpdepfile  Temporary file to use when outputting dependencies.
-  libtool     Whether libtool is used (yes/no).
-
-Report bugs to <bug-automake@gnu.org>.
-EOF
-    exit $?
-    ;;
-  -v | --v*)
-    echo "depcomp $scriptversion"
-    exit $?
-    ;;
-esac
-
-# A tabulation character.
-tab='	'
-# A newline character.
-nl='
-'
-
-if test -z "$depmode" || test -z "$source" || test -z "$object"; then
-  echo "depcomp: Variables source, object and depmode must be set" 1>&2
-  exit 1
-fi
-
-# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po.
-depfile=${depfile-`echo "$object" |
-  sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`}
-tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`}
-
-rm -f "$tmpdepfile"
-
-# Some modes work just like other modes, but use different flags.  We
-# parameterize here, but still list the modes in the big case below,
-# to make depend.m4 easier to write.  Note that we *cannot* use a case
-# here, because this file can only contain one case statement.
-if test "$depmode" = hp; then
-  # HP compiler uses -M and no extra arg.
-  gccflag=-M
-  depmode=gcc
-fi
-
-if test "$depmode" = dashXmstdout; then
-   # This is just like dashmstdout with a different argument.
-   dashmflag=-xM
-   depmode=dashmstdout
-fi
-
-cygpath_u="cygpath -u -f -"
-if test "$depmode" = msvcmsys; then
-   # This is just like msvisualcpp but w/o cygpath translation.
-   # Just convert the backslash-escaped backslashes to single forward
-   # slashes to satisfy depend.m4
-   cygpath_u='sed s,\\\\,/,g'
-   depmode=msvisualcpp
-fi
-
-if test "$depmode" = msvc7msys; then
-   # This is just like msvc7 but w/o cygpath translation.
-   # Just convert the backslash-escaped backslashes to single forward
-   # slashes to satisfy depend.m4
-   cygpath_u='sed s,\\\\,/,g'
-   depmode=msvc7
-fi
-
-if test "$depmode" = xlc; then
-   # IBM C/C++ Compilers xlc/xlC can output gcc-like dependency informations.
-   gccflag=-qmakedep=gcc,-MF
-   depmode=gcc
-fi
-
-case "$depmode" in
-gcc3)
-## gcc 3 implements dependency tracking that does exactly what
-## we want.  Yay!  Note: for some reason libtool 1.4 doesn't like
-## it if -MD -MP comes after the -MF stuff.  Hmm.
-## Unfortunately, FreeBSD c89 acceptance of flags depends upon
-## the command line argument order; so add the flags where they
-## appear in depend2.am.  Note that the slowdown incurred here
-## affects only configure: in makefiles, %FASTDEP% shortcuts this.
-  for arg
-  do
-    case $arg in
-    -c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;;
-    *)  set fnord "$@" "$arg" ;;
-    esac
-    shift # fnord
-    shift # $arg
-  done
-  "$@"
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  mv "$tmpdepfile" "$depfile"
-  ;;
-
-gcc)
-## There are various ways to get dependency output from gcc.  Here's
-## why we pick this rather obscure method:
-## - Don't want to use -MD because we'd like the dependencies to end
-##   up in a subdir.  Having to rename by hand is ugly.
-##   (We might end up doing this anyway to support other compilers.)
-## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like
-##   -MM, not -M (despite what the docs say).
-## - Using -M directly means running the compiler twice (even worse
-##   than renaming).
-  if test -z "$gccflag"; then
-    gccflag=-MD,
-  fi
-  "$@" -Wp,"$gccflag$tmpdepfile"
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  rm -f "$depfile"
-  echo "$object : \\" > "$depfile"
-  alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
-## The second -e expression handles DOS-style file names with drive letters.
-  sed -e 's/^[^:]*: / /' \
-      -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile"
-## This next piece of magic avoids the "deleted header file" problem.
-## The problem is that when a header file which appears in a .P file
-## is deleted, the dependency causes make to die (because there is
-## typically no way to rebuild the header).  We avoid this by adding
-## dummy dependencies for each header file.  Too bad gcc doesn't do
-## this for us directly.
-  tr ' ' "$nl" < "$tmpdepfile" |
-## Some versions of gcc put a space before the ':'.  On the theory
-## that the space means something, we add a space to the output as
-## well.  hp depmode also adds that space, but also prefixes the VPATH
-## to the object.  Take care to not repeat it in the output.
-## Some versions of the HPUX 10.20 sed can't process this invocation
-## correctly.  Breaking it into two sed invocations is a workaround.
-    sed -e 's/^\\$//' -e '/^$/d' -e "s|.*$object$||" -e '/:$/d' \
-      | sed -e 's/$/ :/' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-hp)
-  # This case exists only to let depend.m4 do its work.  It works by
-  # looking at the text of this script.  This case will never be run,
-  # since it is checked for above.
-  exit 1
-  ;;
-
-sgi)
-  if test "$libtool" = yes; then
-    "$@" "-Wp,-MDupdate,$tmpdepfile"
-  else
-    "$@" -MDupdate "$tmpdepfile"
-  fi
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  rm -f "$depfile"
-
-  if test -f "$tmpdepfile"; then  # yes, the sourcefile depend on other files
-    echo "$object : \\" > "$depfile"
-
-    # Clip off the initial element (the dependent).  Don't try to be
-    # clever and replace this with sed code, as IRIX sed won't handle
-    # lines with more than a fixed number of characters (4096 in
-    # IRIX 6.2 sed, 8192 in IRIX 6.5).  We also remove comment lines;
-    # the IRIX cc adds comments like '#:fec' to the end of the
-    # dependency line.
-    tr ' ' "$nl" < "$tmpdepfile" \
-    | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \
-    tr "$nl" ' ' >> "$depfile"
-    echo >> "$depfile"
-
-    # The second pass generates a dummy entry for each header file.
-    tr ' ' "$nl" < "$tmpdepfile" \
-   | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \
-   >> "$depfile"
-  else
-    # The sourcefile does not contain any dependencies, so just
-    # store a dummy comment line, to avoid errors with the Makefile
-    # "include basename.Plo" scheme.
-    echo "#dummy" > "$depfile"
-  fi
-  rm -f "$tmpdepfile"
-  ;;
-
-xlc)
-  # This case exists only to let depend.m4 do its work.  It works by
-  # looking at the text of this script.  This case will never be run,
-  # since it is checked for above.
-  exit 1
-  ;;
-
-aix)
-  # The C for AIX Compiler uses -M and outputs the dependencies
-  # in a .u file.  In older versions, this file always lives in the
-  # current directory.  Also, the AIX compiler puts '$object:' at the
-  # start of each line; $object doesn't have directory information.
-  # Version 6 uses the directory in both cases.
-  dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
-  test "x$dir" = "x$object" && dir=
-  base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
-  if test "$libtool" = yes; then
-    tmpdepfile1=$dir$base.u
-    tmpdepfile2=$base.u
-    tmpdepfile3=$dir.libs/$base.u
-    "$@" -Wc,-M
-  else
-    tmpdepfile1=$dir$base.u
-    tmpdepfile2=$dir$base.u
-    tmpdepfile3=$dir$base.u
-    "$@" -M
-  fi
-  stat=$?
-
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
-    exit $stat
-  fi
-
-  for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
-  do
-    test -f "$tmpdepfile" && break
-  done
-  if test -f "$tmpdepfile"; then
-    # Each line is of the form 'foo.o: dependent.h'.
-    # Do two passes, one to just change these to
-    # '$object: dependent.h' and one to simply 'dependent.h:'.
-    sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
-    sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
-  else
-    # The sourcefile does not contain any dependencies, so just
-    # store a dummy comment line, to avoid errors with the Makefile
-    # "include basename.Plo" scheme.
-    echo "#dummy" > "$depfile"
-  fi
-  rm -f "$tmpdepfile"
-  ;;
-
-icc)
-  # Intel's C compiler anf tcc (Tiny C Compiler) understand '-MD -MF file'.
-  # However on
-  #    $CC -MD -MF foo.d -c -o sub/foo.o sub/foo.c
-  # ICC 7.0 will fill foo.d with something like
-  #    foo.o: sub/foo.c
-  #    foo.o: sub/foo.h
-  # which is wrong.  We want
-  #    sub/foo.o: sub/foo.c
-  #    sub/foo.o: sub/foo.h
-  #    sub/foo.c:
-  #    sub/foo.h:
-  # ICC 7.1 will output
-  #    foo.o: sub/foo.c sub/foo.h
-  # and will wrap long lines using '\':
-  #    foo.o: sub/foo.c ... \
-  #     sub/foo.h ... \
-  #     ...
-  # tcc 0.9.26 (FIXME still under development at the moment of writing)
-  # will emit a similar output, but also prepend the continuation lines
-  # with horizontal tabulation characters.
-  "$@" -MD -MF "$tmpdepfile"
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  rm -f "$depfile"
-  # Each line is of the form 'foo.o: dependent.h',
-  # or 'foo.o: dep1.h dep2.h \', or ' dep3.h dep4.h \'.
-  # Do two passes, one to just change these to
-  # '$object: dependent.h' and one to simply 'dependent.h:'.
-  sed -e "s/^[ $tab][ $tab]*/  /" -e "s,^[^:]*:,$object :," \
-    < "$tmpdepfile" > "$depfile"
-  sed '
-    s/[ '"$tab"'][ '"$tab"']*/ /g
-    s/^ *//
-    s/ *\\*$//
-    s/^[^:]*: *//
-    /^$/d
-    /:$/d
-    s/$/ :/
-  ' < "$tmpdepfile" >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-hp2)
-  # The "hp" stanza above does not work with aCC (C++) and HP's ia64
-  # compilers, which have integrated preprocessors.  The correct option
-  # to use with these is +Maked; it writes dependencies to a file named
-  # 'foo.d', which lands next to the object file, wherever that
-  # happens to be.
-  # Much of this is similar to the tru64 case; see comments there.
-  dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
-  test "x$dir" = "x$object" && dir=
-  base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
-  if test "$libtool" = yes; then
-    tmpdepfile1=$dir$base.d
-    tmpdepfile2=$dir.libs/$base.d
-    "$@" -Wc,+Maked
-  else
-    tmpdepfile1=$dir$base.d
-    tmpdepfile2=$dir$base.d
-    "$@" +Maked
-  fi
-  stat=$?
-  if test $stat -eq 0; then :
-  else
-     rm -f "$tmpdepfile1" "$tmpdepfile2"
-     exit $stat
-  fi
-
-  for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2"
-  do
-    test -f "$tmpdepfile" && break
-  done
-  if test -f "$tmpdepfile"; then
-    sed -e "s,^.*\.[a-z]*:,$object:," "$tmpdepfile" > "$depfile"
-    # Add 'dependent.h:' lines.
-    sed -ne '2,${
-	       s/^ *//
-	       s/ \\*$//
-	       s/$/:/
-	       p
-	     }' "$tmpdepfile" >> "$depfile"
-  else
-    echo "#dummy" > "$depfile"
-  fi
-  rm -f "$tmpdepfile" "$tmpdepfile2"
-  ;;
-
-tru64)
-   # The Tru64 compiler uses -MD to generate dependencies as a side
-   # effect.  'cc -MD -o foo.o ...' puts the dependencies into 'foo.o.d'.
-   # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put
-   # dependencies in 'foo.d' instead, so we check for that too.
-   # Subdirectories are respected.
-   dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
-   test "x$dir" = "x$object" && dir=
-   base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
-
-   if test "$libtool" = yes; then
-      # With Tru64 cc, shared objects can also be used to make a
-      # static library.  This mechanism is used in libtool 1.4 series to
-      # handle both shared and static libraries in a single compilation.
-      # With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d.
-      #
-      # With libtool 1.5 this exception was removed, and libtool now
-      # generates 2 separate objects for the 2 libraries.  These two
-      # compilations output dependencies in $dir.libs/$base.o.d and
-      # in $dir$base.o.d.  We have to check for both files, because
-      # one of the two compilations can be disabled.  We should prefer
-      # $dir$base.o.d over $dir.libs/$base.o.d because the latter is
-      # automatically cleaned when .libs/ is deleted, while ignoring
-      # the former would cause a distcleancheck panic.
-      tmpdepfile1=$dir.libs/$base.lo.d   # libtool 1.4
-      tmpdepfile2=$dir$base.o.d          # libtool 1.5
-      tmpdepfile3=$dir.libs/$base.o.d    # libtool 1.5
-      tmpdepfile4=$dir.libs/$base.d      # Compaq CCC V6.2-504
-      "$@" -Wc,-MD
-   else
-      tmpdepfile1=$dir$base.o.d
-      tmpdepfile2=$dir$base.d
-      tmpdepfile3=$dir$base.d
-      tmpdepfile4=$dir$base.d
-      "$@" -MD
-   fi
-
-   stat=$?
-   if test $stat -eq 0; then :
-   else
-      rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
-      exit $stat
-   fi
-
-   for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
-   do
-     test -f "$tmpdepfile" && break
-   done
-   if test -f "$tmpdepfile"; then
-      sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
-      sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
-   else
-      echo "#dummy" > "$depfile"
-   fi
-   rm -f "$tmpdepfile"
-   ;;
-
-msvc7)
-  if test "$libtool" = yes; then
-    showIncludes=-Wc,-showIncludes
-  else
-    showIncludes=-showIncludes
-  fi
-  "$@" $showIncludes > "$tmpdepfile"
-  stat=$?
-  grep -v '^Note: including file: ' "$tmpdepfile"
-  if test "$stat" = 0; then :
-  else
-    rm -f "$tmpdepfile"
-    exit $stat
-  fi
-  rm -f "$depfile"
-  echo "$object : \\" > "$depfile"
-  # The first sed program below extracts the file names and escapes
-  # backslashes for cygpath.  The second sed program outputs the file
-  # name when reading, but also accumulates all include files in the
-  # hold buffer in order to output them again at the end.  This only
-  # works with sed implementations that can handle large buffers.
-  sed < "$tmpdepfile" -n '
-/^Note: including file:  *\(.*\)/ {
-  s//\1/
-  s/\\/\\\\/g
-  p
-}' | $cygpath_u | sort -u | sed -n '
-s/ /\\ /g
-s/\(.*\)/'"$tab"'\1 \\/p
-s/.\(.*\) \\/\1:/
-H
-$ {
-  s/.*/'"$tab"'/
-  G
-  p
-}' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-msvc7msys)
-  # This case exists only to let depend.m4 do its work.  It works by
-  # looking at the text of this script.  This case will never be run,
-  # since it is checked for above.
-  exit 1
-  ;;
-
-#nosideeffect)
-  # This comment above is used by automake to tell side-effect
-  # dependency tracking mechanisms from slower ones.
-
-dashmstdout)
-  # Important note: in order to support this mode, a compiler *must*
-  # always write the preprocessed file to stdout, regardless of -o.
-  "$@" || exit $?
-
-  # Remove the call to Libtool.
-  if test "$libtool" = yes; then
-    while test "X$1" != 'X--mode=compile'; do
-      shift
-    done
-    shift
-  fi
-
-  # Remove '-o $object'.
-  IFS=" "
-  for arg
-  do
-    case $arg in
-    -o)
-      shift
-      ;;
-    $object)
-      shift
-      ;;
-    *)
-      set fnord "$@" "$arg"
-      shift # fnord
-      shift # $arg
-      ;;
-    esac
-  done
-
-  test -z "$dashmflag" && dashmflag=-M
-  # Require at least two characters before searching for ':'
-  # in the target name.  This is to cope with DOS-style filenames:
-  # a dependency such as 'c:/foo/bar' could be seen as target 'c' otherwise.
-  "$@" $dashmflag |
-    sed 's:^['"$tab"' ]*[^:'"$tab"' ][^:][^:]*\:['"$tab"' ]*:'"$object"'\: :' > "$tmpdepfile"
-  rm -f "$depfile"
-  cat < "$tmpdepfile" > "$depfile"
-  tr ' ' "$nl" < "$tmpdepfile" | \
-## Some versions of the HPUX 10.20 sed can't process this invocation
-## correctly.  Breaking it into two sed invocations is a workaround.
-    sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-dashXmstdout)
-  # This case only exists to satisfy depend.m4.  It is never actually
-  # run, as this mode is specially recognized in the preamble.
-  exit 1
-  ;;
-
-makedepend)
-  "$@" || exit $?
-  # Remove any Libtool call
-  if test "$libtool" = yes; then
-    while test "X$1" != 'X--mode=compile'; do
-      shift
-    done
-    shift
-  fi
-  # X makedepend
-  shift
-  cleared=no eat=no
-  for arg
-  do
-    case $cleared in
-    no)
-      set ""; shift
-      cleared=yes ;;
-    esac
-    if test $eat = yes; then
-      eat=no
-      continue
-    fi
-    case "$arg" in
-    -D*|-I*)
-      set fnord "$@" "$arg"; shift ;;
-    # Strip any option that makedepend may not understand.  Remove
-    # the object too, otherwise makedepend will parse it as a source file.
-    -arch)
-      eat=yes ;;
-    -*|$object)
-      ;;
-    *)
-      set fnord "$@" "$arg"; shift ;;
-    esac
-  done
-  obj_suffix=`echo "$object" | sed 's/^.*\././'`
-  touch "$tmpdepfile"
-  ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@"
-  rm -f "$depfile"
-  # makedepend may prepend the VPATH from the source file name to the object.
-  # No need to regex-escape $object, excess matching of '.' is harmless.
-  sed "s|^.*\($object *:\)|\1|" "$tmpdepfile" > "$depfile"
-  sed '1,2d' "$tmpdepfile" | tr ' ' "$nl" | \
-## Some versions of the HPUX 10.20 sed can't process this invocation
-## correctly.  Breaking it into two sed invocations is a workaround.
-    sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
-  rm -f "$tmpdepfile" "$tmpdepfile".bak
-  ;;
-
-cpp)
-  # Important note: in order to support this mode, a compiler *must*
-  # always write the preprocessed file to stdout.
-  "$@" || exit $?
-
-  # Remove the call to Libtool.
-  if test "$libtool" = yes; then
-    while test "X$1" != 'X--mode=compile'; do
-      shift
-    done
-    shift
-  fi
-
-  # Remove '-o $object'.
-  IFS=" "
-  for arg
-  do
-    case $arg in
-    -o)
-      shift
-      ;;
-    $object)
-      shift
-      ;;
-    *)
-      set fnord "$@" "$arg"
-      shift # fnord
-      shift # $arg
-      ;;
-    esac
-  done
-
-  "$@" -E |
-    sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \
-       -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' |
-    sed '$ s: \\$::' > "$tmpdepfile"
-  rm -f "$depfile"
-  echo "$object : \\" > "$depfile"
-  cat < "$tmpdepfile" >> "$depfile"
-  sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-msvisualcpp)
-  # Important note: in order to support this mode, a compiler *must*
-  # always write the preprocessed file to stdout.
-  "$@" || exit $?
-
-  # Remove the call to Libtool.
-  if test "$libtool" = yes; then
-    while test "X$1" != 'X--mode=compile'; do
-      shift
-    done
-    shift
-  fi
-
-  IFS=" "
-  for arg
-  do
-    case "$arg" in
-    -o)
-      shift
-      ;;
-    $object)
-      shift
-      ;;
-    "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI")
-	set fnord "$@"
-	shift
-	shift
-	;;
-    *)
-	set fnord "$@" "$arg"
-	shift
-	shift
-	;;
-    esac
-  done
-  "$@" -E 2>/dev/null |
-  sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile"
-  rm -f "$depfile"
-  echo "$object : \\" > "$depfile"
-  sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::'"$tab"'\1 \\:p' >> "$depfile"
-  echo "$tab" >> "$depfile"
-  sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile"
-  rm -f "$tmpdepfile"
-  ;;
-
-msvcmsys)
-  # This case exists only to let depend.m4 do its work.  It works by
-  # looking at the text of this script.  This case will never be run,
-  # since it is checked for above.
-  exit 1
-  ;;
-
-none)
-  exec "$@"
-  ;;
-
-*)
-  echo "Unknown depmode $depmode" 1>&2
-  exit 1
-  ;;
-esac
-
-exit 0
-
-# Local Variables:
-# mode: shell-script
-# sh-indentation: 2
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:

config/install-sh

@@ -1,527 +0,0 @@
-#!/bin/sh
-# install - install a program, script, or datafile
-
-scriptversion=2011-01-19.21; # UTC
-
-# This originates from X11R5 (mit/util/scripts/install.sh), which was
-# later released in X11R6 (xc/config/util/install.sh) with the
-# following copyright and license.
-#
-# Copyright (C) 1994 X Consortium
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to
-# deal in the Software without restriction, including without limitation the
-# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-# sell copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
-# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
-# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-# Except as contained in this notice, the name of the X Consortium shall not
-# be used in advertising or otherwise to promote the sale, use or other deal-
-# ings in this Software without prior written authorization from the X Consor-
-# tium.
-#
-#
-# FSF changes to this file are in the public domain.
-#
-# Calling this script install-sh is preferred over install.sh, to prevent
-# `make' implicit rules from creating a file called install from it
-# when there is no Makefile.
-#
-# This script is compatible with the BSD install script, but was written
-# from scratch.
-
-nl='
-'
-IFS=" ""	$nl"
-
-# set DOITPROG to echo to test this script
-
-# Don't use :- since 4.3BSD and earlier shells don't like it.
-doit=${DOITPROG-}
-if test -z "$doit"; then
-  doit_exec=exec
-else
-  doit_exec=$doit
-fi
-
-# Put in absolute file names if you don't have them in your path;
-# or use environment vars.
-
-chgrpprog=${CHGRPPROG-chgrp}
-chmodprog=${CHMODPROG-chmod}
-chownprog=${CHOWNPROG-chown}
-cmpprog=${CMPPROG-cmp}
-cpprog=${CPPROG-cp}
-mkdirprog=${MKDIRPROG-mkdir}
-mvprog=${MVPROG-mv}
-rmprog=${RMPROG-rm}
-stripprog=${STRIPPROG-strip}
-
-posix_glob='?'
-initialize_posix_glob='
-  test "$posix_glob" != "?" || {
-    if (set -f) 2>/dev/null; then
-      posix_glob=
-    else
-      posix_glob=:
-    fi
-  }
-'
-
-posix_mkdir=
-
-# Desired mode of installed file.
-mode=0755
-
-chgrpcmd=
-chmodcmd=$chmodprog
-chowncmd=
-mvcmd=$mvprog
-rmcmd="$rmprog -f"
-stripcmd=
-
-src=
-dst=
-dir_arg=
-dst_arg=
-
-copy_on_change=false
-no_target_directory=
-
-usage="\
-Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
-   or: $0 [OPTION]... SRCFILES... DIRECTORY
-   or: $0 [OPTION]... -t DIRECTORY SRCFILES...
-   or: $0 [OPTION]... -d DIRECTORIES...
-
-In the 1st form, copy SRCFILE to DSTFILE.
-In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
-In the 4th, create DIRECTORIES.
-
-Options:
-     --help     display this help and exit.
-     --version  display version info and exit.
-
-  -c            (ignored)
-  -C            install only if different (preserve the last data modification time)
-  -d            create directories instead of installing files.
-  -g GROUP      $chgrpprog installed files to GROUP.
-  -m MODE       $chmodprog installed files to MODE.
-  -o USER       $chownprog installed files to USER.
-  -s            $stripprog installed files.
-  -t DIRECTORY  install into DIRECTORY.
-  -T            report an error if DSTFILE is a directory.
-
-Environment variables override the default commands:
-  CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
-  RMPROG STRIPPROG
-"
-
-while test $# -ne 0; do
-  case $1 in
-    -c) ;;
-
-    -C) copy_on_change=true;;
-
-    -d) dir_arg=true;;
-
-    -g) chgrpcmd="$chgrpprog $2"
-	shift;;
-
-    --help) echo "$usage"; exit $?;;
-
-    -m) mode=$2
-	case $mode in
-	  *' '* | *'	'* | *'
-'*	  | *'*'* | *'?'* | *'['*)
-	    echo "$0: invalid mode: $mode" >&2
-	    exit 1;;
-	esac
-	shift;;
-
-    -o) chowncmd="$chownprog $2"
-	shift;;
-
-    -s) stripcmd=$stripprog;;
-
-    -t) dst_arg=$2
-	# Protect names problematic for `test' and other utilities.
-	case $dst_arg in
-	  -* | [=\(\)!]) dst_arg=./$dst_arg;;
-	esac
-	shift;;
-
-    -T) no_target_directory=true;;
-
-    --version) echo "$0 $scriptversion"; exit $?;;
-
-    --)	shift
-	break;;
-
-    -*)	echo "$0: invalid option: $1" >&2
-	exit 1;;
-
-    *)  break;;
-  esac
-  shift
-done
-
-if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
-  # When -d is used, all remaining arguments are directories to create.
-  # When -t is used, the destination is already specified.
-  # Otherwise, the last argument is the destination.  Remove it from $@.
-  for arg
-  do
-    if test -n "$dst_arg"; then
-      # $@ is not empty: it contains at least $arg.
-      set fnord "$@" "$dst_arg"
-      shift # fnord
-    fi
-    shift # arg
-    dst_arg=$arg
-    # Protect names problematic for `test' and other utilities.
-    case $dst_arg in
-      -* | [=\(\)!]) dst_arg=./$dst_arg;;
-    esac
-  done
-fi
-
-if test $# -eq 0; then
-  if test -z "$dir_arg"; then
-    echo "$0: no input file specified." >&2
-    exit 1
-  fi
-  # It's OK to call `install-sh -d' without argument.
-  # This can happen when creating conditional directories.
-  exit 0
-fi
-
-if test -z "$dir_arg"; then
-  do_exit='(exit $ret); exit $ret'
-  trap "ret=129; $do_exit" 1
-  trap "ret=130; $do_exit" 2
-  trap "ret=141; $do_exit" 13
-  trap "ret=143; $do_exit" 15
-
-  # Set umask so as not to create temps with too-generous modes.
-  # However, 'strip' requires both read and write access to temps.
-  case $mode in
-    # Optimize common cases.
-    *644) cp_umask=133;;
-    *755) cp_umask=22;;
-
-    *[0-7])
-      if test -z "$stripcmd"; then
-	u_plus_rw=
-      else
-	u_plus_rw='% 200'
-      fi
-      cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
-    *)
-      if test -z "$stripcmd"; then
-	u_plus_rw=
-      else
-	u_plus_rw=,u+rw
-      fi
-      cp_umask=$mode$u_plus_rw;;
-  esac
-fi
-
-for src
-do
-  # Protect names problematic for `test' and other utilities.
-  case $src in
-    -* | [=\(\)!]) src=./$src;;
-  esac
-
-  if test -n "$dir_arg"; then
-    dst=$src
-    dstdir=$dst
-    test -d "$dstdir"
-    dstdir_status=$?
-  else
-
-    # Waiting for this to be detected by the "$cpprog $src $dsttmp" command
-    # might cause directories to be created, which would be especially bad
-    # if $src (and thus $dsttmp) contains '*'.
-    if test ! -f "$src" && test ! -d "$src"; then
-      echo "$0: $src does not exist." >&2
-      exit 1
-    fi
-
-    if test -z "$dst_arg"; then
-      echo "$0: no destination specified." >&2
-      exit 1
-    fi
-    dst=$dst_arg
-
-    # If destination is a directory, append the input filename; won't work
-    # if double slashes aren't ignored.
-    if test -d "$dst"; then
-      if test -n "$no_target_directory"; then
-	echo "$0: $dst_arg: Is a directory" >&2
-	exit 1
-      fi
-      dstdir=$dst
-      dst=$dstdir/`basename "$src"`
-      dstdir_status=0
-    else
-      # Prefer dirname, but fall back on a substitute if dirname fails.
-      dstdir=`
-	(dirname "$dst") 2>/dev/null ||
-	expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
-	     X"$dst" : 'X\(//\)[^/]' \| \
-	     X"$dst" : 'X\(//\)$' \| \
-	     X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
-	echo X"$dst" |
-	    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)[^/].*/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\).*/{
-		   s//\1/
-		   q
-		 }
-		 s/.*/./; q'
-      `
-
-      test -d "$dstdir"
-      dstdir_status=$?
-    fi
-  fi
-
-  obsolete_mkdir_used=false
-
-  if test $dstdir_status != 0; then
-    case $posix_mkdir in
-      '')
-	# Create intermediate dirs using mode 755 as modified by the umask.
-	# This is like FreeBSD 'install' as of 1997-10-28.
-	umask=`umask`
-	case $stripcmd.$umask in
-	  # Optimize common cases.
-	  *[2367][2367]) mkdir_umask=$umask;;
-	  .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
-
-	  *[0-7])
-	    mkdir_umask=`expr $umask + 22 \
-	      - $umask % 100 % 40 + $umask % 20 \
-	      - $umask % 10 % 4 + $umask % 2
-	    `;;
-	  *) mkdir_umask=$umask,go-w;;
-	esac
-
-	# With -d, create the new directory with the user-specified mode.
-	# Otherwise, rely on $mkdir_umask.
-	if test -n "$dir_arg"; then
-	  mkdir_mode=-m$mode
-	else
-	  mkdir_mode=
-	fi
-
-	posix_mkdir=false
-	case $umask in
-	  *[123567][0-7][0-7])
-	    # POSIX mkdir -p sets u+wx bits regardless of umask, which
-	    # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
-	    ;;
-	  *)
-	    tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
-	    trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
-
-	    if (umask $mkdir_umask &&
-		exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
-	    then
-	      if test -z "$dir_arg" || {
-		   # Check for POSIX incompatibilities with -m.
-		   # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
-		   # other-writeable bit of parent directory when it shouldn't.
-		   # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
-		   ls_ld_tmpdir=`ls -ld "$tmpdir"`
-		   case $ls_ld_tmpdir in
-		     d????-?r-*) different_mode=700;;
-		     d????-?--*) different_mode=755;;
-		     *) false;;
-		   esac &&
-		   $mkdirprog -m$different_mode -p -- "$tmpdir" && {
-		     ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
-		     test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
-		   }
-		 }
-	      then posix_mkdir=:
-	      fi
-	      rmdir "$tmpdir/d" "$tmpdir"
-	    else
-	      # Remove any dirs left behind by ancient mkdir implementations.
-	      rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
-	    fi
-	    trap '' 0;;
-	esac;;
-    esac
-
-    if
-      $posix_mkdir && (
-	umask $mkdir_umask &&
-	$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
-      )
-    then :
-    else
-
-      # The umask is ridiculous, or mkdir does not conform to POSIX,
-      # or it failed possibly due to a race condition.  Create the
-      # directory the slow way, step by step, checking for races as we go.
-
-      case $dstdir in
-	/*) prefix='/';;
-	[-=\(\)!]*) prefix='./';;
-	*)  prefix='';;
-      esac
-
-      eval "$initialize_posix_glob"
-
-      oIFS=$IFS
-      IFS=/
-      $posix_glob set -f
-      set fnord $dstdir
-      shift
-      $posix_glob set +f
-      IFS=$oIFS
-
-      prefixes=
-
-      for d
-      do
-	test X"$d" = X && continue
-
-	prefix=$prefix$d
-	if test -d "$prefix"; then
-	  prefixes=
-	else
-	  if $posix_mkdir; then
-	    (umask=$mkdir_umask &&
-	     $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
-	    # Don't fail if two instances are running concurrently.
-	    test -d "$prefix" || exit 1
-	  else
-	    case $prefix in
-	      *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
-	      *) qprefix=$prefix;;
-	    esac
-	    prefixes="$prefixes '$qprefix'"
-	  fi
-	fi
-	prefix=$prefix/
-      done
-
-      if test -n "$prefixes"; then
-	# Don't fail if two instances are running concurrently.
-	(umask $mkdir_umask &&
-	 eval "\$doit_exec \$mkdirprog $prefixes") ||
-	  test -d "$dstdir" || exit 1
-	obsolete_mkdir_used=true
-      fi
-    fi
-  fi
-
-  if test -n "$dir_arg"; then
-    { test -z "$chowncmd" || $doit $chowncmd "$dst"; } &&
-    { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } &&
-    { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false ||
-      test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1
-  else
-
-    # Make a couple of temp file names in the proper directory.
-    dsttmp=$dstdir/_inst.$$_
-    rmtmp=$dstdir/_rm.$$_
-
-    # Trap to clean up those temp files at exit.
-    trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
-
-    # Copy the file name to the temp name.
-    (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
-
-    # and set any options; do chmod last to preserve setuid bits.
-    #
-    # If any of these fail, we abort the whole thing.  If we want to
-    # ignore errors from any of these, just make sure not to ignore
-    # errors from the above "$doit $cpprog $src $dsttmp" command.
-    #
-    { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
-    { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
-    { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
-    { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
-
-    # If -C, don't bother to copy if it wouldn't change the file.
-    if $copy_on_change &&
-       old=`LC_ALL=C ls -dlL "$dst"	2>/dev/null` &&
-       new=`LC_ALL=C ls -dlL "$dsttmp"	2>/dev/null` &&
-
-       eval "$initialize_posix_glob" &&
-       $posix_glob set -f &&
-       set X $old && old=:$2:$4:$5:$6 &&
-       set X $new && new=:$2:$4:$5:$6 &&
-       $posix_glob set +f &&
-
-       test "$old" = "$new" &&
-       $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
-    then
-      rm -f "$dsttmp"
-    else
-      # Rename the file to the real destination.
-      $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
-
-      # The rename failed, perhaps because mv can't rename something else
-      # to itself, or perhaps because mv is so ancient that it does not
-      # support -f.
-      {
-	# Now remove or move aside any old file at destination location.
-	# We try this two ways since rm can't unlink itself on some
-	# systems and the destination file might be busy for other
-	# reasons.  In this case, the final cleanup might fail but the new
-	# file should still install successfully.
-	{
-	  test ! -f "$dst" ||
-	  $doit $rmcmd -f "$dst" 2>/dev/null ||
-	  { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
-	    { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
-	  } ||
-	  { echo "$0: cannot unlink or rename $dst" >&2
-	    (exit 1); exit 1
-	  }
-	} &&
-
-	# Now rename the file to the real destination.
-	$doit $mvcmd "$dsttmp" "$dst"
-      }
-    fi || exit 1
-
-    trap '' 0
-  fi
-done
-
-# Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:

config/m4/.gitignore

@@ -0,0 +1,2 @@
+libtool.m4
+lt*.m4

config/m4/find-func-no-libs.m4

@@ -0,0 +1,8 @@
+dnl Shamelessly stolen from Heimdal
+dnl
+dnl Look for function in any of the specified libraries
+dnl
+
+dnl AC_FIND_FUNC_NO_LIBS(func, libraries, includes, arguments, extra libs, extra args)
+AC_DEFUN([AC_FIND_FUNC_NO_LIBS], [
+AC_FIND_FUNC_NO_LIBS2([$1], ["" $2], [$3], [$4], [$5], [$6])])

config/m4/find-func-no-libs2.m4

@@ -0,0 +1,62 @@
+dnl Shamelessly stolen from Heimdal
+dnl
+dnl Look for function in any of the specified libraries
+dnl
+
+dnl AC_FIND_FUNC_NO_LIBS2(func, libraries, includes, arguments, extra libs, extra args)
+AC_DEFUN([AC_FIND_FUNC_NO_LIBS2], [
+
+AC_MSG_CHECKING([for $1])
+AC_CACHE_VAL(ac_cv_funclib_$1,
+[
+if eval "test \"\$ac_cv_func_$1\" != yes" ; then
+	ac_save_LIBS="$LIBS"
+	for ac_lib in $2; do
+		case "$ac_lib" in
+		"") ;;
+		yes) ac_lib="" ;;
+		no) continue ;;
+		-l*) ;;
+		*) ac_lib="-l$ac_lib" ;;
+		esac
+		LIBS="$6 $ac_lib $5 $ac_save_LIBS"
+		AC_LINK_IFELSE([AC_LANG_PROGRAM([[$3]],[[$1($4)]])],[eval "if test -n \"$ac_lib\";then ac_cv_funclib_$1=$ac_lib; else ac_cv_funclib_$1=yes; fi";break])
+	done
+	eval "ac_cv_funclib_$1=\${ac_cv_funclib_$1-no}"
+	LIBS="$ac_save_LIBS"
+fi
+])
+
+eval "ac_res=\$ac_cv_funclib_$1"
+
+if false; then
+	AC_CHECK_FUNCS($1)
+dnl	AC_CHECK_LIBS($2, foo)
+fi
+# $1
+eval "ac_tr_func=HAVE_[]upcase($1)"
+eval "ac_tr_lib=HAVE_LIB[]upcase($ac_res | sed -e 's/-l//')"
+eval "LIB_$1=$ac_res"
+
+case "$ac_res" in
+	yes)
+	eval "ac_cv_func_$1=yes"
+	eval "LIB_$1="
+	AC_DEFINE_UNQUOTED($ac_tr_func)
+	AC_MSG_RESULT([yes])
+	;;
+	no)
+	eval "ac_cv_func_$1=no"
+	eval "LIB_$1="
+	AC_MSG_RESULT([no])
+	;;
+	*)
+	eval "ac_cv_func_$1=yes"
+	eval "ac_cv_lib_`echo "$ac_res" | sed 's/-l//'`=yes"
+	AC_DEFINE_UNQUOTED($ac_tr_func)
+	AC_DEFINE_UNQUOTED($ac_tr_lib)
+	AC_MSG_RESULT([yes, in $ac_res])
+	;;
+esac
+AC_SUBST(LIB_$1)
+])

config/m4/find-func.m4

@@ -0,0 +1,9 @@
+dnl Shamelessly stolen from Heimdal
+dnl
+dnl AC_FIND_FUNC(func, libraries, includes, arguments)
+AC_DEFUN([AC_FIND_FUNC], [
+AC_FIND_FUNC_NO_LIBS([$1], [$2], [$3], [$4])
+if test -n "$LIB_$1"; then
+	LIBS="$LIB_$1 $LIBS"
+fi
+])

config/m4/misc.m4

@@ -0,0 +1,3 @@
+
+dnl Shamelessly stolen from Heimdal
+AC_DEFUN([upcase],[`echo $1 | tr abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ`])dnl

config/missing

@@ -1,331 +0,0 @@
-#! /bin/sh
-# Common stub for a few missing GNU programs while installing.
-
-scriptversion=2012-01-06.13; # UTC
-
-# Copyright (C) 1996, 1997, 1999, 2000, 2002, 2003, 2004, 2005, 2006,
-# 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
-# Originally by Fran,cois Pinard <pinard@iro.umontreal.ca>, 1996.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# As a special exception to the GNU General Public License, if you
-# distribute this file as part of a program that contains a
-# configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-if test $# -eq 0; then
-  echo 1>&2 "Try \`$0 --help' for more information"
-  exit 1
-fi
-
-run=:
-sed_output='s/.* --output[ =]\([^ ]*\).*/\1/p'
-sed_minuso='s/.* -o \([^ ]*\).*/\1/p'
-
-# In the cases where this matters, `missing' is being run in the
-# srcdir already.
-if test -f configure.ac; then
-  configure_ac=configure.ac
-else
-  configure_ac=configure.in
-fi
-
-msg="missing on your system"
-
-case $1 in
---run)
-  # Try to run requested program, and just exit if it succeeds.
-  run=
-  shift
-  "$@" && exit 0
-  # Exit code 63 means version mismatch.  This often happens
-  # when the user try to use an ancient version of a tool on
-  # a file that requires a minimum version.  In this case we
-  # we should proceed has if the program had been absent, or
-  # if --run hadn't been passed.
-  if test $? = 63; then
-    run=:
-    msg="probably too old"
-  fi
-  ;;
-
-  -h|--h|--he|--hel|--help)
-    echo "\
-$0 [OPTION]... PROGRAM [ARGUMENT]...
-
-Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an
-error status if there is no known handling for PROGRAM.
-
-Options:
-  -h, --help      display this help and exit
-  -v, --version   output version information and exit
-  --run           try to run the given command, and emulate it if it fails
-
-Supported PROGRAM values:
-  aclocal      touch file \`aclocal.m4'
-  autoconf     touch file \`configure'
-  autoheader   touch file \`config.h.in'
-  autom4te     touch the output file, or create a stub one
-  automake     touch all \`Makefile.in' files
-  bison        create \`y.tab.[ch]', if possible, from existing .[ch]
-  flex         create \`lex.yy.c', if possible, from existing .c
-  help2man     touch the output file
-  lex          create \`lex.yy.c', if possible, from existing .c
-  makeinfo     touch the output file
-  yacc         create \`y.tab.[ch]', if possible, from existing .[ch]
-
-Version suffixes to PROGRAM as well as the prefixes \`gnu-', \`gnu', and
-\`g' are ignored when checking the name.
-
-Send bug reports to <bug-automake@gnu.org>."
-    exit $?
-    ;;
-
-  -v|--v|--ve|--ver|--vers|--versi|--versio|--version)
-    echo "missing $scriptversion (GNU Automake)"
-    exit $?
-    ;;
-
-  -*)
-    echo 1>&2 "$0: Unknown \`$1' option"
-    echo 1>&2 "Try \`$0 --help' for more information"
-    exit 1
-    ;;
-
-esac
-
-# normalize program name to check for.
-program=`echo "$1" | sed '
-  s/^gnu-//; t
-  s/^gnu//; t
-  s/^g//; t'`
-
-# Now exit if we have it, but it failed.  Also exit now if we
-# don't have it and --version was passed (most likely to detect
-# the program).  This is about non-GNU programs, so use $1 not
-# $program.
-case $1 in
-  lex*|yacc*)
-    # Not GNU programs, they don't have --version.
-    ;;
-
-  *)
-    if test -z "$run" && ($1 --version) > /dev/null 2>&1; then
-       # We have it, but it failed.
-       exit 1
-    elif test "x$2" = "x--version" || test "x$2" = "x--help"; then
-       # Could not run --version or --help.  This is probably someone
-       # running `$TOOL --version' or `$TOOL --help' to check whether
-       # $TOOL exists and not knowing $TOOL uses missing.
-       exit 1
-    fi
-    ;;
-esac
-
-# If it does not exist, or fails to run (possibly an outdated version),
-# try to emulate it.
-case $program in
-  aclocal*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified \`acinclude.m4' or \`${configure_ac}'.  You might want
-         to install the \`Automake' and \`Perl' packages.  Grab them from
-         any GNU archive site."
-    touch aclocal.m4
-    ;;
-
-  autoconf*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified \`${configure_ac}'.  You might want to install the
-         \`Autoconf' and \`GNU m4' packages.  Grab them from any GNU
-         archive site."
-    touch configure
-    ;;
-
-  autoheader*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified \`acconfig.h' or \`${configure_ac}'.  You might want
-         to install the \`Autoconf' and \`GNU m4' packages.  Grab them
-         from any GNU archive site."
-    files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' ${configure_ac}`
-    test -z "$files" && files="config.h"
-    touch_files=
-    for f in $files; do
-      case $f in
-      *:*) touch_files="$touch_files "`echo "$f" |
-				       sed -e 's/^[^:]*://' -e 's/:.*//'`;;
-      *) touch_files="$touch_files $f.in";;
-      esac
-    done
-    touch $touch_files
-    ;;
-
-  automake*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified \`Makefile.am', \`acinclude.m4' or \`${configure_ac}'.
-         You might want to install the \`Automake' and \`Perl' packages.
-         Grab them from any GNU archive site."
-    find . -type f -name Makefile.am -print |
-	   sed 's/\.am$/.in/' |
-	   while read f; do touch "$f"; done
-    ;;
-
-  autom4te*)
-    echo 1>&2 "\
-WARNING: \`$1' is needed, but is $msg.
-         You might have modified some files without having the
-         proper tools for further handling them.
-         You can get \`$1' as part of \`Autoconf' from any GNU
-         archive site."
-
-    file=`echo "$*" | sed -n "$sed_output"`
-    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
-    if test -f "$file"; then
-	touch $file
-    else
-	test -z "$file" || exec >$file
-	echo "#! /bin/sh"
-	echo "# Created by GNU Automake missing as a replacement of"
-	echo "#  $ $@"
-	echo "exit 0"
-	chmod +x $file
-	exit 1
-    fi
-    ;;
-
-  bison*|yacc*)
-    echo 1>&2 "\
-WARNING: \`$1' $msg.  You should only need it if
-         you modified a \`.y' file.  You may need the \`Bison' package
-         in order for those modifications to take effect.  You can get
-         \`Bison' from any GNU archive site."
-    rm -f y.tab.c y.tab.h
-    if test $# -ne 1; then
-        eval LASTARG=\${$#}
-	case $LASTARG in
-	*.y)
-	    SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'`
-	    if test -f "$SRCFILE"; then
-	         cp "$SRCFILE" y.tab.c
-	    fi
-	    SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'`
-	    if test -f "$SRCFILE"; then
-	         cp "$SRCFILE" y.tab.h
-	    fi
-	  ;;
-	esac
-    fi
-    if test ! -f y.tab.h; then
-	echo >y.tab.h
-    fi
-    if test ! -f y.tab.c; then
-	echo 'main() { return 0; }' >y.tab.c
-    fi
-    ;;
-
-  lex*|flex*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified a \`.l' file.  You may need the \`Flex' package
-         in order for those modifications to take effect.  You can get
-         \`Flex' from any GNU archive site."
-    rm -f lex.yy.c
-    if test $# -ne 1; then
-        eval LASTARG=\${$#}
-	case $LASTARG in
-	*.l)
-	    SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'`
-	    if test -f "$SRCFILE"; then
-	         cp "$SRCFILE" lex.yy.c
-	    fi
-	  ;;
-	esac
-    fi
-    if test ! -f lex.yy.c; then
-	echo 'main() { return 0; }' >lex.yy.c
-    fi
-    ;;
-
-  help2man*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-	 you modified a dependency of a manual page.  You may need the
-	 \`Help2man' package in order for those modifications to take
-	 effect.  You can get \`Help2man' from any GNU archive site."
-
-    file=`echo "$*" | sed -n "$sed_output"`
-    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
-    if test -f "$file"; then
-	touch $file
-    else
-	test -z "$file" || exec >$file
-	echo ".ab help2man is required to generate this page"
-	exit $?
-    fi
-    ;;
-
-  makeinfo*)
-    echo 1>&2 "\
-WARNING: \`$1' is $msg.  You should only need it if
-         you modified a \`.texi' or \`.texinfo' file, or any other file
-         indirectly affecting the aspect of the manual.  The spurious
-         call might also be the consequence of using a buggy \`make' (AIX,
-         DU, IRIX).  You might want to install the \`Texinfo' package or
-         the \`GNU make' package.  Grab either from any GNU archive site."
-    # The file to touch is that specified with -o ...
-    file=`echo "$*" | sed -n "$sed_output"`
-    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
-    if test -z "$file"; then
-      # ... or it is the one specified with @setfilename ...
-      infile=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'`
-      file=`sed -n '
-	/^@setfilename/{
-	  s/.* \([^ ]*\) *$/\1/
-	  p
-	  q
-	}' $infile`
-      # ... or it is derived from the source name (dir/f.texi becomes f.info)
-      test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info
-    fi
-    # If the file does not exist, the user really needs makeinfo;
-    # let's fail without touching anything.
-    test -f $file || exit 1
-    touch $file
-    ;;
-
-  *)
-    echo 1>&2 "\
-WARNING: \`$1' is needed, and is $msg.
-         You might have modified some files without having the
-         proper tools for further handling them.  Check the \`README' file,
-         it often tells you about the needed prerequisites for installing
-         this package.  You may also peek at any GNU archive site, in case
-         some other package would contain this missing \`$1' program."
-    exit 1
-    ;;
-esac
-
-exit 0
-
-# Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:

config/ylwrap

@@ -1,226 +0,0 @@
-#! /bin/sh
-# ylwrap - wrapper for lex/yacc invocations.
-
-scriptversion=2011-08-25.18; # UTC
-
-# Copyright (C) 1996, 1997, 1998, 1999, 2001, 2002, 2003, 2004, 2005,
-# 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
-#
-# Written by Tom Tromey <tromey@cygnus.com>.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# As a special exception to the GNU General Public License, if you
-# distribute this file as part of a program that contains a
-# configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-# This file is maintained in Automake, please report
-# bugs to <bug-automake@gnu.org> or send patches to
-# <automake-patches@gnu.org>.
-
-case "$1" in
-  '')
-    echo "$0: No files given.  Try \`$0 --help' for more information." 1>&2
-    exit 1
-    ;;
-  --basedir)
-    basedir=$2
-    shift 2
-    ;;
-  -h|--h*)
-    cat <<\EOF
-Usage: ylwrap [--help|--version] INPUT [OUTPUT DESIRED]... -- PROGRAM [ARGS]...
-
-Wrapper for lex/yacc invocations, renaming files as desired.
-
-  INPUT is the input file
-  OUTPUT is one file PROG generates
-  DESIRED is the file we actually want instead of OUTPUT
-  PROGRAM is program to run
-  ARGS are passed to PROG
-
-Any number of OUTPUT,DESIRED pairs may be used.
-
-Report bugs to <bug-automake@gnu.org>.
-EOF
-    exit $?
-    ;;
-  -v|--v*)
-    echo "ylwrap $scriptversion"
-    exit $?
-    ;;
-esac
-
-
-# The input.
-input="$1"
-shift
-case "$input" in
-  [\\/]* | ?:[\\/]*)
-    # Absolute path; do nothing.
-    ;;
-  *)
-    # Relative path.  Make it absolute.
-    input="`pwd`/$input"
-    ;;
-esac
-
-pairlist=
-while test "$#" -ne 0; do
-  if test "$1" = "--"; then
-    shift
-    break
-  fi
-  pairlist="$pairlist $1"
-  shift
-done
-
-# The program to run.
-prog="$1"
-shift
-# Make any relative path in $prog absolute.
-case "$prog" in
-  [\\/]* | ?:[\\/]*) ;;
-  *[\\/]*) prog="`pwd`/$prog" ;;
-esac
-
-# FIXME: add hostname here for parallel makes that run commands on
-# other machines.  But that might take us over the 14-char limit.
-dirname=ylwrap$$
-do_exit="cd '`pwd`' && rm -rf $dirname > /dev/null 2>&1;"' (exit $ret); exit $ret'
-trap "ret=129; $do_exit" 1
-trap "ret=130; $do_exit" 2
-trap "ret=141; $do_exit" 13
-trap "ret=143; $do_exit" 15
-mkdir $dirname || exit 1
-
-cd $dirname
-
-case $# in
-  0) "$prog" "$input" ;;
-  *) "$prog" "$@" "$input" ;;
-esac
-ret=$?
-
-if test $ret -eq 0; then
-  set X $pairlist
-  shift
-  first=yes
-  # Since DOS filename conventions don't allow two dots,
-  # the DOS version of Bison writes out y_tab.c instead of y.tab.c
-  # and y_tab.h instead of y.tab.h. Test to see if this is the case.
-  y_tab_nodot="no"
-  if test -f y_tab.c || test -f y_tab.h; then
-    y_tab_nodot="yes"
-  fi
-
-  # The directory holding the input.
-  input_dir=`echo "$input" | sed -e 's,\([\\/]\)[^\\/]*$,\1,'`
-  # Quote $INPUT_DIR so we can use it in a regexp.
-  # FIXME: really we should care about more than `.' and `\'.
-  input_rx=`echo "$input_dir" | sed 's,\\\\,\\\\\\\\,g;s,\\.,\\\\.,g'`
-
-  while test "$#" -ne 0; do
-    from="$1"
-    # Handle y_tab.c and y_tab.h output by DOS
-    if test $y_tab_nodot = "yes"; then
-      if test $from = "y.tab.c"; then
-        from="y_tab.c"
-      else
-        if test $from = "y.tab.h"; then
-          from="y_tab.h"
-        fi
-      fi
-    fi
-    if test -f "$from"; then
-      # If $2 is an absolute path name, then just use that,
-      # otherwise prepend `../'.
-      case "$2" in
-        [\\/]* | ?:[\\/]*) target="$2";;
-        *) target="../$2";;
-      esac
-
-      # We do not want to overwrite a header file if it hasn't
-      # changed.  This avoid useless recompilations.  However the
-      # parser itself (the first file) should always be updated,
-      # because it is the destination of the .y.c rule in the
-      # Makefile.  Divert the output of all other files to a temporary
-      # file so we can compare them to existing versions.
-      if test $first = no; then
-        realtarget="$target"
-        target="tmp-`echo $target | sed s/.*[\\/]//g`"
-      fi
-      # Edit out `#line' or `#' directives.
-      #
-      # We don't want the resulting debug information to point at
-      # an absolute srcdir; it is better for it to just mention the
-      # .y file with no path.
-      #
-      # We want to use the real output file name, not yy.lex.c for
-      # instance.
-      #
-      # We want the include guards to be adjusted too.
-      FROM=`echo "$from" | sed \
-            -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'\
-            -e 's/[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]/_/g'`
-      TARGET=`echo "$2" | sed \
-            -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'\
-            -e 's/[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]/_/g'`
-
-      sed -e "/^#/!b" -e "s,$input_rx,," -e "s,$from,$2," \
-          -e "s,$FROM,$TARGET," "$from" >"$target" || ret=$?
-
-      # Check whether header files must be updated.
-      if test $first = no; then
-        if test -f "$realtarget" && cmp -s "$realtarget" "$target"; then
-          echo "$2" is unchanged
-          rm -f "$target"
-        else
-          echo updating "$2"
-          mv -f "$target" "$realtarget"
-        fi
-      fi
-    else
-      # A missing file is only an error for the first file.  This
-      # is a blatant hack to let us support using "yacc -d".  If -d
-      # is not specified, we don't want an error when the header
-      # file is "missing".
-      if test $first = yes; then
-        ret=1
-      fi
-    fi
-    shift
-    shift
-    first=no
-  done
-else
-  ret=$?
-fi
-
-# Remove the directory.
-cd ..
-rm -rf $dirname
-
-exit $ret
-
-# Local Variables:
-# mode: shell-script
-# sh-indentation: 2
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
-# time-stamp-end: "; # UTC"
-# End:

configure.ac

@@ -1,21 +1,37 @@
-AC_INIT([jq], [1.3], [mu@netsoc.tcd.ie],
+m4_define([jq_version], m4_esyscmd_s([git describe --tags --dirty --match 'jq-*'|sed 's/^jq-//']))
+
+AC_INIT([jq], [jq_version], [mu@netsoc.tcd.ie],
              [jq], [http://stedolan.github.com/jq/])
 
 dnl Created autoconf implementation thompson@dtosolutions, 26NOV12
 AC_PREREQ([2.61])
 AC_CONFIG_AUX_DIR([config])
-AM_INIT_AUTOMAKE([parallel-tests foreign])
+AM_INIT_AUTOMAKE([parallel-tests foreign -Wall])
 AM_SILENT_RULES([yes])
+AM_PROG_AR
 AC_PROG_CC
 AC_PROG_CC_STDC
 AC_PROG_CPP_WERROR
 AC_PROG_YACC
-
+AC_OBJEXT
+AC_EXEEXT
+LT_INIT([shared static win32-dll])
 AM_PROG_CC_C_O
 
 dnl couldn't use AM_PROG_LEX as it doesn't support header files like the
 dnl AC_PROG_YACC macros...
 
+dnl check bison version
+AC_MSG_CHECKING([bison version])
+if test "$YACC" != "bison -y"; then
+  AC_MSG_RESULT([not bison])
+elif $YACC --help | grep -- --warnings >/dev/null; then 
+  AC_MSG_RESULT([ok])
+else 
+  AC_MSG_RESULT([no])
+  AC_MSG_ERROR([You need bison version 2.5 or more recent.])
+fi
+
 dnl
 dnl these program checks should probably be deleted
 dnl
@@ -39,10 +55,13 @@ AC_ARG_ENABLE([docs],
 
 AS_IF([test "x$enable_docs" != "xno"],[
    AC_CHECK_PROGS(bundle_cmd, bundle)
-   AS_IF([test "x$bundle_cmd" = "x" || \
-      ! ( cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null )],[
-         AC_MSG_WARN([no bundler])
-         cat <<EOF
+
+   AC_CACHE_CHECK([for Ruby dependencies], [jq_cv_ruby_deps],
+     [jq_cv_ruby_deps=yes;
+        AS_IF([test "x$bundle_cmd" = "x" || \
+        ! bmsg="`cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null`"],[
+           AC_MSG_WARN([$bmsg])
+           cat <<EOF
 *****************************************************************
 *  Ruby dependencies for building jq documentation not found.   *
 *  You can still build, install and hack on jq, but the manpage *
@@ -50,16 +69,44 @@ AS_IF([test "x$enable_docs" != "xno"],[
 *  See docs/README.md for how to install the docs dependencies. *
 *****************************************************************
 EOF
-         enable_docs=no
-   ])
+           jq_cv_ruby_deps=no
+     ])])
+
+   if test "x$jq_cv_ruby_deps" != "xyes"; then
+     enable_docs=no
+   fi
 ])
 
 AM_CONDITIONAL([ENABLE_DOCS], [test "x$enable_docs" != xno])
+
+AC_ARG_ENABLE([pthread-tls],
+              [AC_HELP_STRING([--enable-pthread-tls],
+                              [Enable use of pthread thread local storage])],
+              [],
+              [enable_pthread_tls=no])
+
+if test $enable_pthread_tls = yes; then
+    AC_FIND_FUNC([pthread_key_create], [pthread], [#include <pthread.h>], [NULL, NULL])
+    AC_FIND_FUNC([pthread_once], [pthread], [#include <pthread.h>], [NULL, NULL])
+    AC_FIND_FUNC([atexit], [pthread], [#include <stdlib.h>], [NULL])
+fi
+
+dnl Thread local storage
+have___thread=no
+AC_MSG_CHECKING(for thread-local storage)
+AC_LINK_IFELSE([AC_LANG_SOURCE([
+static __thread int x ;
+int main () { x = 123; return x; }
+])], have___thread=yes)
+if test $have___thread = yes; then 
+   AC_DEFINE([HAVE___THREAD],1,[Define to 1 if the system supports __thread])
+fi
+AC_MSG_RESULT($have___thread)
+
+
 AC_SUBST([BUNDLER], ["$bundle_cmd"])
 
-dnl AC_CONFIG_MACRO_DIR([m4])
-AC_CONFIG_HEADERS(config.h)
-AM_INIT_AUTOMAKE([-Wall])
+AC_CONFIG_MACRO_DIR([config/m4])
 AC_CONFIG_FILES([Makefile])
 AC_OUTPUT
 

docs/Rakefile

@@ -75,7 +75,7 @@ $BINARIES = {
 
 $BINARIES.each do |name, args|
   file "../build/#{name}" do |t|
-    sh "MAKEFLAGS=-j4 ../scripts/crosscompile #{name} #{args}"
+    sh "../scripts/crosscompile -j4 #{name} #{args}"
   end
   task :binaries => ["../build/#{name}"]
 end
@@ -91,10 +91,6 @@ def load_manual
   YAML::load(File.open("content/3.manual/manual.yml"))
 end
 
-def jq_version
-  YAML::load(File.open("site.yml"))["jq_version"]
-end
-
 task :manpage do
   Tempfile.open "manpage" do |f|
     manual = load_manual
@@ -141,14 +137,10 @@ task :mantests do
   end
 end
 
-$TARBALL = "../jq-#{jq_version}.tar.gz"
-file $TARBALL do
-  sh "cd ..; ./configure && make dist && make distclean"  
-end
-
 directory "output/download/source"
-task :tarball => ["output/download/source", $TARBALL] do
-  sh "cp #{$TARBALL} output/download/source"
+task :tarball => ["output/download/source"] do
+  sh "cd ..; ./configure && make dist && make distclean"
+  sh "cp ../jq-*.tar.gz output/download/source"
 end
 
 task :dist => [:build, :binaries, :tarball]

docs/content/1.tutorial/default.yml

@@ -2,123 +2,205 @@ headline: Tutorial
 body:
   - text: |
 
-      Twitter has a JSON API, so let's play with that. This URL gets
-      us the last 5 tweets about JSON:
-      
-  - command: "curl 'http://search.twitter.com/search.json?q=json&rpp=5&include_entities=true'"
+      GitHub has a JSON API, so let's play with that. This URL gets
+      us the last 5 commits from the jq repo:
+
+  - command: "curl 'https://api.github.com/repos/stedolan/jq/commits?per_page=5'"
     result: |
-      {"completed_in":0.108,"max_id":247677287108067328,"max_id_str":"247677287108067328","next_page":"?page=2&max_id=247677287108067328&q=json&rpp=5&include_entities=1","page":1,"query":"json","refresh_url":"?since_id=247677287108067328&q=json&include_entities=1","results":[{"created_at":"Mon, 17 Sep 2012 12:44:01 +0000","entities":{"hashtags":[],"urls":[{"url":"http:\/\/t.co\/XRvh1ZVw","expanded_url":"http:\/\/jase.im\/Ri7I0M","display_url":"jase.im\/Ri7I0M","indices":[112,132]}],"user_mentions":[{"screen_name":"imagemechanics","name":"Jason Cotterell","id":57271393,"id_str":"57271393","indices":[3,18]}]},"from_user":"_AaronNg","from_user_id":79771704,"from_user_id_str":"79771704","from_user_name":"NgChenChong","geo":null,"id":247677287108067328,"id_str":"247677287108067328","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/2523558403\/ek8mo4j4beq84iw28gjl_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/2523558403\/ek8mo4j4beq84iw28gjl_normal.jpeg","source":"<a href="http:\/\/twitter.com\/">web<\/a>","text":"RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas>.\nhttp:\/\/t.co\/XRvh1ZVw","to_user":null,"to_user_id":0,"to_user_id_str":"0","to_user_name":null}, ...
+      [
+        {
+          "sha": "d25341478381063d1c76e81b3a52e0592a7c997f",
+          "commit": {
+            "author": {
+              "name": "Stephen Dolan",
+              "email": "mu@netsoc.tcd.ie",
+              "date": "2013-06-22T16:30:59Z"
+            },
+            "committer": {
+              "name": "Stephen Dolan",
+              "email": "mu@netsoc.tcd.ie",
+              "date": "2013-06-22T16:30:59Z"
+            },
+            "message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161",
+            "tree": {
+              "sha": "6ab697a8dfb5a96e124666bf6d6213822599fb40",
+              "url": "https://api.github.com/repos/stedolan/jq/git/trees/6ab697a8dfb5a96e124666bf6d6213822599fb40"
+            },
+            "url": "https://api.github.com/repos/stedolan/jq/git/commits/d25341478381063d1c76e81b3a52e0592a7c997f",
+            "comment_count": 0
+          },
+          "url": "https://api.github.com/repos/stedolan/jq/commits/d25341478381063d1c76e81b3a52e0592a7c997f",
+          "html_url": "https://github.com/stedolan/jq/commit/d25341478381063d1c76e81b3a52e0592a7c997f",
+          "comments_url": "https://api.github.com/repos/stedolan/jq/commits/d25341478381063d1c76e81b3a52e0592a7c997f/comments",
+          "author": {
+            "login": "stedolan",
+          ...
+
+  - text: |
+      Github returns a ton of info here, and it's relatively hard to
+      read as it is. To syntax highlight it and order the attributes we pipe
+      it through jq, telling jq to just spit the input back at us using
+      the expression `.`:
+
+  - command: "curl 'https://api.github.com/repos/stedolan/jq/commits?per_page=5' | jq '.'"
+    result: |
+      [
+        {
+          "parents": [
+            {
+              "html_url": "https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
+              "url": "https://api.github.com/repos/stedolan/jq/commits/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
+              "sha": "54b9c9bdb225af5d886466d72f47eafc51acb4f7"
+            },
+            {
+              "html_url": "https://github.com/stedolan/jq/commit/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
+              "url": "https://api.github.com/repos/stedolan/jq/commits/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
+              "sha": "8b1b503609c161fea4b003a7179b3fbb2dd4345a"
+            }
+          ],
+          "committer": {
+            "type": "User",
+            "received_events_url": "https://api.github.com/users/stedolan/received_events",
+            "events_url": "https://api.github.com/users/stedolan/events{/privacy}",
+            "repos_url": "https://api.github.com/users/stedolan/repos",
+            "organizations_url": "https://api.github.com/users/stedolan/orgs",
+      ...
 
   - text: |
 
-      There's lots of info and no whitespace, so to make it a bit more
-      legible we pipe it through jq, telling jq to just spit the input
-      back at us using the expression `.`:
+      There's still far too much info for our purposes, so we can try
+      just grab the first commit:
 
-    
-  - command: "curl 'http://search.twitter.com/search.json?q=json&rpp=5&include_entities=true' | jq '.'"
+
+  - command: "curl 'https://api.github.com/repos/stedolan/jq/commits?per_page=5' | jq '.[0]'"
     result: |
-
       {
-        "since_id_str": "0",
-        "since_id": 0,
-        "results_per_page": 5,
-        "completed_in": 0.108,
-        "max_id": 247677287108067330,
-        "max_id_str": "247677287108067328",
-        "next_page": "?page=2&max_id=247677287108067328&q=json&rpp=5&include_entities=1",
-        "page": 1,
-        "query": "json",
-        "refresh_url": "?since_id=247677287108067328&q=json&include_entities=1",
-        "results": [
+        "parents": [
           {
-            "from_user_name": "NgChenChong",
-            "from_user_id_str": "79771704",
-            "from_user_id": 79771704,
-            "from_user": "_AaronNg",
-            "iso_language_code": "en",
-            "text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas>.\nhttp://t.co/XRvh1ZVw",
-            "to_user": null
-            /* lots more fields... */
+            "html_url": "https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
+            "url": "https://api.github.com/repos/stedolan/jq/commits/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
+            "sha": "54b9c9bdb225af5d886466d72f47eafc51acb4f7"
           },
-          /* lots more results... */
-        ]
+          {
+            "html_url": "https://github.com/stedolan/jq/commit/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
+            "url": "https://api.github.com/repos/stedolan/jq/commits/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
+            "sha": "8b1b503609c161fea4b003a7179b3fbb2dd4345a"
+          }
+        ],
+        "committer": {
+          "type": "User",
+          "received_events_url": "https://api.github.com/users/stedolan/received_events",
+          "events_url": "https://api.github.com/users/stedolan/events{/privacy}",
+          "repos_url": "https://api.github.com/users/stedolan/repos",
+          "organizations_url": "https://api.github.com/users/stedolan/orgs",
+          "subscriptions_url": "https://api.github.com/users/stedolan/subscriptions",
+          "starred_url": "https://api.github.com/users/stedolan/starred{/owner}{/repo}",
+          "gists_url": "https://api.github.com/users/stedolan/gists{/gist_id}",
+          "login": "stedolan",
+          "id": 79765,
+          "avatar_url": "https://1.gravatar.com/avatar/31de909d8e55dd07ed782d92ece59842?d=https%3A%2F%2Fidenticons.github.com%2Ffc5b6765b1c9cfaecea48ae71df4d279.png",
+          "gravatar_id": "31de909d8e55dd07ed782d92ece59842",
+          "url": "https://api.github.com/users/stedolan",
+          "html_url": "https://github.com/stedolan",
+          "followers_url": "https://api.github.com/users/stedolan/followers",
+          "following_url": "https://api.github.com/users/stedolan/following{/other_user}"
+        },
+        "author": {
+          "type": "User",
+          "received_events_url": "https://api.github.com/users/stedolan/received_events",
+          "events_url": "https://api.github.com/users/stedolan/events{/privacy}",
+          "repos_url": "https://api.github.com/users/stedolan/repos",
+          "organizations_url": "https://api.github.com/users/stedolan/orgs",
+          "subscriptions_url": "https://api.github.com/users/stedolan/subscriptions",
+          "starred_url": "https://api.github.com/users/stedolan/starred{/owner}{/repo}",
+          "gists_url": "https://api.github.com/users/stedolan/gists{/gist_id}",
+          "login": "stedolan",
+          "id": 79765,
+          "avatar_url": "https://1.gravatar.com/avatar/31de909d8e55dd07ed782d92ece59842?d=https%3A%2F%2Fidenticons.github.com%2Ffc5b6765b1c9cfaecea48ae71df4d279.png",
+          "gravatar_id": "31de909d8e55dd07ed782d92ece59842",
+          "url": "https://api.github.com/users/stedolan",
+          "html_url": "https://github.com/stedolan",
+          "followers_url": "https://api.github.com/users/stedolan/followers",
+          "following_url": "https://api.github.com/users/stedolan/following{/other_user}"
+        },
+        "comments_url": "https://api.github.com/repos/stedolan/jq/commits/d25341478381063d1c76e81b3a52e0592a7c997f/comments",
+        "html_url": "https://github.com/stedolan/jq/commit/d25341478381063d1c76e81b3a52e0592a7c997f",
+        "url": "https://api.github.com/repos/stedolan/jq/commits/d25341478381063d1c76e81b3a52e0592a7c997f",
+        "commit": {
+          "comment_count": 0,
+          "url": "https://api.github.com/repos/stedolan/jq/git/commits/d25341478381063d1c76e81b3a52e0592a7c997f",
+          "tree": {
+            "url": "https://api.github.com/repos/stedolan/jq/git/trees/6ab697a8dfb5a96e124666bf6d6213822599fb40",
+            "sha": "6ab697a8dfb5a96e124666bf6d6213822599fb40"
+          },
+          "message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161",
+          "committer": {
+            "date": "2013-06-22T16:30:59Z",
+            "email": "mu@netsoc.tcd.ie",
+            "name": "Stephen Dolan"
+          },
+          "author": {
+            "date": "2013-06-22T16:30:59Z",
+            "email": "mu@netsoc.tcd.ie",
+            "name": "Stephen Dolan"
+          }
+        },
+        "sha": "d25341478381063d1c76e81b3a52e0592a7c997f"
       }
 
-    
-  - text: |
-          
-      Let's pull out the first tweet:
-          
-          
-  - command: "curl 'http://search.twitter.com/search.json?q=json&rpp=5&include_entities=true' | jq '.results[0]'"
-    result: |
-      {
-        "from_user_name": "NgChenChong",
-        "from_user_id_str": "79771704",
-        "from_user_id": 79771704,
-        "from_user": "_AaronNg",
-        "iso_language_code": "en",
-        "text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas>.\nhttp://t.co/XRvh1ZVw",
-        "to_user": null
-        /* lots more fields... */
-      },
-          
   - text: |
 
       For the rest of the examples, I'll leave out the `curl` command - it's not going to change.
 
-    
       There's still a lot of info we don't care about there, so we'll
       restrict it down to the most interesting fields.
-          
 
-  - command: "jq '.results[0] | {from_user, text}'"
+
+  - command: "jq '.[0] | {message: .commit.message, name: .commit.committer.name}'"
     result: |
       {
-        "text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas>.\nhttp://t.co/XRvh1ZVw",
-        "from_user": "_AaronNg"
+        "name": "Stephen Dolan",
+        "message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161"
       }
-      
+
   - text: |
-    
+
       The `|` operator in jq feeds the output of one filter
-      (`.results[0]` which gets the first element of the results
-      array) into the input of another (`{from_user, text}` which
-      builds an object of those two fields).
-    
-      Now let's get the rest of the tweets:
-    
-          
+      (`.[0]` which gets the first element of the array in the response)
+      into the input of another (`{...}` which
+      builds an object out of those fields). You can access nested attributes,
+      such as `.commit.message`:
 
-  - command: "jq '.results[] | {from_user, text}'"
+      Now let's get the rest of the commits:
+
+
+  - command:  "jq '.[] | {message: .commit.message, name: .commit.committer.name}'"
     result: |
       {
-        "text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas>.\nhttp://t.co/XRvh1ZVw",
-        "from_user": "_AaronNg"
+        "name": "Stephen Dolan",
+        "message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161"
       }
       {
-        "text": "RT @_kud: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas> -- http://t.co/Lhp92IqD",
-        "from_user": "blouerat"
+        "name": "Stephen Dolan",
+        "message": "Reject all overlong UTF8 sequences."
       }
       {
-        "text": "Dynamic Forms Mobile App by pmadiset: Few Forms details are hosted on our server in the form of JSON. This JSON ... http://t.co/7KALdWaX",
-        "from_user": "cebu_iphone"
+        "name": "Stephen Dolan",
+        "message": "Fix various UTF8 parsing bugs.\n\nIn particular, parse bad UTF8 by replacing the broken bits with U+FFFD\nand resychronise correctly after broken sequences."
       }
       {
-        "text": "iPhone 5 website insanity - Video compressing using JPEG, JSON, and <canvas> http://t.co/28Jesbio (Oh #Apple, U So #Awesome...)",
-        "from_user": "dieswaytoofast"
+        "name": "Stephen Dolan",
+        "message": "Fix example in manual for `floor`. See #155."
       }
       {
-        "text": "RT @umutm: A very nice web-based JSON editor - http://t.co/M70snaIf",
-        "from_user": "Leolik"
+        "name": "Nicolas Williams",
+        "message": "Document floor"
       }
-         
-          
+
   - text: |
-    
-      `.results[]` returns each element of the results array, one at a
-      time, which are all fed into `{from_user, text}`.
+
+      `.[]` returns each element of the array returned in the response,
+      one at a time, which are all fed into `{message: .commit.message, name: .commit.committer.name}`.
 
 
       Data in jq is represented as streams of JSON values - every jq
@@ -128,109 +210,108 @@ body:
       Streams are serialised by just separating JSON values with
       whitespace. This is a `cat`-friendly format - you can just join
       two JSON streams together and get a valid JSON stream.
-    
+
       If you want to get the output as a single array, you can tell jq to
       "collect" all of the answers by wrapping the filter in square
       brackets:
 
-  - command: "jq '[.results[] | {from_user, text}]'"
+  - command: "jq '[.[] | {message: .commit.message, name: .commit.committer.name}]'"
     result: |
       [
         {
-          "text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas>.\nhttp://t.co/XRvh1ZVw",
-          "from_user": "_AaronNg"
+          "name": "Stephen Dolan",
+          "message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161"
         },
         {
-          "text": "RT @_kud: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas> -- http://t.co/Lhp92IqD",
-          "from_user": "blouerat"
+          "name": "Stephen Dolan",
+          "message": "Reject all overlong UTF8 sequences."
         },
         {
-          "text": "Dynamic Forms Mobile App by pmadiset: Few Forms details are hosted on our server in the form of JSON. This JSON ... http://t.co/7KALdWaX",
-          "from_user": "cebu_iphone"
+          "name": "Stephen Dolan",
+          "message": "Fix various UTF8 parsing bugs.\n\nIn particular, parse bad UTF8 by replacing the broken bits with U+FFFD\nand resychronise correctly after broken sequences."
         },
         {
-          "text": "iPhone 5 website insanity - Video compressing using JPEG, JSON, and <canvas> http://t.co/28Jesbio (Oh #Apple, U So #Awesome...)",
-          "from_user": "dieswaytoofast"
+          "name": "Stephen Dolan",
+          "message": "Fix example in manual for `floor`. See #155."
         },
         {
-          "text": "RT @umutm: A very nice web-based JSON editor - http://t.co/M70snaIf",
-          "from_user": "Leolik"
+          "name": "Nicolas Williams",
+          "message": "Document floor"
         }
       ]
-          
-          
   - text: |
 
       - - -
-    
-      Next, let's try getting the URLs out of those API results as
-      well. In each tweet, the Twitter API includes a field called
-      "entities" which looks like this:
 
-          {
-            "user_mentions": [],
-            "urls": [{
-                "indices": [83, 103],
-                "display_url": "bit.ly/StniqT",
-                "expanded_url": "http://bit.ly/StniqT",
-                "url": "http://t.co/28Jesbio"
-            }],
-            "hashtags": [
-              {"indices": [108, 114], "text": "Apple" },
-              {"indices": [121, 129], "text": "Awesome"}
-            ]
-          }
+      Next, let's try getting the URLs of the parrent commits out of the
+      API results as well. In each commit, the GitHub API includes information
+      about "parent" commits. There can be one or many.
 
-      We want to pull out all of the "url" fields inside that array of url
-      objects, and make a simple list of strings to go along with the
-      "from_user" and "text" fields we already have.
-    
-          
+          "parents": [
+            {
+              "html_url": "https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
+              "url": "https://api.github.com/repos/stedolan/jq/commits/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
+              "sha": "54b9c9bdb225af5d886466d72f47eafc51acb4f7"
+            },
+            {
+              "html_url": "https://github.com/stedolan/jq/commit/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
+              "url": "https://api.github.com/repos/stedolan/jq/commits/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
+              "sha": "8b1b503609c161fea4b003a7179b3fbb2dd4345a"
+            }
+          ]
 
-  - command: "jq '.results[] | {from_user, text, urls: [.entities.urls[].url]}'"
+      We want to pull out all of the "html_url" fields inside that array of parent
+      commits and make a simple list of strings to go along with the
+      "message" and "author" fields we already have.
+
+
+
+  - command: "jq '[.[] | {message: .commit.message, name: .commit.committer.name, parents: [.parents[].html_url]}]'"
     result: |
-      {
-       "urls": [
-         "http://t.co/XRvh1ZVw"
-       ],
-       "text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas>.\nhttp://t.co/XRvh1ZVw",
-        "from_user": "_AaronNg"
-      }
-      {
-        "urls": [
-          "http://t.co/Lhp92IqD"
-        ],
-        "text": "RT @_kud: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and <canvas> -- http://t.co/Lhp92IqD",
-        "from_user": "blouerat"
-      }
-      {
-        "urls": [
-          "http://t.co/7KALdWaX"
-        ],
-        "text": "Dynamic Forms Mobile App by pmadiset: Few Forms details are hosted on our server in the form of JSON. This JSON ... http://t.co/7KALdWaX",
-        "from_user": "cebu_iphone"
-      }
-      {
-        "urls": [
-          "http://t.co/28Jesbio"
-        ],
-        "text": "iPhone 5 website insanity - Video compressing using JPEG, JSON, and <canvas> http://t.co/28Jesbio (Oh #Apple, U So #Awesome...)",
-        "from_user": "dieswaytoofast"
-      }
-      {
-        "urls": [
-          "http://t.co/M70snaIf"
-        ],
-        "text": "RT @umutm: A very nice web-based JSON editor - http://t.co/M70snaIf",
-        "from_user": "Leolik"
-      }
-    
-          
+      [
+        {
+          "parents": [
+            "https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
+            "https://github.com/stedolan/jq/commit/8b1b503609c161fea4b003a7179b3fbb2dd4345a"
+          ],
+          "name": "Stephen Dolan",
+          "message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161"
+        },
+        {
+          "parents": [
+            "https://github.com/stedolan/jq/commit/ff48bd6ec538b01d1057be8e93b94eef6914e9ef"
+          ],
+          "name": "Stephen Dolan",
+          "message": "Reject all overlong UTF8 sequences."
+        },
+        {
+          "parents": [
+            "https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7"
+          ],
+          "name": "Stephen Dolan",
+          "message": "Fix various UTF8 parsing bugs.\n\nIn particular, parse bad UTF8 by replacing the broken bits with U+FFFD\nand resychronise correctly after broken sequences."
+        },
+        {
+          "parents": [
+            "https://github.com/stedolan/jq/commit/3dcdc582ea993afea3f5503a78a77675967ecdfa"
+          ],
+          "name": "Stephen Dolan",
+          "message": "Fix example in manual for `floor`. See #155."
+        },
+        {
+          "parents": [
+            "https://github.com/stedolan/jq/commit/7c4171d414f647ab08bcd20c76a4d8ed68d9c602"
+          ],
+          "name": "Nicolas Williams",
+          "message": "Document floor"
+        }
+      ]
+
   - text: |
-    
-      Here we're making an object as before, but this time the urls
-      field is being set to `[.entities.urls[].url]`, which collects
-      all of the URLs defined in the entities object.
+
+      Here we're making an object as before, but this time the `parents`
+      field is being set to `[.parents[].html_url]}]`, which collects
+      all of the parent commit urls defined in the parents object.
 
   - text: |
 
@@ -238,4 +319,4 @@ body:
 
       Here endeth the tutorial! There's lots more to play with, go
       read [the manual](../manual/) if you're interested, and [download
-      jq](../download/) if you haven't already.
+      jq](../download/) if you haven't already.

docs/content/2.download/default.yml

@@ -31,7 +31,7 @@ body:
 
       ### OS X
 
-       * `brew install jq` using [homebrew](http://mxcl.github.io/homebrew/)
+       * `brew install jq` using [homebrew](http://mxcl.github.com/homebrew/)
 
        * Or, grab prebuilt [64-bit binaries](osx64/jq) or [32-bit
          binaries](osx32/jq)
@@ -47,11 +47,11 @@ body:
       You can build it using the usual `./configure && make && sudo
       make install` rigmarole.
 
-      If you're interested in using the latest development version, try:
+      If you're interested in using the lastest development version, try:
 
           git clone https://github.com/stedolan/jq.git
           cd jq
-          autoreconf
+          autoreconf -i
           ./configure
           make
           sudo make install
@@ -72,7 +72,7 @@ body:
       On OS X, these are all included in Apple's command line tools, which
       can be installed from [Xcode](http://developer.apple.com/technologies/tools/).
       However, you may find that you need a newer version of Bison than the one provided
-      by Apple. This can be found in [Homebrew](http://mxcl.github.io/homebrew/)
+      by Apple. This can be found in [Homebrew](http://mxcl.github.com/homebrew/)
       or [MacPorts](http://macports.org/).
 
       `flex` and `bison` are used to generate the lexer and parser for

docs/content/3.manual/manual.yml

@@ -72,14 +72,35 @@ sections:
       output(s) of the filter are written to standard out, again as a
       sequence of whitespace-separated JSON data.
 
+      Note: it is important to mind the shell's quoting rules.  As a
+      general rule it's best to always quote (with single-quote
+      characters) the jq program, as too many characters with special
+      meaning to jq are also shell meta-characters.  For example, `jq
+      "foo"` will fail on most Unix shells because that will be the same
+      as `jq foo`, which will generally fail because `foo is not
+      defined`.  When using the Windows command shell (cmd.exe) it's
+      best to use double quotes around your jq program when given on the
+      command-line (instead of the `-f program-file` option), but then
+      double-quotes in the jq program need backslash escaping.
+
       You can affect how jq reads and writes its input and output
       using some command-line options:
 
+      * `--version`:
+
+        Output the jq version and exit with zero.
+
       * `--slurp`/`-s`:
   
         Instead of running the filter for each JSON object in the
         input, read the entire input stream into a large array and run
         the filter just once.
+
+      * `--online-input`/`-I`:
+  
+        When the top-level input value is an array produce its elements
+        instead of the array.  This allows on-line processing of
+        potentially very large top-level arrays' elements.
       
       * `--raw-input`/`-R`:
       
@@ -113,7 +134,17 @@ sections:
         "\u03bc"). Using this option, you can force jq to produce pure
         ASCII output with every non-ASCII character replaced with the
         equivalent escape sequence.
-  
+
+      * `--unbuffered`
+
+        Flush the output after each JSON object is printed (useful if
+        you're piping a slow data source into jq and piping jq's
+        output elsewhere).
+
+      * `--sort-keys` / `-S`:
+        
+        Output the fields of each object with the keys in sorted order.
+
       * `--raw-output` / `-r`:
   
         With this option, if the filter's result is a string then it
@@ -121,12 +152,34 @@ sections:
         formatted as a JSON string with quotes. This can be useful for
         making jq filters talk to non-JSON-based systems.
 
+      * `-f filename` / `--from-file filename`:
+
+        Read filter from the file rather than from a command line, like
+        awk's -f option. You can also use '#' to make comments.
+
+      * `-e` / `--exit-status`:
+
+        Sets the exit status of jq to 0 if the last output values was
+        neither `false` nor `null`, 1 if the last output value was
+        either `false` or `null`, or 4 if no valid result was ever
+        produced.  Normally jq exits with 2 if there was any usage
+        problem or system error, 3 if there was a jq program compile
+        error, or 0 if the jq program ran.
+
       * `--arg name value`:
 
         This option passes a value to the jq program as a predefined
         variable. If you run jq with `--arg foo bar`, then `$foo` is
         available in the program and has the value `"bar"`.
 
+      * `--argfile name filename`:
+
+        This option passes the first value from the named file as a
+        value to the jq program as a predefined variable. If you run jq
+        with `--argfile foo bar`, then `$foo` is available in the
+        program and has the value resulting from parsing the content of
+        the file named `bar`.
+
   - title: Basic filters
     entries:
       - title: "`.`"
@@ -145,13 +198,18 @@ sections:
             input: '"Hello, world!"'
             output: ['"Hello, world!"']
 
-      - title: "`.foo`"
+      - title: "`.foo`, `.foo.bar`"
         body: |
           
-          The simplest *useful* filter is .foo. When given a
+          The simplest *useful* filter is `.foo`. When given a
           JSON object (aka dictionary or hash) as input, it produces
           the value at the key "foo", or null if there's none present.
 
+          If the key contains special characters, you need to surround
+          it with double quotes like this: `."foo$"`.
+
+          A filter of the form `.foo.bar` is equivalent to `.foo|.bar`.
+          
         examples:
           - program: '.foo'
             input: '{"foo": 42, "bar": "less interesting data"}'
@@ -159,8 +217,31 @@ sections:
           - program: '.foo'
             input: '{"notfoo": true, "alsonotfoo": false}'
             output: ['null']
+          - program: '.["foo"]'
+            input: '{"foo": 42}'
+            output: [42]
 
-      - title: "`.[foo]`, `.[2]`, `.[10:15]`"
+      - title: "`.foo?`"
+        body: |
+          
+          Just like `.foo`, but does not output even an error when `.`
+          is not an array or an object.
+
+        examples:
+          - program: '.foo?'
+            input: '{"foo": 42, "bar": "less interesting data"}'
+            output: [42]
+          - program: '.foo?'
+            input: '{"notfoo": true, "alsonotfoo": false}'
+            output: ['null']
+          - program: '.["foo"]?'
+            input: '{"foo": 42}'
+            output: [42]
+          - program: '[.foo?]'
+            input: '[1,2]'
+            output: ['[]']
+
+      - title: "`.[<string>]`, `.[2]`, `.[10:15]`"
         body: |
           
           You can also look up fields of an object using syntax like
@@ -170,11 +251,16 @@ sections:
           returns the third element of the array.
 
           The `.[10:15]` syntax can be used to return a subarray of an
-          array. The array returned by `.[10:15]` will be of length 5,
-          containing the elements from index 10 (inclusive) to index
-          15 (exclusive). Either index may be negative (in which case
-          it counts backwards from the end of the array), or omitted
-          (in which case it refers to the start or end of the array).
+          array or substring of a string. The array returned by
+          `.[10:15]` will be of length 5, containing the elements from
+          index 10 (inclusive) to index 15 (exclusive). Either index may
+          be negative (in which case it counts backwards from the end of
+          the array), or omitted (in which case it refers to the start
+          or end of the array).
+
+          The `?` "operator" can also be used with the slice operator,
+          as in `.[10:15]?`, which outputs values where the inputs are
+          slice-able.
 
         examples:
           - program: '.[0]'
@@ -188,6 +274,10 @@ sections:
           - program: '.[2:4]'
             input: '["a","b","c","d","e"]'
             output: ['["c", "d"]']
+
+          - program: '.[2:4]'
+            input: '"abcdefghi"'
+            output: ['"cd"']
           
           - program: '.[:3]'
             input: '["a","b","c","d","e"]'
@@ -200,7 +290,7 @@ sections:
       - title: "`.[]`"
         body: |
           
-          If you use the `.[foo]` syntax, but omit the index
+          If you use the `.[index]` syntax, but omit the index
           entirely, it will return *all* of the elements of an
           array. Running `.[]` with the input `[1,2,3]` will produce the
           numbers as three separate results, rather than as a single
@@ -224,6 +314,12 @@ sections:
             input: '{"a": 1, "b": 1}'
             output: ['1', '1']
 
+      - title: "`.[]?`"
+        body: |
+          
+          Like `.[]`, but no errors will be output if . is not an array
+          or object.
+
       - title: "`,`"
         body: |
           
@@ -387,7 +483,8 @@ sections:
           - **Objects** are added by merging, that is, inserting all
               the key-value pairs from both objects into a single
               combined object. If both objects contain a value for the
-              same key, the object on the right of the `+` wins.
+              same key, the object on the right of the `+` wins. (For
+              recursive merge use the `*` operator.)
           
           `null` can be added to any value, and returns the other
           value unchanged.
@@ -424,17 +521,34 @@ sections:
             input: '["xml", "yaml", "json"]'
             output: ['["json"]']
 
-      - title: Multiplication, division - `*` and `/`
+      - title: Multiplication, division, modulo - `*`, `/`, and `%`
         body: |
 
           These operators only work on numbers, and do the expected.
 
+          Multiplying a string by a number produces the concatenation of
+          that string that many times.
+
+          Dividing a string by another splits the first using the second
+          as separators.
+
+          Multiplying two objects will merge them recursively: this works
+          like addition but if both objects contain a value for the
+          same key, and the values are objects, the two are merged with
+          the same strategy.
+
         examples:
           - program: '10 / . * 3'
             input: 5
             output: [6]
+          - program: '. / ", "'
+            input: '"a, b,c,d, e"'
+            output: ['["a","b,c,d","e"]']
+          - program: '{"k": {"a": 1, "b": 2}} * {"k": {"a": 0,"c": 3}}'
+            input: 'null'
+            output: ['{"k": {"a": 0, "b": 2, "c": 3}}']
 
-      - title: `length`
+      - title: "`length`"
         body: |
 
           The builtin function `length` gets the length of various
@@ -455,7 +569,7 @@ sections:
             input: '[[1,2], "string", {"a":2}, null]'
             output: [2, 6, 1, 0]
 
-      - title: `keys`
+      - title: "`keys`"
         body: |
           
           The builtin function `keys`, when given an object, returns
@@ -478,7 +592,7 @@ sections:
             input: '[42,3,35]'
             output: ['[0,1,2]']
 
-      - title: `has`
+      - title: "`has`"
         body: |
           
           The builtin function `has` returns whether the input object
@@ -497,7 +611,21 @@ sections:
             input: '[[0,1], ["a","b","c"]]'
             output: ['[false, true]']
 
-      - title: `to_entries`, `from_entries`, `with_entries`
+      - title: "`del`"
+        body: |
+          
+          The builtin function `del` removes a key and its corresponding
+          value from an object.
+        
+        examples:
+          - program: 'del(.foo)'
+            input: '[{"foo": 42, "bar": 9001, "baz": 42}]'
+            output: ['{"bar": 9001, "baz": 42}']
+          - program: 'del(.[1, 2])'
+            input: '[["foo", "bar", "baz"]]'
+            output: ['["foo"]']
+
+      - title: "`to_entries`, `from_entries`, `with_entries`"
         body: |
           
           These functions convert between an object and an array of
@@ -522,7 +650,7 @@ sections:
             output: ['{"KEY_a": 1, "KEY_b": 2}']
 
 
-      - title: `select`
+      - title: "`select`"
         body: |
           
           The function `select(foo)` produces its input unchanged if
@@ -530,14 +658,27 @@ sections:
           otherwise.
 
           It's useful for filtering lists: `[1,2,3] | map(select(. >= 2))`
-          will give you `[3]`.
+          will give you `[2,3]`.
 
         examples:
           - program: 'map(select(. >= 2))'
             input: '[1,5,3,0,7]'
             output: ['[5,3,7]']
 
-      - title: `empty`
+
+      - title: "`arrays`, `objects`, `iterables`, `booleans`, `numbers`, `strings`, `nulls`, `values`, `scalars`"
+        body: |
+          
+          These built-ins select only inputs that are arrays, objects,
+          iterables (arrays or objects), booleans, numbers, strings,
+          null, non-null values, and non-iterables, respectively.
+
+        examples:
+          - program: '.[]|numbers'
+            input: '[[],{},1,"foo",null,true,false]'
+            output: ['1']
+
+      - title: "`empty`"
         body: |
           
           `empty` returns no results. None at all. Not even `null`.
@@ -552,7 +693,7 @@ sections:
             input: 'null'
             output: ['[1,2,3]']
 
-      - title: `map(x)`
+      - title: "`map(x)`"
         body: |
 
           For any filter `x`, `map(x)` will run that filter for each
@@ -567,7 +708,33 @@ sections:
             input: '[1,2,3]'
             output: ['[2,3,4]']
 
-      - title: `add`
+      - title: "`paths`"
+        body: |
+
+          Outputs the paths to all the elements in its input (except it
+          does not output the empty list, representing . itself).
+
+          `paths` is equivalent to 
+
+              def paths: path(recurse(if (type|. == "array" or . == "object") then .[] else empty end))|select(length > 0);
+
+        examples:
+          - program: '[paths]'
+            input: '[1,[[],{"a":2}]]'
+            output: ['[[0],[1],[1,0],[1,1],[1,1,"a"]]']
+
+      - title: "`leaf_paths`"
+        body: |
+
+          Outputs the paths to all the leaves (non-array, non-object
+          elements) in its input.
+
+        examples:
+          - program: '[leaf_paths]'
+            input: '[1,[[],{"a":2}]]'
+            output: ['[[0],[1,1,"a"]]']
+
+      - title: "`add`"
         body: |
 
           The filter `add` takes as input an array, and produces as
@@ -589,7 +756,47 @@ sections:
             input: '[]'
             output: ["null"]
 
-      - title: `range`
+      - title: "`any`"
+        body: |
+      
+          The filter `any` takes as input an array of boolean values,
+          and produces `true` as output if any of the the elements of
+          the array is `true`.
+        
+          If the input is an empty array, `any` returns `false`.
+        
+        examples:
+          - program: any
+            input: '[true, false]'
+            output: ["true"]
+          - program: any
+            input: '[false, false]'
+            output: ["false"]
+          - program: any
+            input: '[]'
+            output: ["false"]
+
+      - title: "`all`"
+        body: |
+      
+          The filter `all` takes as input an array of boolean values,
+          and produces `true` as output if all of the the elements of
+          the array are `true`.
+        
+          If the input is an empty array, `all` returns `true`.
+        
+        examples:
+          - program: all
+            input: '[true, false]'
+            output: ["false"]
+          - program: all
+            input: '[true, true]'
+            output: ["true"]
+          - program: all
+            input: '[]'
+            output: ["true"]
+
+      - title: "`range`"
         body: |
           
           The `range` function produces a range of numbers. `range(4;10)`
@@ -605,7 +812,27 @@ sections:
             input: 'null'
             output: ['[2,3]']
             
-      - title: `tonumber`
+      - title: "`floor`"
+        body: |
+
+          The `floor` function returns the floor of its numeric input.
+
+        examples:
+          - program: 'floor'
+            input: '3.14159'
+            output: ['3']
+            
+      - title: "`sqrt`"
+        body: |
+
+          The `sqrt` function returns the square root of its numeric input.
+
+        examples:
+          - program: 'sqrt'
+            input: '9'
+            output: ['3']
+            
+      - title: "`tonumber`"
         body: |
 
           The `tonumber` function parses its input as a number. It
@@ -617,7 +844,7 @@ sections:
             input: '[1, "1"]'
             output: [1, 1]
 
-      - title: `tostring`
+      - title: "`tostring`"
         body: |
 
           The `tostring` function prints its input as a
@@ -629,7 +856,7 @@ sections:
             input: '[1, "1", [1]]'
             output: ['"1"', '"1"', '"[1]"']
 
-      - title: `type`
+      - title: "`type`"
         body: |
           
           The `type` function returns the type of its argument as a
@@ -641,7 +868,7 @@ sections:
             input: '[0, false, [], {}, null, "hello"]'
             output: ['["number", "boolean", "array", "object", "null", "string"]']
 
-      - title: `sort, sort_by`
+      - title: "`sort, sort_by`"
         body: |
           
           The `sort` functions sorts its input, which must be an
@@ -673,7 +900,7 @@ sections:
             input: '[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}]'
             output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}]']
 
-      - title: `group_by`
+      - title: "`group_by`"
         body: |
           
           `group_by(.foo)` takes as input an array, groups the
@@ -690,7 +917,7 @@ sections:
             input: '[{"foo":1, "bar":10}, {"foo":3, "bar":100}, {"foo":1, "bar":1}]'
             output: ['[[{"foo":1, "bar":10}, {"foo":1, "bar":1}], [{"foo":3, "bar":100}]]']
 
-      - title: `min`, `max`, `min_by`, `max_by`
+      - title: "`min`, `max`, `min_by`, `max_by`"
         body: |
           
           Find the minimum or maximum element of the input array. The
@@ -706,7 +933,7 @@ sections:
             input: '[{"foo":1, "bar":14}, {"foo":2, "bar":3}]'
             output: ['{"foo":2, "bar":3}']
 
-      - title: `unique`
+      - title: "`unique`"
         body: |
           
           The `unique` function takes as input an array and produces
@@ -718,7 +945,25 @@ sections:
             input: '[1,2,5,3,5,3,1,3]'
             output: ['[1,2,3,5]']
 
-      - title: `reverse`
+      - title: "`unique_by`"
+        body: |
+          
+          The `unique_by(.foo)` function takes as input an array and produces
+          an array of the same elements, in sorted order, with
+          elqements with a duplicate `.foo` field removed. Think of it as making
+          an array by taking one element out of every group produced by
+          `group_by`.
+        
+        examples:
+          - program: 'unique_by(.foo)'
+            input: '[{"foo": 1, "bar": 2}, {"foo": 1, "bar": 3}, {"foo": 4, "bar": 5}]'
+            output: ['[{"foo": 1, "bar": 2}, {"foo": 4, "bar": 5}]']
+          - program: 'unique_by(length)'
+            input: '["chunky", "bacon", "kitten", "cicada", "asparagus"]'
+            output: ['["chunky", "bacon", "asparagus"]']
+
+
+      - title: "`reverse`"
         body: |
           
           This function reverses an array.
@@ -728,7 +973,7 @@ sections:
             input: '[1,2,3,4]'
             output: ['[4,3,2,1]']
 
-      - title: `contains`
+      - title: "`contains`"
         body: |
 
           The filter `contains(b)` will produce true if b is
@@ -757,7 +1002,127 @@ sections:
             input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
             output: ['false']
 
-      - title: `recurse`
+      - title: "`indices(s)`"
+        body: |
+
+          Outputs an array containing the indices in `.` where `s`
+          occurs.  The input may be an array, in which case if `s` is an
+          array then the indices output will be those where all elements
+          in `.` match those of `s`.
+
+        examples:
+          - program: 'indices(", ")'
+            input: '"a,b, cd, efg, hijk"'
+            output: ['[3,7,12]']
+          - program: 'indices(1)'
+            input: '[0,1,2,1,3,1,4]'
+            output: ['[1,3,5]']
+          - program: 'indices([1,2])'
+            input: '[0,1,2,3,1,4,2,5,1,2,6,7]'
+            output: ['[1,8]']
+
+      - title: "`index(s)`, `rindex(s)`"
+        body: |
+
+          Outputs the index of the first (`index`) or last (`rindex`)
+          occurrence of `s` in the input.
+
+        examples:
+          - program: 'index(", ")'
+            input: '"a,b, cd, efg, hijk"'
+            output: ['3']
+          - program: 'rindex(", ")]'
+            input: '"a,b, cd, efg, hijk"'
+            output: ['12']
+
+      - title: "`startswith`"
+        body: |
+
+          Outputs `true` if . starts with the given string argument.
+
+        examples:
+          - program: '[.[]|startswith("foo")]'
+            input: '["fo", "foo", "barfoo", "foobar", "barfoob"]'
+            output: ['[false, true, false, true, false]']
+
+      - title: "`endswith`"
+        body: |
+
+          Outputs `true` if . ends with the given string argument.
+
+        examples:
+          - program: '[.[]|endswith("foo")]'
+            input: '["foobar", "barfoo"]'
+            output: ['[false, true, true, false, false]']
+
+      - title: "`ltrimstr`"
+        body: |
+
+          Outputs its input with the given prefix string removed, if it
+          starts with it.
+
+        examples:
+          - program: '[.[]|ltrimstr("foo")]'
+            input: '["fo", "foo", "barfoo", "foobar", "afoo"]'
+            output: ['["fo","","barfoo","bar","afoo"]']
+
+      - title: "`rtrimstr`"
+        body: |
+
+          Outputs its input with the given suffix string removed, if it
+          starts with it.
+
+        examples:
+          - program: '[.[]|rtrimstr("foo")]'
+            input: '["fo", "foo", "barfoo", "foobar", "foob"]'
+            output: ['["fo","","bar","foobar","foob"]']
+
+      - title: "`explode`"
+        body: |
+
+          Converts an input string into an array of the string's
+          codepoint numbers.
+
+        examples:
+          - program: 'explode'
+            input: '"foobar"'
+            output: ['[102,111,111,98,97,114]']
+
+      - title: "`implode`"
+        body: |
+
+          The inverse of explode.
+
+        examples:
+          - program: 'implode'
+            input: '[65, 66, 67]'
+            output: ['"ABC"']
+
+      - title: "`split`"
+        body: |
+
+          Splits an input string on the separator argument.
+
+        examples:
+          - program: 'split(", ")'
+            input: '"a, b,c,d, e"'
+            output: ['["a","b,c,d","e"]']
+
+      - title: "`join`"
+        body: |
+        
+          Joins the array of elements given as input, using the
+          argument as separator. It is the inverse of `split`: that is,
+          running `split("foo") | join("foo")` over any input string
+          returns said input string.
+          
+        examples:
+          - program: 'join(", ")'
+            input: '["a","b,c,d","e"]'
+            output: ['"a, b,c,d, e"']
+
+
+      - title: "`recurse`"
         body: |
           
           The `recurse` function allows you to search through a
@@ -788,8 +1153,26 @@ sections:
               - '{"foo":[{"foo":[]}]}'
               - '{"foo":[]}'
 
+      - title: "`recurse_down`"
+        body: |
+          
+          A quieter version of `recurse(.[])`, equivalent to:
 
-      - title: "String interpolation - `\(foo)`"
+              def recurse_down: recurse(.[]?);
+
+      - title: "`..`"
+        body: |
+          
+          Short-hand for `recurse_down`.  This is intended to resemble
+          the XPath `//` operator.  Note that `..a` does not work; use
+          `..|a` instead.
+
+        examples:
+          - program: '..|.a?'
+            input: '[[{"a":1}]]'
+            output: ['1']
+
+      - title: "String interpolation - `\\(foo)`"
         body: |
 
           Inside a string, you can put an expression inside parens
@@ -801,6 +1184,25 @@ sections:
             input: '42'
             output: ['"The input was 42, which is one less than 43"']
           
+      - title: "Convert to/from JSON"
+        body: |
+
+          The `tojson` and `fromjson` builtins dump values as JSON texts
+          or parse JSON texts into values, respectively.  The tojson
+          builtin differs from tostring in that tostring returns strings
+          unmodified, while tojson encodes strings as JSON strings.
+
+        examples:
+          - program: '[.[]|tostring]'
+            input: '[1, "foo", ["foo"]]'
+            output: ['["1","foo","[\"foo\"]"]']
+          - program: '[.[]|tojson]'
+            input: '[1, "foo", ["foo"]]'
+            output: ['["1","\"foo\"","[\"foo\"]"]']
+          - program: '[.[]|tojson|fromjson]'
+            input: '[1, "foo", ["foo"]]'
+            output: ['[1,"foo",["foo"]]']
+
       - title: "Format strings and escaping"
         body: |
 
@@ -853,9 +1255,9 @@ sections:
               @uri "http://www.google.com/search?q=\(.search)"
 
           will produce the following output for the input
-          `{"search":"jq!"}`:
+          `{"search":"what is jq?"}`:
 
-              http://www.google.com/search?q=jq%21
+              "http://www.google.com/search?q=what%20is%20jq%3f"
 
           Note that the slashes, question mark, etc. in the URL are
           not escaped, as they were part of the string literal.
@@ -875,7 +1277,7 @@ sections:
         
   - title: Conditionals and Comparisons
     entries:
-      - title: `==`, `!=`
+      - title: "`==`, `!=`"
         body: |
 
           The expression 'a == b' will produce 'true' if the result of a and b
@@ -903,7 +1305,7 @@ sections:
           means that you'll sometimes have to be more explicit about
           the condition you want: you can't test whether, e.g. a
           string is empty using `if .name then A else B end`, you'll
-          need something more like `if (.name | count) > 0 then A else
+          need something more like `if (.name | length) > 0 then A else
           B end` instead.
           
           If the condition A produces multiple results, it is
@@ -924,7 +1326,7 @@ sections:
             input: 2
             output: ['"many"']
       
-      - title: `>, >=, <=, <`
+      - title: "`>, >=, <=, <`"
         body: |
           
           The comparison operators `>`, `>=`, `<=`, `<` return whether
@@ -1038,7 +1440,7 @@ sections:
           produces its sum, and the `length` expression is given the array and
           produces its length.
           
-          So, there's generally a cleaner way to solve most problems in jq that
+          So, there's generally a cleaner way to solve most problems in jq than
           defining variables. Still, sometimes they do make things easier, so jq
           lets you define variables using `expression as $variable`. All
           variable names start with `$`. Here's a slightly uglier version of the
@@ -1211,7 +1613,7 @@ sections:
           input, and produce the output {"a": 20}. The latter will set the "a"
           field of the input to the "a" field's "b" field, producing {"a": 10}.
 
-      - title: "`+=`, `-=`, `*=`, `/=`, `//=`"
+      - title: "`+=`, `-=`, `*=`, `/=`, `%=`, `//=`"
         body: |
 
           jq has a few operators of the form `a op= b`, which are all

docs/content/index/index.yml

@@ -27,8 +27,13 @@ tail: |
   for *way* more.
 
 news:
+  - date: 2014-06-08
+    body: |
+      
+      jq 1.4 (finally) released! Get it on the [download](download/) page.
+
   - date: 2013-05-19
     body: |
       
-      jq 1.3 (finally) released! Get it on the [download](download/) page.
+      jq 1.3 released.
 

docs/site.yml

@@ -4,7 +4,7 @@
 
 # This line is modified by the Makefile. To change the version number,
 # edit the Autoconf version number at the top of configure.ac
-jq_version: "1.3"
+jq_version: "1.4"
 
 root: '/jq'
 footer: |

exec_stack.h

@@ -0,0 +1,82 @@
+#ifndef EXEC_STACK_H
+#define EXEC_STACK_H
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct determine_alignment {
+  char x;
+  union { int i; double d; uint64_t u64; size_t sz; void* ptr; } u;
+};
+enum {ALIGNMENT = offsetof(struct determine_alignment, u)};
+
+static size_t align_round_up(size_t sz) {
+  return ((sz + (ALIGNMENT - 1)) / ALIGNMENT) * ALIGNMENT;
+}
+
+typedef int stack_ptr;
+
+struct stack {
+  char* mem_end; // one-past-the-end of allocated region
+  stack_ptr bound;
+  stack_ptr limit; // 0 - stack is empty
+};
+
+static void stack_init(struct stack* s) {
+  s->mem_end = 0;
+  s->bound = ALIGNMENT;
+  s->limit = 0;
+}
+
+static void stack_reset(struct stack* s) {
+  assert(s->limit == 0 && "stack freed while not empty");
+  char* mem_start = s->mem_end - ( -s->bound + ALIGNMENT);
+  free(mem_start);
+  stack_init(s);
+}
+
+static int stack_pop_will_free(struct stack* s, stack_ptr p) {
+  return p == s->limit;
+}
+
+static void* stack_block(struct stack* s, stack_ptr p) {
+  return (void*)(s->mem_end + p);
+}
+
+static stack_ptr* stack_block_next(struct stack* s, stack_ptr p) {
+  return &((stack_ptr*)stack_block(s, p))[-1];
+}
+
+static void stack_reallocate(struct stack* s, size_t sz) {
+  int old_mem_length = -(s->bound) + ALIGNMENT;
+  char* old_mem_start = s->mem_end - old_mem_length;
+  
+  int new_mem_length = align_round_up((old_mem_length + sz + 256) * 2);
+  char* new_mem_start = realloc(old_mem_start, new_mem_length);
+  memmove(new_mem_start + (new_mem_length - old_mem_length),
+            new_mem_start, old_mem_length);
+  s->mem_end = new_mem_start + new_mem_length;
+  s->bound = -(new_mem_length - ALIGNMENT);
+}
+
+static stack_ptr stack_push_block(struct stack* s, stack_ptr p, size_t sz) {
+  int alloc_sz = align_round_up(sz) + ALIGNMENT;
+  stack_ptr r = s->limit - alloc_sz;
+  if (r < s->bound) {
+    stack_reallocate(s, alloc_sz);
+  }
+  s->limit = r;
+  *stack_block_next(s, r) = p;
+  return r;
+}
+
+static stack_ptr stack_pop_block(struct stack* s, stack_ptr p, size_t sz) {
+  stack_ptr r = *stack_block_next(s, p);
+  if (p == s->limit) {
+    int alloc_sz = align_round_up(sz) + ALIGNMENT;
+    s->limit += alloc_sz;
+  }
+  return r;
+}
+#endif

execute.c

@@ -1,80 +1,190 @@
 #include <assert.h>
+#include <errno.h>
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
 
-#include "execute.h"
-
-#include "opcode.h"
+#include "exec_stack.h"
 #include "bytecode.h"
 
-#include "forkable_stack.h"
-#include "frame_layout.h"
-
 #include "jv_alloc.h"
 #include "jq_parser.h"
 #include "locfile.h"
 #include "jv.h"
-#include "jv_aux.h"
+#include "jq.h"
 #include "parser.h"
 #include "builtin.h"
 
 struct jq_state {
-  struct forkable_stack data_stk;
-  struct forkable_stack frame_stk;
-  struct forkable_stack fork_stk;
+  void (*nomem_handler)(void *);
+  void *nomem_handler_data;
+  struct bytecode* bc;
+
+  jq_err_cb err_cb;
+  void *err_cb_data;
+
+  struct stack stk;
+  stack_ptr curr_frame;
+  stack_ptr stk_top;
+  stack_ptr fork_top;
+
   jv path;
   int subexp_nest;
   int debug_trace_enabled;
   int initial_execution;
 };
 
-typedef struct {
-  FORKABLE_STACK_HEADER;
-  jv val;
-} data_stk_elem;
+struct closure {
+  struct bytecode* bc;
+  stack_ptr env;
+};
+
+union frame_entry {
+  struct closure closure;
+  jv localvar;
+};
+
+struct frame {
+  struct bytecode* bc;
+  stack_ptr env;
+  stack_ptr retdata;
+  uint16_t* retaddr;
+  /* bc->nclosures closures followed by bc->nlocals local variables */
+  union frame_entry entries[0]; 
+};
+
+static int frame_size(struct bytecode* bc) {
+  return sizeof(struct frame) + sizeof(union frame_entry) * (bc->nclosures + bc->nlocals);
+}
+
+static struct frame* frame_current(struct jq_state* jq) {
+  struct frame* fp = stack_block(&jq->stk, jq->curr_frame);
+
+  stack_ptr next = *stack_block_next(&jq->stk, jq->curr_frame);
+  if (next) {
+    struct frame* fpnext = stack_block(&jq->stk, next);
+    struct bytecode* bc = fpnext->bc;
+    assert(fp->retaddr >= bc->code && fp->retaddr < bc->code + bc->codelen);
+  } else {
+    assert(fp->retaddr == 0);
+  }
+  return fp;
+}
+
+static stack_ptr frame_get_level(struct jq_state* jq, int level) {
+  stack_ptr fr = jq->curr_frame;
+  for (int i=0; i<level; i++) {
+    struct frame* fp = stack_block(&jq->stk, fr);
+    fr = fp->env;
+  }
+  return fr;
+}
+
+static jv* frame_local_var(struct jq_state* jq, int var, int level) {
+  struct frame* fr = stack_block(&jq->stk, frame_get_level(jq, level));
+  assert(var >= 0);
+  assert(var < fr->bc->nlocals);
+  return &fr->entries[fr->bc->nclosures + var].localvar;
+}
+
+static struct closure make_closure(struct jq_state* jq, uint16_t* pc) {
+  uint16_t level = *pc++;
+  uint16_t idx = *pc++;
+  stack_ptr fridx = frame_get_level(jq, level);
+  struct frame* fr = stack_block(&jq->stk, fridx);
+  if (idx & ARG_NEWCLOSURE) {
+    int subfn_idx = idx & ~ARG_NEWCLOSURE;
+    assert(subfn_idx < fr->bc->nsubfunctions);
+    struct closure cl = {fr->bc->subfunctions[subfn_idx],
+                         fridx};
+    return cl;
+  } else {
+    int closure = idx;
+    assert(closure >= 0);
+    assert(closure < fr->bc->nclosures);
+    return fr->entries[closure].closure;
+  }
+}
+
+static struct frame* frame_push(struct jq_state* jq, struct closure callee, 
+                                uint16_t* argdef, int nargs) {
+  stack_ptr new_frame_idx = stack_push_block(&jq->stk, jq->curr_frame, frame_size(callee.bc));
+  struct frame* new_frame = stack_block(&jq->stk, new_frame_idx);
+  new_frame->bc = callee.bc;
+  new_frame->env = callee.env;
+  assert(nargs == new_frame->bc->nclosures);
+  union frame_entry* entries = new_frame->entries;
+  for (int i=0; i<nargs; i++) {
+    entries->closure = make_closure(jq, argdef + i * 2);
+    entries++;
+  }
+  for (int i=0; i<callee.bc->nlocals; i++) {
+    entries->localvar = jv_invalid();
+    entries++;
+  }
+  jq->curr_frame = new_frame_idx;
+  return new_frame;
+}
+
+static void frame_pop(struct jq_state* jq) {
+  assert(jq->curr_frame);
+  struct frame* fp = frame_current(jq);
+  if (stack_pop_will_free(&jq->stk, jq->curr_frame)) {
+    int nlocals = fp->bc->nlocals;
+    for (int i=0; i<nlocals; i++) {
+      jv_free(*frame_local_var(jq, i, 0));
+    }
+  }
+  jq->curr_frame = stack_pop_block(&jq->stk, jq->curr_frame, frame_size(fp->bc));
+}
 
 void stack_push(jq_state *jq, jv val) {
   assert(jv_is_valid(val));
-  data_stk_elem* s = forkable_stack_push(&jq->data_stk, sizeof(data_stk_elem));
-  s->val = val;
+  jq->stk_top = stack_push_block(&jq->stk, jq->stk_top, sizeof(jv));
+  jv* sval = stack_block(&jq->stk, jq->stk_top);
+  *sval = val;
 }
 
 jv stack_pop(jq_state *jq) {
-  data_stk_elem* s = forkable_stack_peek(&jq->data_stk);
-  jv val = s->val;
-  if (!forkable_stack_pop_will_free(&jq->data_stk)) {
+  jv* sval = stack_block(&jq->stk, jq->stk_top);
+  jv val = *sval;
+  if (!stack_pop_will_free(&jq->stk, jq->stk_top)) {
     val = jv_copy(val);
   }
-  forkable_stack_pop(&jq->data_stk);
+  jq->stk_top = stack_pop_block(&jq->stk, jq->stk_top, sizeof(jv));
   assert(jv_is_valid(val));
   return val;
 }
 
 
 struct forkpoint {
-  FORKABLE_STACK_HEADER;
-  struct forkable_stack_state saved_data_stack;
-  struct forkable_stack_state saved_call_stack;
+  stack_ptr saved_data_stack;
+  stack_ptr saved_curr_frame;
   int path_len, subexp_nest;
   uint16_t* return_address;
 };
 
+struct stack_pos {
+  stack_ptr saved_data_stack, saved_curr_frame;
+};
 
-void stack_save(jq_state *jq, uint16_t* retaddr){
-  struct forkpoint* fork = forkable_stack_push(&jq->fork_stk, sizeof(struct forkpoint));
-  forkable_stack_save(&jq->data_stk, &fork->saved_data_stack);
-  forkable_stack_save(&jq->frame_stk, &fork->saved_call_stack);
+struct stack_pos stack_get_pos(jq_state* jq) {
+  struct stack_pos sp = {jq->stk_top, jq->curr_frame};
+  return sp;
+}
+
+void stack_save(jq_state *jq, uint16_t* retaddr, struct stack_pos sp){
+  jq->fork_top = stack_push_block(&jq->stk, jq->fork_top, sizeof(struct forkpoint));
+  struct forkpoint* fork = stack_block(&jq->stk, jq->fork_top);
+  fork->saved_data_stack = jq->stk_top;
+  fork->saved_curr_frame = jq->curr_frame;
   fork->path_len = 
     jv_get_kind(jq->path) == JV_KIND_ARRAY ? jv_array_length(jv_copy(jq->path)) : 0;
   fork->subexp_nest = jq->subexp_nest;
   fork->return_address = retaddr;
-}
-
-void stack_switch(jq_state *jq) {
-  struct forkpoint* fork = forkable_stack_peek(&jq->fork_stk);
-  forkable_stack_switch(&jq->data_stk, &fork->saved_data_stack);
-  forkable_stack_switch(&jq->frame_stk, &fork->saved_call_stack);
+  jq->stk_top = sp.saved_data_stack;
+  jq->curr_frame = sp.saved_curr_frame;
 }
 
 void path_append(jq_state* jq, jv component) {
@@ -89,23 +199,24 @@ void path_append(jq_state* jq, jv component) {
 }
 
 uint16_t* stack_restore(jq_state *jq){
-  while (!forkable_stack_empty(&jq->data_stk) && 
-         forkable_stack_pop_will_free(&jq->data_stk)) {
-    jv_free(stack_pop(jq));
-  }
-  while (!forkable_stack_empty(&jq->frame_stk) && 
-         forkable_stack_pop_will_free(&jq->frame_stk)) {
-    frame_pop(&jq->frame_stk);
+  while (!stack_pop_will_free(&jq->stk, jq->fork_top)) {
+    if (stack_pop_will_free(&jq->stk, jq->stk_top)) {
+      jv_free(stack_pop(jq));
+    } else if (stack_pop_will_free(&jq->stk, jq->curr_frame)) {
+      frame_pop(jq);
+    } else {
+      assert(0);
+    }
   }
 
-  if (forkable_stack_empty(&jq->fork_stk)) {
+  if (jq->fork_top == 0) {
     return 0;
   }
 
-  struct forkpoint* fork = forkable_stack_peek(&jq->fork_stk);
+  struct forkpoint* fork = stack_block(&jq->stk, jq->fork_top);
   uint16_t* retaddr = fork->return_address;
-  forkable_stack_restore(&jq->data_stk, &fork->saved_data_stack);
-  forkable_stack_restore(&jq->frame_stk, &fork->saved_call_stack);
+  jq->stk_top = fork->saved_data_stack;
+  jq->curr_frame = fork->saved_curr_frame;
   int path_len = fork->path_len;
   if (jv_get_kind(jq->path) == JV_KIND_ARRAY) {
     assert(path_len >= 0);
@@ -114,39 +225,49 @@ uint16_t* stack_restore(jq_state *jq){
     assert(path_len == 0);
   }
   jq->subexp_nest = fork->subexp_nest;
-  forkable_stack_pop(&jq->fork_stk);
+  jq->fork_top = stack_pop_block(&jq->stk, jq->fork_top, sizeof(struct forkpoint));
   return retaddr;
 }
 
+static void jq_reset(jq_state *jq) {
+  while (stack_restore(jq)) {}
 
-static struct closure make_closure(struct forkable_stack* stk, frame_ptr fr, uint16_t* pc) {
-  uint16_t level = *pc++;
-  uint16_t idx = *pc++;
-  fr = frame_get_level(stk, fr, level);
-  if (idx & ARG_NEWCLOSURE) {
-    int subfn_idx = idx & ~ARG_NEWCLOSURE;
-    assert(subfn_idx < frame_self(fr)->bc->nsubfunctions);
-    struct closure cl = {frame_self(fr)->bc->subfunctions[subfn_idx],
-                         forkable_stack_to_idx(stk, fr)};
-    return cl;
-  } else {
-    return *frame_closure_arg(fr, idx);
-  }
+  assert(jq->stk_top == 0);
+  assert(jq->fork_top == 0);
+  assert(jq->curr_frame == 0);
+  stack_reset(&jq->stk);
+
+  if (jv_get_kind(jq->path) != JV_KIND_INVALID)
+    jv_free(jq->path);
+  jq->path = jv_null();
+  jq->subexp_nest = 0;
 }
 
-void print_error(jv value) {
+static void print_error(jq_state *jq, jv value) {
   assert(!jv_is_valid(value));
   jv msg = jv_invalid_get_msg(value);
-  if (jv_get_kind(msg) == JV_KIND_STRING) {
-    fprintf(stderr, "jq: error: %s\n", jv_string_value(msg));
-  }
+  jv msg2;
+  if (jv_get_kind(msg) == JV_KIND_STRING)
+    msg2 = jv_string_fmt("jq: error: %s", jv_string_value(msg));
+  else
+    msg2 = jv_string_fmt("jq: error: <unknown>");
   jv_free(msg);
+
+  if (jq->err_cb)
+    jq->err_cb(jq->err_cb_data, msg2);
+  else if (jv_get_kind(msg2) == JV_KIND_STRING)
+    fprintf(stderr, "%s\n", jv_string_value(msg2));
+  else
+    fprintf(stderr, "jq: error: %s\n", strerror(ENOMEM));
+  jv_free(msg2);
 }
 #define ON_BACKTRACK(op) ((op)+NUM_OPCODES)
 
 jv jq_next(jq_state *jq) {
   jv cfunc_input[MAX_CFUNCTION_ARGS];
 
+  jv_nomem_handler(jq->nomem_handler, jq->nomem_handler_data);
+
   uint16_t* pc = stack_restore(jq);
   assert(pc);
 
@@ -156,30 +277,33 @@ jv jq_next(jq_state *jq) {
     uint16_t opcode = *pc;
 
     if (jq->debug_trace_enabled) {
-      dump_operation(frame_current_bytecode(&jq->frame_stk), pc);
+      dump_operation(frame_current(jq)->bc, pc);
       printf("\t");
       const struct opcode_description* opdesc = opcode_describe(opcode);
-      data_stk_elem* param = 0;
-      int stack_in = opdesc->stack_in;
-      if (stack_in == -1) stack_in = pc[1];
-      for (int i=0; i<stack_in; i++) {
-        if (i == 0) {
-          param = forkable_stack_peek(&jq->data_stk);
-        } else {
-          printf(" | ");
-          param = forkable_stack_peek_next(&jq->data_stk, param);
+      stack_ptr param = 0;
+      if (!backtracking) {
+        int stack_in = opdesc->stack_in;
+        if (stack_in == -1) stack_in = pc[1];
+        for (int i=0; i<stack_in; i++) {
+          if (i == 0) {
+            param = jq->stk_top;
+          } else {
+            printf(" | ");
+            param = *stack_block_next(&jq->stk, param);
+          }
+          if (!param) break;
+          jv_dump(jv_copy(*(jv*)stack_block(&jq->stk, param)), 0);
+          //printf("<%d>", jv_get_refcnt(param->val));
+          //printf(" -- ");
+          //jv_dump(jv_copy(jq->path), 0);
         }
-        if (!param) break;
-        jv_dump(jv_copy(param->val), 0);
-        //printf("<%d>", jv_get_refcnt(param->val));
-        //printf(" -- ");
-        //jv_dump(jv_copy(jq->path), 0);
+      } else {
+        printf("\t<backtracking>");
       }
 
-      if (backtracking) printf("\t<backtracking>");
-
       printf("\n");
     }
+
     if (backtracking) {
       opcode = ON_BACKTRACK(opcode);
       backtracking = 0;
@@ -190,7 +314,7 @@ jv jq_next(jq_state *jq) {
     default: assert(0 && "invalid instruction");
 
     case LOADK: {
-      jv v = jv_array_get(jv_copy(frame_current_bytecode(&jq->frame_stk)->constants), *pc++);
+      jv v = jv_array_get(jv_copy(frame_current(jq)->bc->constants), *pc++);
       assert(jv_is_valid(v));
       jv_free(stack_pop(jq));
       stack_push(jq, v);
@@ -240,8 +364,7 @@ jv jq_next(jq_state *jq) {
       jv v = stack_pop(jq);
       uint16_t level = *pc++;
       uint16_t vidx = *pc++;
-      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
-      jv* var = frame_local_var(fp, vidx);
+      jv* var = frame_local_var(jq, vidx, level);
       assert(jv_get_kind(*var) == JV_KIND_ARRAY);
       *var = jv_array_append(*var, v);
       break;
@@ -257,8 +380,8 @@ jv jq_next(jq_state *jq) {
         stack_push(jq, jv_object_set(objv, k, v));
         stack_push(jq, stktop);
       } else {
-        print_error(jv_invalid_with_msg(jv_string_fmt("Cannot use %s as object key",
-                                                      jv_kind_name(jv_get_kind(k)))));
+        print_error(jq, jv_invalid_with_msg(jv_string_fmt("Cannot use %s as object key",
+                                                          jv_kind_name(jv_get_kind(k)))));
         jv_free(stktop);
         jv_free(v);
         jv_free(k);
@@ -272,12 +395,11 @@ jv jq_next(jq_state *jq) {
     case RANGE: {
       uint16_t level = *pc++;
       uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
-      jv* var = frame_local_var(fp, v);
+      jv* var = frame_local_var(jq, v, level);
       jv max = stack_pop(jq);
       if (jv_get_kind(*var) != JV_KIND_NUMBER ||
           jv_get_kind(max) != JV_KIND_NUMBER) {
-        print_error(jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric")));
+        print_error(jq, jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric")));
         jv_free(max);
         goto do_backtrack;
       } else if (jv_number_value(jv_copy(*var)) >= jv_number_value(jv_copy(max))) {
@@ -287,9 +409,10 @@ jv jq_next(jq_state *jq) {
         jv curr = jv_copy(*var);
         *var = jv_number(jv_number_value(*var) + 1);
 
-        stack_save(jq, pc - 3);
+        struct stack_pos spos = stack_get_pos(jq);
         stack_push(jq, jv_copy(max));
-        stack_switch(jq);
+        stack_save(jq, pc - 3, spos);
+
         stack_push(jq, curr);
       }
       break;
@@ -299,8 +422,7 @@ jv jq_next(jq_state *jq) {
     case LOADV: {
       uint16_t level = *pc++;
       uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
-      jv* var = frame_local_var(fp, v);
+      jv* var = frame_local_var(jq, v, level);
       if (jq->debug_trace_enabled) {
         printf("V%d = ", v);
         jv_dump(jv_copy(*var), 0);
@@ -315,8 +437,7 @@ jv jq_next(jq_state *jq) {
     case LOADVN: {
       uint16_t level = *pc++;
       uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
-      jv* var = frame_local_var(fp, v);
+      jv* var = frame_local_var(jq, v, level);
       if (jq->debug_trace_enabled) {
         printf("V%d = ", v);
         jv_dump(jv_copy(*var), 0);
@@ -331,8 +452,7 @@ jv jq_next(jq_state *jq) {
     case STOREV: {
       uint16_t level = *pc++;
       uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
-      jv* var = frame_local_var(fp, v);
+      jv* var = frame_local_var(jq, v, level);
       jv val = stack_pop(jq);
       if (jq->debug_trace_enabled) {
         printf("V%d = ", v);
@@ -348,8 +468,7 @@ jv jq_next(jq_state *jq) {
       jv v = stack_pop(jq);
       stack_push(jq, jq->path);
 
-      stack_save(jq, pc - 1);
-      stack_switch(jq);
+      stack_save(jq, pc - 1, stack_get_pos(jq));
 
       stack_push(jq, jv_number(jq->subexp_nest));
       stack_push(jq, v);
@@ -368,9 +487,9 @@ jv jq_next(jq_state *jq) {
       jv path = jq->path;
       jq->path = stack_pop(jq);
 
-      stack_save(jq, pc - 1);
+      struct stack_pos spos = stack_get_pos(jq);
       stack_push(jq, jv_copy(path));
-      stack_switch(jq);
+      stack_save(jq, pc - 1, spos);
 
       stack_push(jq, path);
       jq->subexp_nest = old_subexp_nest;
@@ -384,7 +503,8 @@ jv jq_next(jq_state *jq) {
       goto do_backtrack;
     }
 
-    case INDEX: {
+    case INDEX:
+    case INDEX_OPT: {
       jv t = stack_pop(jq);
       jv k = stack_pop(jq);
       path_append(jq, jv_copy(k));
@@ -392,7 +512,10 @@ jv jq_next(jq_state *jq) {
       if (jv_is_valid(v)) {
         stack_push(jq, v);
       } else {
-        print_error(v);
+        if (opcode == INDEX)
+          print_error(jq, v);
+        else
+          jv_free(v);
         goto do_backtrack;
       }
       break;
@@ -417,16 +540,18 @@ jv jq_next(jq_state *jq) {
     }
 
     case EACH: 
+    case EACH_OPT: 
       stack_push(jq, jv_number(-1));
       // fallthrough
-    case ON_BACKTRACK(EACH): {
+    case ON_BACKTRACK(EACH):
+    case ON_BACKTRACK(EACH_OPT): {
       int idx = jv_number_value(stack_pop(jq));
       jv container = stack_pop(jq);
 
       int keep_going, is_last = 0;
       jv key, value;
       if (jv_get_kind(container) == JV_KIND_ARRAY) {
-        if (opcode == EACH) idx = 0;
+        if (opcode == EACH || opcode == EACH_OPT) idx = 0;
         else idx = idx + 1;
         int len = jv_array_length(jv_copy(container));
         keep_going = idx < len;
@@ -436,7 +561,7 @@ jv jq_next(jq_state *jq) {
           value = jv_array_get(jv_copy(container), idx);
         }
       } else if (jv_get_kind(container) == JV_KIND_OBJECT) {
-        if (opcode == EACH) idx = jv_object_iter(container);
+        if (opcode == EACH || opcode == EACH_OPT) idx = jv_object_iter(container);
         else idx = jv_object_iter_next(container, idx);
         keep_going = jv_object_iter_valid(container, idx);
         if (keep_going) {
@@ -444,9 +569,12 @@ jv jq_next(jq_state *jq) {
           value = jv_object_iter_value(container, idx);
         }
       } else {
-        assert(opcode == EACH);
-        print_error(jv_invalid_with_msg(jv_string_fmt("Cannot iterate over %s",
-                                                      jv_kind_name(jv_get_kind(container)))));
+        assert(opcode == EACH || opcode == EACH_OPT);
+        if (opcode == EACH) {
+          print_error(jq,
+                      jv_invalid_with_msg(jv_string_fmt("Cannot iterate over %s",
+                                                        jv_kind_name(jv_get_kind(container)))));
+        }
         keep_going = 0;
       }
 
@@ -459,10 +587,10 @@ jv jq_next(jq_state *jq) {
         path_append(jq, key);
         stack_push(jq, value);
       } else {
-        stack_save(jq, pc - 1);
+        struct stack_pos spos = stack_get_pos(jq);
         stack_push(jq, container);
         stack_push(jq, jv_number(idx));
-        stack_switch(jq);
+        stack_save(jq, pc - 1, spos);
         path_append(jq, key);
         stack_push(jq, value);
       }
@@ -480,8 +608,7 @@ jv jq_next(jq_state *jq) {
     }
 
     case FORK: {
-      stack_save(jq, pc - 1);
-      stack_switch(jq);
+      stack_save(jq, pc - 1, stack_get_pos(jq));
       pc++; // skip offset this time
       break;
     }
@@ -495,53 +622,64 @@ jv jq_next(jq_state *jq) {
     case CALL_BUILTIN: {
       int nargs = *pc++;
       jv top = stack_pop(jq);
-      cfunc_input[0] = top;
+      jv* in = cfunc_input;
+      in[0] = top;
       for (int i = 1; i < nargs; i++) {
-        cfunc_input[i] = stack_pop(jq);
+        in[i] = stack_pop(jq);
       }
-      struct cfunction* func = &frame_current_bytecode(&jq->frame_stk)->globals->cfunctions[*pc++];
-      top = cfunction_invoke(func, cfunc_input);
+      struct cfunction* function = &frame_current(jq)->bc->globals->cfunctions[*pc++];
+      typedef jv (*func_1)(jv);
+      typedef jv (*func_2)(jv,jv);
+      typedef jv (*func_3)(jv,jv,jv);
+      typedef jv (*func_4)(jv,jv,jv,jv);
+      typedef jv (*func_5)(jv,jv,jv,jv,jv);
+      switch (function->nargs) {
+      case 1: top = ((func_1)function->fptr)(in[0]); break;
+      case 2: top = ((func_2)function->fptr)(in[0], in[1]); break;
+      case 3: top = ((func_3)function->fptr)(in[0], in[1], in[2]); break;
+      case 4: top = ((func_4)function->fptr)(in[0], in[1], in[2], in[3]); break;
+      case 5: top = ((func_5)function->fptr)(in[0], in[1], in[2], in[3], in[4]); break;
+      default: return jv_invalid_with_msg(jv_string("Function takes too many arguments"));
+      }
+      
       if (jv_is_valid(top)) {
         stack_push(jq, top);
       } else {
-        print_error(top);
+        print_error(jq, top);
         goto do_backtrack;
       }
       break;
     }
 
     case CALL_JQ: {
+      jv input = stack_pop(jq);
       uint16_t nclosures = *pc++;
       uint16_t* retaddr = pc + 2 + nclosures*2;
-      frame_ptr new_frame = frame_push(&jq->frame_stk, 
-                                       make_closure(&jq->frame_stk, frame_current(&jq->frame_stk), pc),
-                                       retaddr);
-      pc += 2;
-      frame_ptr old_frame = forkable_stack_peek_next(&jq->frame_stk, new_frame);
-      assert(nclosures == frame_self(new_frame)->bc->nclosures);
-      for (int i=0; i<nclosures; i++) {
-        *frame_closure_arg(new_frame, i) = make_closure(&jq->frame_stk, old_frame, pc);
-        pc += 2;
-      }
-
-      pc = frame_current_bytecode(&jq->frame_stk)->code;
+      struct frame* new_frame = frame_push(jq, make_closure(jq, pc),
+                                           pc + 2, nclosures);
+      new_frame->retdata = jq->stk_top;
+      new_frame->retaddr = retaddr;
+      pc = new_frame->bc->code;
+      stack_push(jq, input);
       break;
     }
 
     case RET: {
-      uint16_t* retaddr = *frame_current_retaddr(&jq->frame_stk);
+      jv value = stack_pop(jq);
+      assert(jq->stk_top == frame_current(jq)->retdata);
+      uint16_t* retaddr = frame_current(jq)->retaddr;
       if (retaddr) {
         // function return
         pc = retaddr;
-        frame_pop(&jq->frame_stk);
+        frame_pop(jq);
       } else {
         // top-level return, yielding value
-        jv value = stack_pop(jq);
-        stack_save(jq, pc - 1);
+        struct stack_pos spos = stack_get_pos(jq);
         stack_push(jq, jv_null());
-        stack_switch(jq);
+        stack_save(jq, pc - 1, spos);
         return value;
       }
+      stack_push(jq, value);
       break;
     }
     case ON_BACKTRACK(RET): {
@@ -552,28 +690,60 @@ jv jq_next(jq_state *jq) {
   }
 }
 
+jq_state *jq_init(void) {
+  jq_state *jq;
+  jq = jv_mem_alloc_unguarded(sizeof(*jq));
+  if (jq == NULL)
+    return NULL;
 
-void jq_init(struct bytecode* bc, jv input, jq_state **jq, int flags) {
-  jq_state *new_jq;
-  new_jq = jv_mem_alloc(sizeof(*new_jq));
-  memset(new_jq, 0, sizeof(*new_jq));
-  new_jq->path = jv_null();
-  forkable_stack_init(&new_jq->data_stk, sizeof(data_stk_elem) * 100);
-  forkable_stack_init(&new_jq->frame_stk, 1024);
-  forkable_stack_init(&new_jq->fork_stk, 1024);
+  jq->bc = 0;
+
+  stack_init(&jq->stk);
+  jq->stk_top = 0;
+  jq->fork_top = 0;
+  jq->curr_frame = 0;
+
+  jq->err_cb = NULL;
+  jq->err_cb_data = NULL;
+
+  jq->path = jv_null();
+  return jq;
+}
+
+void jq_set_error_cb(jq_state *jq, jq_err_cb cb, void *data) {
+  jq->err_cb = cb;
+  jq->err_cb_data = data;
+}
+
+void jq_get_error_cb(jq_state *jq, jq_err_cb *cb, void **data) {
+  *cb = jq->err_cb;
+  *data = jq->err_cb_data;
+}
+
+void jq_set_nomem_handler(jq_state *jq, void (*nomem_handler)(void *), void *data) {
+  jv_nomem_handler(nomem_handler, data);
+  jq->nomem_handler = nomem_handler;
+  jq->nomem_handler_data = data;
+}
+
+
+void jq_start(jq_state *jq, jv input, int flags) {
+  jv_nomem_handler(jq->nomem_handler, jq->nomem_handler_data);
+  jq_reset(jq);
   
-  stack_push(new_jq, input);
-  struct closure top = {bc, -1};
-  frame_push(&new_jq->frame_stk, top, 0);
-  stack_save(new_jq, bc->code);
-  stack_switch(new_jq);
+  struct closure top = {jq->bc, -1};
+  struct frame* top_frame = frame_push(jq, top, 0, 0);
+  top_frame->retdata = 0;
+  top_frame->retaddr = 0;
+
+  stack_push(jq, input);
+  stack_save(jq, jq->bc->code, stack_get_pos(jq));
   if (flags & JQ_DEBUG_TRACE) {
-    new_jq->debug_trace_enabled = 1;
+    jq->debug_trace_enabled = 1;
   } else {
-    new_jq->debug_trace_enabled = 0;
+    jq->debug_trace_enabled = 0;
   }
-  new_jq->initial_execution = 1;
-  *jq = new_jq;
+  jq->initial_execution = 1;
 }
 
 void jq_teardown(jq_state **jq) {
@@ -582,25 +752,24 @@ void jq_teardown(jq_state **jq) {
     return;
   *jq = NULL;
 
-  while (stack_restore(old_jq)) {}
+  jq_reset(old_jq);
+  bytecode_free(old_jq->bc);
+  old_jq->bc = 0;
 
-  assert(forkable_stack_empty(&old_jq->fork_stk));
-  assert(forkable_stack_empty(&old_jq->data_stk));
-  assert(forkable_stack_empty(&old_jq->frame_stk));
-  forkable_stack_free(&old_jq->fork_stk);
-  forkable_stack_free(&old_jq->data_stk);
-  forkable_stack_free(&old_jq->frame_stk);
-
-  jv_free(old_jq->path);
   jv_mem_free(old_jq);
 }
 
-struct bytecode* jq_compile_args(const char* str, jv args) {
+int jq_compile_args(jq_state *jq, const char* str, jv args) {
+  jv_nomem_handler(jq->nomem_handler, jq->nomem_handler_data);
   assert(jv_get_kind(args) == JV_KIND_ARRAY);
   struct locfile locations;
-  locfile_init(&locations, str, strlen(str));
+  locfile_init(&locations, jq, str, strlen(str));
   block program;
-  struct bytecode* bc = 0;
+  jq_reset(jq);
+  if (jq->bc) {
+    bytecode_free(jq->bc);
+    jq->bc = 0;
+  }
   int nerrors = jq_parse(&locations, &program);
   if (nerrors == 0) {
     for (int i=0; i<jv_array_length(jv_copy(args)); i++) {
@@ -610,17 +779,31 @@ struct bytecode* jq_compile_args(const char* str, jv args) {
       program = gen_var_binding(gen_const(value), jv_string_value(name), program);
       jv_free(name);
     }
-    jv_free(args);
-    program = builtins_bind(program);
-    nerrors = block_compile(program, &locations, &bc);
+    nerrors = builtins_bind(jq, &program);
+    if (nerrors == 0) {
+      nerrors = block_compile(program, &locations, &jq->bc);
+    }
   }
+  jv_free(args);
   if (nerrors) {
-    fprintf(stderr, "%d compile %s\n", nerrors, nerrors > 1 ? "errors" : "error");
+    jv s = jv_string_fmt("%d compile %s", nerrors,
+                         nerrors > 1 ? "errors" : "error");
+    if (jq->err_cb != NULL)
+      jq->err_cb(jq->err_cb_data, s);
+    else if (!jv_is_valid(s))
+      fprintf(stderr, "Error formatting jq compilation errors: %s\n", strerror(errno));
+    else
+      fprintf(stderr, "%s\n", jv_string_value(s));
+    jv_free(s);
   }
   locfile_free(&locations);
-  return bc;
+  return jq->bc != NULL;
 }
 
-struct bytecode* jq_compile(const char* str) {
-  return jq_compile_args(str, jv_array());
+int jq_compile(jq_state *jq, const char* str) {
+  return jq_compile_args(jq, str, jv_array());
+}
+
+void jq_dump_disassembly(jq_state *jq, int indent) {
+  dump_disassembly(indent, jq->bc);
 }

execute.h

@@ -1,20 +0,0 @@
-#ifndef EXECUTE_H
-#define EXECUTE_H
-#include "bytecode.h"
-
-
-
-struct bytecode* jq_compile(const char* str);
-
-/* args must be an array of the form [{name:"foo", value:"thing"}, {name:"bar",value:3}] */
-struct bytecode* jq_compile_args(const char* str, jv args);
-
-typedef struct jq_state jq_state;
-enum {JQ_DEBUG_TRACE = 1};
-
-void jq_init(struct bytecode* bc, jv value, jq_state **, int flags);
-jv jq_next(jq_state *);
-void jq_teardown(jq_state **);
-
-
-#endif

forkable_stack.h

@@ -1,172 +0,0 @@
-#ifndef FORKABLE_STACK_H
-#define FORKABLE_STACK_H
-#include <stddef.h>
-#include <assert.h>
-#include <string.h>
-#include "jv_alloc.h"
-
-struct forkable_stack_header {
-  /* distance in bytes between this header and the header
-     of the next object on the stack */
-  int next_delta;
-};
-
-#define FORKABLE_STACK_HEADER struct forkable_stack_header fk_header_
-
-struct forkable_stack {
-  // Stack grows down from stk+length
-  char* stk;
-
-  // stk+length is just past end of allocated area
-  // stk = jv_mem_alloc(length)
-  int length;
-
-  // stk+pos is header of top object, or stk+length if empty
-  int pos;
-
-  // everything after-or-including stk+savedlimit must be preserved
-  int savedlimit;
-};
-
-static void forkable_stack_check(struct forkable_stack* s) {
-  assert(s->stk);
-  assert(s->length > 0);
-  assert(s->pos >= 0 && s->pos <= s->length);
-  assert(s->savedlimit >= 0 && s->savedlimit <= s->length);
-}
-
-static int forkable_stack_empty(struct forkable_stack* s) {
-  return s->pos == s->length;
-}
-
-static void forkable_stack_init(struct forkable_stack* s, size_t sz) {
-  s->stk = jv_mem_alloc(sz);
-  s->length = sz;
-  s->pos = s->length;
-  s->savedlimit = s->length;
-  forkable_stack_check(s);
-}
-
-static void forkable_stack_free(struct forkable_stack* s) {
-  assert(forkable_stack_empty(s));
-  assert(s->savedlimit == s->length);
-  jv_mem_free(s->stk);
-  s->stk = 0;
-}
-
-static void* forkable_stack_push(struct forkable_stack* s, size_t sz_size) {
-  assert(sz_size > 0 && sz_size % sizeof(struct forkable_stack_header) == 0);
-  int size = (int)sz_size;
-  forkable_stack_check(s);
-  int curr = s->pos < s->savedlimit ? s->pos : s->savedlimit;
-  if (curr - size < 0) {
-    int oldlen = s->length;
-    s->length = (size + oldlen + 1024) * 2;
-    s->stk = jv_mem_realloc(s->stk, s->length);
-    int shift = s->length - oldlen;
-    memmove(s->stk + shift, s->stk, oldlen);
-    s->pos += shift;
-    s->savedlimit += shift;
-    curr += shift;
-  }
-  int newpos = curr - size;
-  assert(newpos < s->pos);
-  void* ret = (void*)(s->stk + newpos);
-  ((struct forkable_stack_header*)ret)->next_delta = s->pos - newpos;
-  s->pos = newpos;
-  forkable_stack_check(s);
-  return ret;
-}
-
-static void* forkable_stack_peek(struct forkable_stack* s) {
-  assert(!forkable_stack_empty(s));
-  forkable_stack_check(s);
-  return (void*)(s->stk + s->pos);
-}
-
-static void* forkable_stack_peek_next(struct forkable_stack* s, void* top) {
-  forkable_stack_check(s);
-  struct forkable_stack_header* elem = top;
-  int elempos = (char*)top - s->stk;
-  assert(elempos >= 0 && elempos < s->length);
-  assert(elempos + elem->next_delta <= s->length);
-  if (elempos + elem->next_delta < s->length) {
-    return (void*)(s->stk + elempos + elem->next_delta);
-  } else {
-    return 0;
-  }
-}
-
-// Returns 1 if the next forkable_stack_pop will permanently remove an
-// object from the stack (i.e. the top object was not saved with a fork)
-static int forkable_stack_pop_will_free(struct forkable_stack* s) {
-  return s->pos < s->savedlimit;
-}
-
-static void forkable_stack_pop(struct forkable_stack* s) {
-  forkable_stack_check(s);
-  struct forkable_stack_header* elem = forkable_stack_peek(s);
-  int reclaim_upto = s->pos + elem->next_delta < s->savedlimit ? 
-    s->pos + elem->next_delta : s->savedlimit;
-  assert(reclaim_upto <= s->length);
-  int reclaimed = reclaim_upto > s->pos ? reclaim_upto - s->pos : 0;
-  assert(0 <= reclaimed && s->pos + reclaimed <= s->length);
-  // s->pos <= i < reclaim_upto means that position i is to be freed
-  assert((reclaimed > 0) == forkable_stack_pop_will_free(s));
-  int new_pos = s->pos + elem->next_delta;
-  jv_mem_invalidate(elem, reclaimed);
-  s->pos = new_pos;
-}
-
-
-
-struct forkable_stack_state {
-  // We save the previous pos, savedlimit as
-  // length-pos, length-savedlimit since these
-  // values are stable across stack reallocations.
-  int prev_pos_delta, prev_limit_delta;
-};
-
-static void forkable_stack_save(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  state->prev_pos_delta = s->length - s->pos;
-  state->prev_limit_delta = s->length - s->savedlimit;
-  if (s->pos < s->savedlimit) s->savedlimit = s->pos;
-}
-
-static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  int curr_pos = s->pos;
-  s->pos = s->length - state->prev_pos_delta;
-  state->prev_pos_delta = s->length - curr_pos;
-
-  int curr_limit = s->savedlimit;
-  if (curr_pos < curr_limit) s->savedlimit = curr_pos;
-  //state->prevlimit = curr_limit; FIXME clean up
-  forkable_stack_check(s);
-}
-
-static void forkable_stack_restore(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  assert(s->savedlimit <= s->length - state->prev_pos_delta);
-  assert(s->savedlimit <= s->length - state->prev_limit_delta);
-  s->pos = s->length - state->prev_pos_delta;
-  s->savedlimit = s->length - state->prev_limit_delta;
-  forkable_stack_check(s);
-}
-
-typedef int stack_idx;
-
-static stack_idx forkable_stack_to_idx(struct forkable_stack* s, void* ptr) {
-  char* item = ptr;
-  int pos = item - s->stk;
-  assert(pos >= 0 && pos < s->length);
-  return s->length - pos;
-}
-
-static void* forkable_stack_from_idx(struct forkable_stack* s, stack_idx idx) {
-  assert(idx >= 1 && idx <= s->length);
-  return &s->stk[s->length - idx];
-}
-
-#endif

frame_layout.h

@@ -1,104 +0,0 @@
-#ifndef FRAME_LAYOUT_H
-#include "forkable_stack.h"
-#include "bytecode.h"
-
-struct closure {
-  struct bytecode* bc;
-  stack_idx env;
-};
-
-struct continuation {
-  struct bytecode* bc;
-  stack_idx env;
-  uint16_t* retaddr;
-};
-
-typedef union frame_elem {
-  FORKABLE_STACK_HEADER;
-  struct continuation cont;
-  struct closure closure;
-  jv jsonval;
-} *frame_ptr;
-
-/*
- * Frame layout
- * fr[0] - FORKABLE_STACK_HEADER (next pointer)
- * fr[1] - self (used to store return addresses, etc)
- * fr[2...nclosures+2] - closure params
- * fr[nclosures+2..nclosures+nlocals+2] - local variables
- */
-
-static int frame_size(struct bytecode* bc) {
-  return sizeof(union frame_elem) * (bc->nclosures + bc->nlocals + 2);
-}
-
-static struct continuation* frame_self(frame_ptr fr) {
-  return &fr[1].cont;
-}
-
-static struct closure* frame_closure_arg(frame_ptr fr, int closure) {
-  assert(closure >= 0);
-  assert(closure < frame_self(fr)->bc->nclosures);
-  return &fr[2+closure].closure;
-}
-
-static jv* frame_local_var(frame_ptr fr, int var) {
-  assert(var >= 0);
-  assert(var < frame_self(fr)->bc->nlocals);
-  return &fr[2 + frame_self(fr)->bc->nclosures + var].jsonval;
-}
-
-
-static frame_ptr frame_current(struct forkable_stack* stk) {
-  frame_ptr fp = forkable_stack_peek(stk);
-  if (forkable_stack_peek_next(stk, fp)) {
-    struct bytecode* bc = frame_self(forkable_stack_peek_next(stk, fp))->bc;
-    assert(frame_self(fp)->retaddr >= bc->code && frame_self(fp)->retaddr < bc->code + bc->codelen);
-  } else {
-    assert(frame_self(fp)->retaddr == 0);
-  }
-  return fp;
-}
-
-static struct bytecode* frame_current_bytecode(struct forkable_stack* stk) {
-  return frame_self(frame_current(stk))->bc;
-}
-static uint16_t** frame_current_retaddr(struct forkable_stack* stk) {
-  return &frame_self(frame_current(stk))->retaddr;
-}
-
-static frame_ptr frame_get_parent(struct forkable_stack* stk, frame_ptr fr) {
-  return forkable_stack_from_idx(stk, frame_self(fr)->env);
-}
-
-static frame_ptr frame_get_level(struct forkable_stack* stk, frame_ptr fr, int level) {
-  for (int i=0; i<level; i++) {
-    fr = frame_get_parent(stk, fr);
-  }
-  return fr;
-}
-
-static frame_ptr frame_push(struct forkable_stack* stk, struct closure cl, uint16_t* retaddr) {
-  frame_ptr fp = forkable_stack_push(stk, frame_size(cl.bc));
-  struct continuation* cc = frame_self(fp);
-  cc->bc = cl.bc;
-  cc->env = cl.env;
-  cc->retaddr = retaddr;
-  for (int i=0; i<cl.bc->nlocals; i++) {
-    *frame_local_var(fp, i) = jv_invalid();
-  }
-  return fp;
-}
-
-static void frame_pop(struct forkable_stack* stk) {
-  frame_ptr fp = frame_current(stk);
-  if (forkable_stack_pop_will_free(stk)) {
-    int nlocals = frame_self(fp)->bc->nlocals;
-    for (int i=0; i<nlocals; i++) {
-      jv_free(*frame_local_var(fp, i));
-    }
-  }
-  forkable_stack_pop(stk);
-}
-
-#endif

jq.h

@@ -0,0 +1,23 @@
+#ifndef _JQ_H_
+#define _JQ_H_
+
+#include <stdio.h>
+#include <jv.h>
+
+enum {JQ_DEBUG_TRACE = 1};
+
+typedef struct jq_state jq_state;
+typedef void (*jq_err_cb)(void *, jv);
+
+jq_state *jq_init(void);
+void jq_set_error_cb(jq_state *, jq_err_cb, void *);
+void jq_get_error_cb(jq_state *, jq_err_cb *, void **);
+void jq_set_nomem_handler(jq_state *, void (*)(void *), void *);
+int jq_compile(jq_state *, const char* str);
+int jq_compile_args(jq_state *, const char* str, jv args);
+void jq_dump_disassembly(jq_state *, int);
+void jq_start(jq_state *, jv value, int flags);
+jv jq_next(jq_state *);
+void jq_teardown(jq_state **);
+
+#endif /* !_JQ_H_ */

jq.spec

@@ -52,11 +52,16 @@ rm -rf %{buildroot}
 %endif
 %{_datadir}/doc/%{name}/AUTHORS
 %{_datadir}/doc/%{name}/COPYING
-%{_datadir}/doc/%{name}/INSTALL
-%{_datadir}/doc/%{name}/NEWS
 %{_datadir}/doc/%{name}/README
 %{_datadir}/doc/%{name}/README.md
 %{_datadir}/man/man1/jq.1
+%{_includedir}/jq.h
+%{_includedir}/jv.h
+%{_prefix}/lib/libjq.a
+%{_prefix}/lib/libjq.la
+%{_prefix}/lib/libjq.so
+%{_prefix}/lib/libjq.so.1
+%{_prefix}/lib/libjq.so.1.0.4
 
 %changelog
 

jq_parser.h

@@ -1,5 +1,7 @@
 #ifndef JQ_PARSER_H
 #define JQ_PARSER_H
+#include "locfile.h"
+#include "compile.h"
 
 int jq_parse(struct locfile* source, block* answer);
 int jq_parse_library(struct locfile* locations, block* answer);

jq_test.c

@@ -3,15 +3,23 @@
 #include <string.h>
 #include <stdlib.h>
 #include "jv.h"
-#include "execute.h"
+#include "jq.h"
 
 static void jv_test();
 static void run_jq_tests();
 
 
 int jq_testsuite(int argc, char* argv[]) {
+  FILE *testdata = stdin;
   jv_test();
-  run_jq_tests(stdin);
+  if (argc > 2) {
+    testdata = fopen(argv[2], "r");
+    if (!testdata) {
+      perror("fopen");
+      exit(1);
+    }
+  }
+  run_jq_tests(testdata);
   return 0;
 }
 
@@ -27,6 +35,9 @@ static void run_jq_tests(FILE *testdata) {
   int tests = 0, passed = 0, invalid = 0;
   jq_state *jq = NULL;
 
+  jq = jq_init();
+  assert(jq);
+
   while (1) {
     if (!fgets(buf, sizeof(buf), testdata)) break;
     if (skipline(buf)) continue;
@@ -34,15 +45,15 @@ static void run_jq_tests(FILE *testdata) {
     printf("Testing %s\n", buf);
     int pass = 1;
     tests++;
-    struct bytecode* bc = jq_compile(buf);
-    if (!bc) {invalid++; continue;}
+    int compiled = jq_compile(jq, buf);
+    if (!compiled) {invalid++; continue;}
     printf("Disassembly:\n");
-    dump_disassembly(2, bc);
+    jq_dump_disassembly(jq, 2);
     printf("\n");
-    fgets(buf, sizeof(buf), testdata);
+    if (!fgets(buf, sizeof(buf), testdata)) { invalid++; break; }
     jv input = jv_parse(buf);
     if (!jv_is_valid(input)){ invalid++; continue; }
-    jq_init(bc, input, &jq, JQ_DEBUG_TRACE);
+    jq_start(jq, input, JQ_DEBUG_TRACE);
 
     while (fgets(buf, sizeof(buf), testdata)) {
       if (skipline(buf)) break;
@@ -81,10 +92,9 @@ static void run_jq_tests(FILE *testdata) {
         jv_free(extra);
       }
     }
-    jq_teardown(&jq);
-    bytecode_free(bc);
     passed+=pass;
   }
+  jq_teardown(&jq);
   printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
   if (passed != tests) exit(1);
 }
@@ -115,9 +125,9 @@ static void jv_test() {
     jv_free(a2);
 
 
-    assert(a.val.nontrivial.ptr->count == 1);
+    assert(jv_get_refcnt(a) == 1);
     a = jv_array_append(a, jv_copy(a));
-    assert(a.val.nontrivial.ptr->count == 1);
+    assert(jv_get_refcnt(a) == 1);
 
     assert(jv_array_length(jv_copy(a)) == 2);
     assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
@@ -150,9 +160,9 @@ static void jv_test() {
 
     jv_free(subarray);
 
-    void* before = sub2.val.nontrivial.ptr;
+    void* before = sub2.u.ptr;
     sub2 = jv_array_append(sub2, jv_number(200));
-    void* after = sub2.val.nontrivial.ptr;
+    void* after = sub2.u.ptr;
     assert(before == after);
     jv_free(sub2);
 
@@ -172,8 +182,10 @@ static void jv_test() {
     a4 = jv_array_slice(a4, 0, 1);
     assert(jv_array_length(jv_copy(a4)) == 1);
     a4 = jv_array_append(a4, jv_number(4));
-    assert(jv_array_length(a4) == 2);
-    assert(jv_array_length(a5) == 2);
+    assert(jv_array_length(jv_copy(a4)) == 2);
+    assert(jv_array_length(jv_copy(a5)) == 2);
+    jv_free(a4);
+    jv_free(a5);
 
 
     assert(jv_array_length(jv_copy(a)) == 2);
@@ -191,8 +203,10 @@ static void jv_test() {
     assert(jv_equal(jv_string("foo"), jv_string_sized("foo", 3)));
     char nasty[] = "foo\0";
     jv shortstr = jv_string(nasty), longstr = jv_string_sized(nasty, sizeof(nasty));
-    assert(jv_string_length_bytes(shortstr) == (int)strlen(nasty));
-    assert(jv_string_length_bytes(longstr) == (int)sizeof(nasty));
+    assert(jv_string_length_bytes(jv_copy(shortstr)) == (int)strlen(nasty));
+    assert(jv_string_length_bytes(jv_copy(longstr)) == (int)sizeof(nasty));
+    jv_free(shortstr);
+    jv_free(longstr);
 
   
     char a1s[] = "hello", a2s[] = "hello", bs[] = "goodbye";
@@ -238,4 +252,11 @@ static void jv_test() {
 
     jv_free(o2);
   }
+
+  /// Compile errors
+  {
+    jq_state *jq = jq_init();
+    jq_compile_args(jq, "}", jv_array());
+    jq_teardown(&jq);
+  }
 }

jv.h

@@ -1,11 +1,9 @@
 #ifndef JV_H
 #define JV_H
 
+#include <stdarg.h>
 #include <stdint.h>
-#include <assert.h>
-#include <stddef.h>
-
-
+#include <stdio.h>
 
 typedef enum {
   JV_KIND_INVALID,
@@ -18,21 +16,19 @@ typedef enum {
   JV_KIND_OBJECT
 } jv_kind;
 
-typedef struct {
-  size_t count;
-} jv_refcnt;
-
-typedef struct{
-  jv_refcnt* ptr;
-  int i[2];
-} jv_nontrivial;
+struct jv_refcnt;
 
+/* All of the fields of this struct are private.
+   Really. Do not play with them. */
 typedef struct {
-  jv_kind kind;
+  unsigned char kind_flags;
+  unsigned char pad_;
+  unsigned short offset;  /* array offsets */
+  int size;
   union {
+    struct jv_refcnt* ptr;
     double number;
-    jv_nontrivial nontrivial;
-  } val;
+  } u;
 } jv;
 
 /*
@@ -47,24 +43,27 @@ static int jv_is_valid(jv x) { return jv_get_kind(x) != JV_KIND_INVALID; }
 jv jv_copy(jv);
 void jv_free(jv);
 
+int jv_get_refcnt(jv);
+
 int jv_equal(jv, jv);
 int jv_contains(jv, jv);
 
-jv jv_invalid();
+jv jv_invalid(void);
 jv jv_invalid_with_msg(jv);
 jv jv_invalid_get_msg(jv);
 int jv_invalid_has_msg(jv);
 
 
-jv jv_null();
-jv jv_true();
-jv jv_false();
+jv jv_null(void);
+jv jv_true(void);
+jv jv_false(void);
 jv jv_bool(int);
 
 jv jv_number(double);
 double jv_number_value(jv);
+int jv_is_integer(jv);
 
-jv jv_array();
+jv jv_array(void);
 jv jv_array_sized(int);
 int jv_array_length(jv);
 jv jv_array_get(jv, int);
@@ -72,6 +71,7 @@ jv jv_array_set(jv, int, jv);
 jv jv_array_append(jv, jv);
 jv jv_array_concat(jv, jv);
 jv jv_array_slice(jv, int, int);
+jv jv_array_indexes(jv, jv);
 #define jv_array_foreach(a, i, x) \
   for (int jv_len__ = jv_array_length(jv_copy(a)), i=0, jv_j__ = 1;     \
        jv_j__; jv_j__ = 0)                                              \
@@ -82,21 +82,30 @@ jv jv_array_slice(jv, int, int);
 
 jv jv_string(const char*);
 jv jv_string_sized(const char*, int);
+jv jv_string_empty(int len);
 int jv_string_length_bytes(jv);
 int jv_string_length_codepoints(jv);
-uint32_t jv_string_hash(jv);
+unsigned long jv_string_hash(jv);
 const char* jv_string_value(jv);
+jv jv_string_indexes(jv j, jv k);
+jv jv_string_slice(jv j, int start, int end);
 jv jv_string_concat(jv, jv);
+jv jv_string_vfmt(const char*, va_list);
 jv jv_string_fmt(const char*, ...);
+jv jv_string_append_codepoint(jv a, uint32_t c);
 jv jv_string_append_buf(jv a, const char* buf, int len);
 jv jv_string_append_str(jv a, const char* str);
+jv jv_string_split(jv j, jv sep);
+jv jv_string_explode(jv j);
+jv jv_string_implode(jv j);
 
-jv jv_object();
+jv jv_object(void);
 jv jv_object_get(jv object, jv key);
 jv jv_object_set(jv object, jv key, jv value);
 jv jv_object_delete(jv object, jv key);
 int jv_object_length(jv object);
 jv jv_object_merge(jv, jv);
+jv jv_object_merge_recursive(jv, jv);
 
 int jv_object_iter(jv);
 int jv_object_iter_next(jv, int);
@@ -117,20 +126,36 @@ jv jv_object_iter_value(jv, int);
 
 int jv_get_refcnt(jv);
 
-enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4 };
+enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4, JV_PRINT_SORTED = 8 };
+void jv_dumpf(jv, FILE *f, int flags);
 void jv_dump(jv, int flags);
+void jv_show(jv, int flags);
 jv jv_dump_string(jv, int flags);
 
 jv jv_parse(const char* string);
 jv jv_parse_sized(const char* string, int length);
 
+typedef void (*jv_nomem_handler_f)(void *);
+void jv_nomem_handler(jv_nomem_handler_f, void *);
 
+jv jv_load_file(const char *, int);
 
+struct jv_parser;
+struct jv_parser* jv_parser_new(int);
+void jv_parser_set_buf(struct jv_parser*, const char*, int, int);
+jv jv_parser_next(struct jv_parser*);
+void jv_parser_free(struct jv_parser*);
 
-
-
-
-
+jv jv_get(jv, jv);
+jv jv_set(jv, jv, jv);
+jv jv_has(jv, jv);
+jv jv_setpath(jv, jv, jv);
+jv jv_getpath(jv, jv);
+jv jv_delpaths(jv, jv);
+jv jv_keys(jv /*object or array*/);
+int jv_cmp(jv, jv);
+jv jv_group(jv, jv);
+jv jv_sort(jv, jv);
 
 
 #endif

jv_alloc.c

@@ -2,11 +2,121 @@
 #include <stdio.h>
 #include "jv_alloc.h"
 
+struct nomem_handler {
+    jv_nomem_handler_f handler;
+    void *data;
+};
+
+#if !defined(HAVE_PTHREAD_KEY_CREATE) || \
+    !defined(HAVE_PTHREAD_ONCE) || \
+    !defined(HAVE_ATEXIT)
+
+/* Try thread-local storage? */
+
+#ifdef _MSC_VER
+/* Visual C++: yes */
+static __declspec(thread) struct nomem_handler nomem_handler;
+#define USE_TLS
+#else
+#ifdef HAVE___THREAD
+/* GCC and friends: yes */
+static __thread struct nomem_handler nomem_handler;
+#define USE_TLS
+#endif /* HAVE___THREAD */
+#endif /* _MSC_VER */
+
+#endif /* !HAVE_PTHREAD_KEY_CREATE */
+
+#ifdef USE_TLS
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  nomem_handler.handler = handler;
+}
+
+static void memory_exhausted() {
+  if (nomem_handler.handler)
+    nomem_handler.handler(nomem_handler.data); // Maybe handler() will longjmp() to safety
+  // Or not
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+#else /* USE_TLS */
+
+#ifdef HAVE_PTHREAD_KEY_CREATE
+#include <pthread.h>
+
+pthread_key_t nomem_handler_key;
+pthread_once_t mem_once = PTHREAD_ONCE_INIT;
+
+static void tsd_fini(void) {
+  struct nomem_handler *nomem_handler;
+  nomem_handler = pthread_getspecific(nomem_handler_key);
+  if (nomem_handler) {
+    (void) pthread_setspecific(nomem_handler_key, NULL);
+    free(nomem_handler);
+  }
+}
+
+static void tsd_init(void) {
+  if (pthread_key_create(&nomem_handler_key, NULL) != 0) {
+    fprintf(stderr, "error: cannot create thread specific key");
+    abort();
+  }
+  if (atexit(tsd_fini) != 0) {
+    fprintf(stderr, "error: cannot set an exit handler");
+    abort();
+  }
+  struct nomem_handler *nomem_handler = calloc(1, sizeof(struct nomem_handler));
+  if (pthread_setspecific(nomem_handler_key, nomem_handler) != 0) {
+    fprintf(stderr, "error: cannot set thread specific data");
+    abort();
+  }
+}
+
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  pthread_once(&mem_once, tsd_init); // cannot fail
+  struct nomem_handler *nomem_handler;
+
+  nomem_handler = pthread_getspecific(nomem_handler_key);
+  if (nomem_handler == NULL) {
+    handler(data);
+    fprintf(stderr, "error: cannot allocate memory\n");
+    abort();
+  }
+  nomem_handler->handler = handler;
+  nomem_handler->data = data;
+}
+
+static void memory_exhausted() {
+  struct nomem_handler *nomem_handler;
+
+  pthread_once(&mem_once, tsd_init);
+  nomem_handler = pthread_getspecific(nomem_handler_key);
+  if (nomem_handler)
+    nomem_handler->handler(nomem_handler->data); // Maybe handler() will longjmp() to safety
+  // Or not
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+
+#else
+
+/* No thread-local storage of any kind that we know how to handle */
+
+static struct nomem_handler nomem_handler;
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  nomem_handler.handler = handler;
+  nomem_handler.data = data;
+}
+
 static void memory_exhausted() {
   fprintf(stderr, "error: cannot allocate memory\n");
   abort();
 }
 
+#endif /* HAVE_PTHREAD_KEY_CREATE */
+#endif /* USE_TLS */
+
+
 void* jv_mem_alloc(size_t sz) {
   void* p = malloc(sz);
   if (!p) {
@@ -15,6 +125,10 @@ void* jv_mem_alloc(size_t sz) {
   return p;
 }
 
+void* jv_mem_alloc_unguarded(size_t sz) {
+  return malloc(sz);
+}
+
 void jv_mem_free(void* p) {
   free(p);
 }

jv_alloc.h

@@ -2,6 +2,7 @@
 #define JV_ALLOC_H
 
 #include <stddef.h>
+#include "jv.h"
 
 #ifndef NDEBUG
 extern volatile char jv_mem_uninitialised;
@@ -15,6 +16,7 @@ static void jv_mem_invalidate(void* mem, size_t n) {
 }
 
 void* jv_mem_alloc(size_t);
+void* jv_mem_alloc_unguarded(size_t);
 void jv_mem_free(void*);
 __attribute__((warn_unused_result)) void* jv_mem_realloc(void*, size_t);
 

jv_aux.c

@@ -1,17 +1,21 @@
-#include "jv_aux.h"
 #include <string.h>
 #include <stdlib.h>
+#include <assert.h>
 #include "jv_alloc.h"
 
-static int parse_slice(jv array, jv slice, int* pstart, int* pend) {
+static int parse_slice(jv j, jv slice, int* pstart, int* pend) {
   // Array slices
-  int len = jv_array_length(jv_copy(array));
   jv start_jv = jv_object_get(jv_copy(slice), jv_string("start"));
   jv end_jv = jv_object_get(slice, jv_string("end"));
   if (jv_get_kind(start_jv) == JV_KIND_NULL) {
     jv_free(start_jv);
     start_jv = jv_number(0);
   }
+  int len;
+  if (jv_get_kind(j) == JV_KIND_ARRAY)
+    len = jv_array_length(jv_copy(j));
+  else
+    len = jv_string_length_codepoints(jv_copy(j));
   if (jv_get_kind(end_jv) == JV_KIND_NULL) {
     jv_free(end_jv);
     end_jv = jv_number(len);
@@ -22,13 +26,18 @@ static int parse_slice(jv array, jv slice, int* pstart, int* pend) {
     jv_free(end_jv);
     return 0;
   } else {
-    int start = (int)jv_number_value(start_jv);
-    int end = (int)jv_number_value(end_jv);
-    if (start < 0) start = len + start;
-    if (end < 0) end = len + end;
+    double dstart = jv_number_value(start_jv);
+    double dend = jv_number_value(end_jv);
+    if (dstart < 0) dstart += len;
+    if (dend < 0) dend += len;
+    if (dstart < 0) dstart = 0;
+    if (dstart > len) dstart = len;
+
+    int start = (int)dstart;
+    int end = (dend > len) ? len : (int)dend;
+    // Ends are exclusive but e.g. 1 < 1.5 so :1.5 should be :2 not :1
+    if(end < dend) end += 1;
 
-    if (start < 0) start = 0;
-    if (start > len) start = len;
     if (end > len) end = len;
     if (end < start) end = start;
     assert(0 <= start && start <= end && end <= len);
@@ -47,10 +56,15 @@ jv jv_get(jv t, jv k) {
       v = jv_null();
     }
   } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_NUMBER) {
-    // FIXME: don't do lookup for noninteger index
-    v = jv_array_get(t, (int)jv_number_value(k));
-    if (!jv_is_valid(v)) {
-      jv_free(v);
+    if(jv_is_integer(k)){
+      v = jv_array_get(t, (int)jv_number_value(k));
+      if (!jv_is_valid(v)) {
+        jv_free(v);
+        v = jv_null();
+      }
+    } else {
+      jv_free(t);
+      jv_free(k);
       v = jv_null();
     }
   } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_OBJECT) {
@@ -61,6 +75,18 @@ jv jv_get(jv t, jv k) {
       v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
       jv_free(t);
     }
+  } else if (jv_get_kind(t) == JV_KIND_STRING && jv_get_kind(k) == JV_KIND_OBJECT) {
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      v = jv_string_slice(t, start, end);
+    } else {
+      v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an string slice must be numbers"));
+      jv_free(t);
+    }
+  } else if (jv_get_kind(t) == JV_KIND_STRING && jv_get_kind(k) == JV_KIND_STRING) {
+    v = jv_string_indexes(t, k);
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_ARRAY) {
+    v = jv_array_indexes(t, k);
   } else if (jv_get_kind(t) == JV_KIND_NULL && 
              (jv_get_kind(k) == JV_KIND_STRING || 
               jv_get_kind(k) == JV_KIND_NUMBER || 
@@ -545,21 +571,23 @@ jv jv_group(jv objects, jv keys) {
   int n = jv_array_length(jv_copy(objects));
   struct sort_entry* entries = sort_items(objects, keys);
   jv ret = jv_array();
-  jv curr_key = entries[0].key;
-  jv group = jv_array_append(jv_array(), entries[0].object);
-  for (int i = 1; i < n; i++) {
-    if (jv_equal(jv_copy(curr_key), jv_copy(entries[i].key))) {
-      jv_free(entries[i].key);
-    } else {
-      jv_free(curr_key);
-      curr_key = entries[i].key;
-      ret = jv_array_append(ret, group);
-      group = jv_array();
+  if (n > 0) {
+    jv curr_key = entries[0].key;
+    jv group = jv_array_append(jv_array(), entries[0].object);
+    for (int i = 1; i < n; i++) {
+      if (jv_equal(jv_copy(curr_key), jv_copy(entries[i].key))) {
+        jv_free(entries[i].key);
+      } else {
+        jv_free(curr_key);
+        curr_key = entries[i].key;
+        ret = jv_array_append(ret, group);
+        group = jv_array();
+      }
+      group = jv_array_append(group, entries[i].object);
     }
-    group = jv_array_append(group, entries[i].object);
+    jv_free(curr_key);
+    ret = jv_array_append(ret, group);
   }
-  jv_free(curr_key);
-  ret = jv_array_append(ret, group);
   jv_mem_free(entries);
   return ret;
 }

jv_aux.h

@@ -1,20 +0,0 @@
-#ifndef JV_AUX_H
-#define JV_AUX_H
-
-#include "jv.h"
-
-jv jv_get(jv t, jv k);
-jv jv_set(jv t, jv k, jv v);
-jv jv_has(jv t, jv k);
-jv jv_setpath(jv root, jv path, jv value);
-jv jv_getpath(jv root, jv path);
-jv jv_delpaths(jv root, jv paths);
-
-jv jv_keys(jv /*object or array*/);
-int jv_cmp(jv, jv);
-
-jv jv_group(jv objects, jv keys);
-jv jv_sort(jv objects, jv keys);
-
-
-#endif

jv_dtoa.c

@@ -4225,7 +4225,7 @@ jvp_dtoa
 		while((*b++ = *s++));
 		goto done0;
 		}
-	if (decpt <= -4 || decpt > se - s + 5) {
+	if (decpt <= -4 || decpt > se - s + 15) {
 		*b++ = *s++;
 		if (*s) {
 			*b++ = '.';

jv_dtoa.h

@@ -17,6 +17,6 @@ double jvp_strtod(struct dtoa_context* C, const char* s, char** se);
 char* jvp_dtoa(struct dtoa_context* C, double dd, int mode, int ndigits, int *decpt, int *sign, char **rve);
 void jvp_freedtoa(struct dtoa_context* C, char *s);
 
-#define JVP_DTOA_FMT_MAX_LEN 32
+#define JVP_DTOA_FMT_MAX_LEN 64
 char* jvp_dtoa_fmt(struct dtoa_context* C, register char *b, double x);
 #endif

jv_file.c

@@ -0,0 +1,46 @@
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "jv.h"
+
+jv jv_load_file(const char* filename, int raw) {
+  FILE* file = fopen(filename, "r");
+  struct jv_parser* parser;
+  jv data;
+  if (!file) {
+    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
+                                             filename,
+                                             strerror(errno)));
+  }
+  if (raw) {
+    data = jv_string("");
+  } else {
+    data = jv_array();
+    parser = jv_parser_new(0);
+  }
+  while (!feof(file) && !ferror(file)) {
+    char buf[4096];
+    size_t n = fread(buf, 1, sizeof(buf), file);
+    if (raw) {
+      data = jv_string_concat(data, jv_string_sized(buf, (int)n));
+    } else {
+      jv_parser_set_buf(parser, buf, n, !feof(file));
+      jv value;
+      while (jv_is_valid((value = jv_parser_next(parser))))
+        data = jv_array_append(data, value);
+      jv_free(value);
+    }
+  }
+  if (!raw)
+      jv_parser_free(parser);
+  int badread = ferror(file);
+  fclose(file);
+  if (badread) {
+    jv_free(data);
+    return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
+                                             filename));
+  }
+  return data;
+}

jv_parse.c

@@ -1,11 +1,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 #include "jv.h"
 #include "jv_dtoa.h"
-#include "jv_parse.h"
 #include "jv_unicode.h"
 #include "jv_alloc.h"
+#include "jv_dtoa.h"
 
 typedef const char* presult;
 
@@ -16,7 +17,35 @@ typedef const char* presult;
 #define pfunc presult
 #endif
 
-void jv_parser_init(struct jv_parser* p) {
+struct jv_parser {
+  const char* curr_buf;
+  int curr_buf_length;
+  int curr_buf_pos;
+  int curr_buf_is_partial;
+  unsigned bom_strip_position;
+
+  jv* stack;
+  int stackpos;
+  int stacklen;
+  jv next;
+  
+  char* tokenbuf;
+  int tokenpos;
+  int tokenlen;
+
+  int line, column;
+  
+  struct dtoa_context dtoa;
+
+  enum {
+    JV_PARSER_NORMAL,
+    JV_PARSER_STRING,
+    JV_PARSER_STRING_ESCAPE
+  } st;
+};
+
+
+static void parser_init(struct jv_parser* p) {
   p->stack = 0;
   p->stacklen = p->stackpos = 0;
   p->next = jv_invalid();
@@ -31,7 +60,7 @@ void jv_parser_init(struct jv_parser* p) {
   jvp_dtoa_context_init(&p->dtoa);
 }
 
-void jv_parser_free(struct jv_parser* p) {
+static void parser_free(struct jv_parser* p) {
   jv_free(p->next);
   for (int i=0; i<p->stackpos; i++) 
     jv_free(p->stack[i]);
@@ -341,6 +370,17 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
   return answer;
 }
 
+struct jv_parser* jv_parser_new(int flags) {
+  struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser));
+  parser_init(p);
+  return p;
+}
+
+void jv_parser_free(struct jv_parser* p) {
+  parser_free(p);
+  jv_mem_free(p);
+}
+
 static const unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};
 
 void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) {
@@ -404,7 +444,7 @@ jv jv_parser_next(struct jv_parser* p) {
 
 jv jv_parse_sized(const char* string, int length) {
   struct jv_parser parser;
-  jv_parser_init(&parser);
+  parser_init(&parser);
   jv_parser_set_buf(&parser, string, length, 0);
   jv value = jv_parser_next(&parser);
   if (jv_is_valid(value)) {
@@ -429,7 +469,7 @@ jv jv_parse_sized(const char* string, int length) {
     jv_free(value);
     value = jv_invalid_with_msg(jv_string("Expected JSON value"));
   }
-  jv_parser_free(&parser);
+  parser_free(&parser);
 
   if (!jv_is_valid(value) && jv_invalid_has_msg(jv_copy(value))) {
     jv msg = jv_invalid_get_msg(value);

jv_parse.h

@@ -1,37 +0,0 @@
-#ifndef JV_PARSE_H
-#define JV_PARSE_H
-#include "jv_dtoa.h"
-struct jv_parser {
-  const char* curr_buf;
-  int curr_buf_length;
-  int curr_buf_pos;
-  int curr_buf_is_partial;
-  unsigned bom_strip_position;
-
-  jv* stack;
-  int stackpos;
-  int stacklen;
-  jv next;
-  
-  char* tokenbuf;
-  int tokenpos;
-  int tokenlen;
-
-  int line, column;
-  
-  struct dtoa_context dtoa;
-
-  enum {
-    JV_PARSER_NORMAL,
-    JV_PARSER_STRING,
-    JV_PARSER_STRING_ESCAPE
-  } st;
-};
-
-void jv_parser_init(struct jv_parser* p);
-void jv_parser_free(struct jv_parser* p);
-
-void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial);
-
-jv jv_parser_next(struct jv_parser* p);
-#endif

jv_print.c

@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <float.h>
 #include <string.h>
+#include <assert.h>
 
 #include "jv_dtoa.h"
 #include "jv_unicode.h"
@@ -194,7 +195,34 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       put_space(indent+INDENT, F, S);
     }
     int first = 1;
-    jv_object_foreach(x, key, value) {
+    int i = 0;
+    jv keyset = jv_null();
+    while (1) {
+      jv key, value;
+      if (flags & JV_PRINT_SORTED) {
+        if (first) {
+          keyset = jv_keys(jv_copy(x));
+          i = 0;
+        } else {
+          i++;
+        }
+        if (i >= jv_array_length(jv_copy(keyset))) {
+          jv_free(keyset);
+          break;
+        }
+        key = jv_array_get(jv_copy(keyset), i);
+        value = jv_object_get(jv_copy(x), jv_copy(key));
+      } else {
+        if (first) {
+          i = jv_object_iter(x);
+        } else {
+          i = jv_object_iter_next(x, i);
+        }
+        if (!jv_object_iter_valid(x, i)) break;
+        key = jv_object_iter_key(x, i);
+        value = jv_object_iter_value(x, i);
+      }
+
       if (!first) {
         if (flags & JV_PRINT_PRETTY){
           put_str(",\n", F, S);
@@ -232,13 +260,24 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
   }
 }
 
-void jv_dump(jv x, int flags) {
+void jv_dumpf(jv x, FILE *f, int flags) {
   struct dtoa_context C;
   jvp_dtoa_context_init(&C);
-  jv_dump_term(&C, x, flags, 0, stdout, 0);
+  jv_dump_term(&C, x, flags, 0, f, 0);
   jvp_dtoa_context_free(&C);
 }
 
+void jv_dump(jv x, int flags) {
+  jv_dumpf(x, stdout, flags);
+}
+
+void jv_show(jv x, int flags) {
+  if (flags == -1)
+    flags = JV_PRINT_PRETTY | JV_PRINT_COLOUR;
+  jv_dumpf(jv_copy(x), stderr, flags);
+  fflush(stderr);
+}
+
 jv jv_dump_string(jv x, int flags) {
   struct dtoa_context C;
   jvp_dtoa_context_init(&C);

jv_unicode.c

@@ -3,35 +3,60 @@
 #include "jv_unicode.h"
 #include "jv_utf8_tables.h"
 
-const char* jvp_utf8_next(const char* in, const char* end, int* codepoint) {
+const char* jvp_utf8_next(const char* in, const char* end, int* codepoint_ret) {
+  assert(in <= end);
   if (in == end) {
-    codepoint = 0;
     return 0;
   }
+  int codepoint = -1;
   unsigned char first = (unsigned char)in[0];
   int length = utf8_coding_length[first];
-  if (length == 0 || length == UTF8_CONTINUATION_BYTE || in + length > end) {
-    *codepoint = -1;
-    return 0;
-  }
-  *codepoint = ((unsigned)in[0]) & utf8_coding_bits[first];
-  for (int i=1; i<length; i++) {
-    int ch = (unsigned char)in[i];
-    if (utf8_coding_length[(unsigned char)in[i]] != UTF8_CONTINUATION_BYTE){
-      *codepoint = -1;
-      return 0;
+  if ((first & 0x80) == 0) {
+    /* Fast-path for ASCII */
+    codepoint = first;
+    length = 1;
+  } else if (length == 0 || length == UTF8_CONTINUATION_BYTE) {
+    /* Bad single byte - either an invalid byte or an out-of-place continuation byte */
+    length = 1;
+  } else if (in + length > end) {
+    /* String ends before UTF8 sequence ends */
+    length = end - in;
+  } else {
+    codepoint = ((unsigned)in[0]) & utf8_coding_bits[first];
+    for (int i=1; i<length; i++) {
+      unsigned ch = (unsigned char)in[i];
+      if (utf8_coding_length[ch] != UTF8_CONTINUATION_BYTE){
+        /* Invalid UTF8 sequence - not followed by the right number of continuation bytes */
+        codepoint = -1;
+        length = i;
+        break;
+      }
+      codepoint = (codepoint << 6) | (ch & 0x3f);
+    }
+    if (codepoint < utf8_first_codepoint[length]) {
+      /* Overlong UTF8 sequence */
+      codepoint = -1;
+    }
+    if (0xD800 <= codepoint && codepoint <= 0xDFFF) {
+      /* Surrogate codepoints can't be encoded in UTF8 */
+      codepoint = -1;
+    }
+    if (codepoint > 0x10FFFF) {
+      /* Outside Unicode range */
+      codepoint = -1;
     }
-    *codepoint = (*codepoint << 6) | (ch & 0x3f);
   }
+  assert(length > 0);
+  *codepoint_ret = codepoint;
   return in + length;
 }
 
-int jvp_utf8_verify(const char* in, const char* end) {
-  int codepoint = 0;
+int jvp_utf8_is_valid(const char* in, const char* end) {
+  int codepoint;
   while ((in = jvp_utf8_next(in, end, &codepoint))) {
     if (codepoint == -1) return 0;
   }
-  return codepoint != -1;
+  return 1;
 }
 
 int jvp_utf8_encode_length(int codepoint) {

jv_unicode.h

@@ -2,7 +2,7 @@
 #define JV_UNICODE_H
 
 const char* jvp_utf8_next(const char* in, const char* end, int* codepoint);
-
+int jvp_utf8_is_valid(const char* in, const char* end);
 
 int jvp_utf8_decode_length(char startchar);
 

jv_utf8_tables.h

@@ -33,3 +33,5 @@ static const unsigned char utf8_coding_bits[] =
   0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f,
   0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
   0x07, 0x07, 0x07, 0x07, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+static const int utf8_first_codepoint[] =
+ {0x00, 0x00, 0x80, 0x800, 0x10000};

lexer.l

@@ -1,4 +1,5 @@
 %{
+#include <assert.h>
 #include "jv_alloc.h"
 #include "compile.h"
 
@@ -56,10 +57,12 @@ struct lexer_param;
 "-=" { return SETMINUS; }
 "*=" { return SETMULT; }
 "/=" { return SETDIV; }
+"%=" { return SETMOD; }
 "//=" { return SETDEFINEDOR; }
 "<=" { return LESSEQ; }
 ">=" { return GREATEREQ; }
-"."|"="|";"|","|":"|"|"|"+"|"-"|"*"|"/"|"\$"|"<"|">" { return yytext[0];}
+".." { return REC; }
+"."|"?"|"="|";"|","|":"|"|"|"+"|"-"|"*"|"/"|"%"|"\$"|"<"|">" { return yytext[0];}
 
 "["|"{"|"(" {
   return enter(yytext[0], YY_START, yyscanner);
@@ -108,6 +111,7 @@ struct lexer_param;
   
 
 [a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext); return IDENT;}
+\.[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext+1); return FIELD;}
 
 [ \n\t]+  {}
 

libm.h

@@ -0,0 +1,34 @@
+LIBM_DD(acos)
+LIBM_DD(acosh)
+LIBM_DD(asin)
+LIBM_DD(asinh)
+LIBM_DD(atan)
+LIBM_DD(atanh)
+LIBM_DD(cbrt)
+LIBM_DD(cos)
+LIBM_DD(cosh)
+LIBM_DD(exp2)
+LIBM_DD(exp)
+LIBM_DD(floor)
+LIBM_DD(j0)
+LIBM_DD(j1)
+LIBM_DD(log10)
+LIBM_DD(log2)
+LIBM_DD(log)
+LIBM_DD(sin)
+LIBM_DD(sinh)
+LIBM_DD(sqrt)
+LIBM_DD(tan)
+LIBM_DD(tanh)
+LIBM_DD(tgamma)
+LIBM_DD(y0)
+LIBM_DD(y1)
+/* LIBM_DID(jn) */
+/* LIBM_DID(yn) */
+/* LIBM_DDD(pow) */
+/* LIBM_DDD(atan2) */
+/* LIBM_DDD(hypot) */
+/* LIBM_DDD(remainder) */
+/* LIBM_DDI(scalbn) */
+/* LIBM_DDIP(lgamma_r) */
+

locfile.c

@@ -0,0 +1,103 @@
+#include <assert.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "jq.h"
+#include "jv_alloc.h"
+#include "locfile.h"
+
+
+void locfile_init(struct locfile* l, jq_state *jq, const char* data, int length) {
+  l->jq = jq;
+  l->data = data;
+  l->length = length;
+  l->nlines = 1;
+  for (int i=0; i<length; i++) {
+    if (data[i] == '\n') l->nlines++;
+  }
+  l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
+  l->linemap[0] = 0;
+  int line = 1;
+  for (int i=0; i<length; i++) {
+    if (data[i] == '\n') {
+      l->linemap[line] = i+1;   // at start of line, not of \n
+      line++;
+    }
+  }
+  l->linemap[l->nlines] = length+1;   // virtual last \n
+}
+
+void locfile_free(struct locfile* l) {
+  jv_mem_free(l->linemap);
+}
+
+static int locfile_get_line(struct locfile* l, int pos) {
+  assert(pos < l->length);
+  int line = 1;
+  while (l->linemap[line] <= pos) line++;   // == if pos at start (before, never ==, because pos never on \n)
+  assert(line-1 < l->nlines);
+  return line-1;
+}
+
+static int locfile_line_length(struct locfile* l, int line) {
+  assert(line < l->nlines);
+  return l->linemap[line+1] - l->linemap[line] -1;   // -1 to omit \n
+}
+
+void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
+  jq_err_cb cb;
+  void *cb_data;
+  va_list fmtargs;
+  va_start(fmtargs, fmt);
+  int startline;
+  int offset;
+
+  if (loc.start != -1) {
+    startline = locfile_get_line(l, loc.start);
+    offset = l->linemap[startline];
+  }
+
+  jq_get_error_cb(l->jq, &cb, &cb_data);
+
+  jv m1 = jv_string_vfmt(fmt, fmtargs);
+  if (!jv_is_valid(m1)) {
+    jv_free(m1);
+    goto enomem;
+  }
+  jv m2;
+  if (loc.start == -1) {
+    m2 = jv_string_fmt("%s\n<unknown location>", jv_string_value(m1));
+    if (cb)
+      cb(cb_data, m2);
+    else
+      fprintf(stderr, "%s", jv_string_value(m2));
+    jv_free(m1);
+    jv_free(m2);
+    return;
+  }
+  m2 = jv_string_fmt("%s\n%.*s%*s", jv_string_value(m1),
+                     locfile_line_length(l, startline), l->data + offset,
+                     loc.start - offset, "");
+  jv_free(m1);
+  if (!jv_is_valid(m2)) {
+    jv_free(m2);
+    goto enomem;
+  }
+  if (cb)
+    cb(cb_data, m2);
+  else
+    fprintf(stderr, "%s", jv_string_value(m2));
+  jv_free(m2);
+  return;
+
+enomem:
+  if (cb != NULL)
+    cb(cb_data, jv_invalid());
+  else if (errno == ENOMEM || errno == 0)
+    fprintf(stderr, "Error formatting jq compilation error: %s", strerror(errno ? errno : ENOMEM));
+  else
+    fprintf(stderr, "Error formatting jq compilation error: %s", strerror(errno));
+  return;
+}

locfile.h

@@ -1,10 +1,8 @@
 #ifndef _LOCFILE_H
 #define _LOCFILE_H
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include <stdarg.h>
-#include "jv_alloc.h"
+
+#include "jq.h"
+
 typedef struct {
   int start, end;
 } location;
@@ -16,64 +14,14 @@ struct locfile {
   int length;
   int* linemap;
   int nlines;
+  char *error;
+  jq_state *jq;
 };
 
-static void locfile_init(struct locfile* l, const char* data, int length) {
-  l->data = data;
-  l->length = length;
-  l->nlines = 1;
-  for (int i=0; i<length; i++) {
-    if (data[i] == '\n') l->nlines++;
-  }
-  l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
-  l->linemap[0] = 0;
-  int line = 1;
-  for (int i=0; i<length; i++) {
-    if (data[i] == '\n') {
-      l->linemap[line] = i;
-      line++;
-    }
-  }
-  l->linemap[l->nlines] = length;
-}
+void locfile_init(struct locfile* l, jq_state *jq, const char* data, int length);
 
-static void locfile_free(struct locfile* l) {
-  jv_mem_free(l->linemap);
-}
+void locfile_free(struct locfile* l);
 
-static int locfile_get_line(struct locfile* l, int pos) {
-  assert(pos < l->length);
-  int line = 0;
-  while (l->linemap[line+1] < pos) line++;
-  assert(line < l->nlines);
-  return line;
-}
-
-static int locfile_line_length(struct locfile* l, int line) {
-  assert(line < l->nlines);
-  return l->linemap[line+1] - l->linemap[line];
-}
-
-static void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
-  va_list fmtargs;
-  va_start(fmtargs, fmt);
-  vfprintf(stderr, fmt, fmtargs);
-  va_end(fmtargs);
-  fprintf(stderr, "\n");
-  if (loc.start == -1) {
-    fprintf(stderr, "<unknown location>\n");
-    return;
-  }
-  int startline = locfile_get_line(l, loc.start);
-  int offset = l->linemap[startline];
-  fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset);
-  fprintf(stderr, "%*s", loc.start - offset, "");
-  for (int i = loc.start; 
-       i < loc.end && i < offset + locfile_line_length(l, startline);
-       i++){
-    fprintf(stderr, "^");
-  }
-  fprintf(stderr, "\n");
-}
+void locfile_locate(struct locfile* l, location loc, const char* fmt, ...);
 
 #endif

main.c

@@ -1,3 +1,4 @@
+#include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
@@ -5,29 +6,28 @@
 #include <unistd.h>
 #include "compile.h"
 #include "jv.h"
-#include "jv_parse.h"
-#include "execute.h"
-#include "config.h"  /* Autoconf generated header file */
+#include "jq.h"
 #include "jv_alloc.h"
+#include "version.h"
 
 int jq_testsuite(int argc, char* argv[]);
 
 static const char* progname;
 
 static void usage() {
-  fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", PACKAGE_VERSION);
+  fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", JQ_VERSION);
   fprintf(stderr, "Usage: %s [options] <jq filter> [file...]\n\n", progname);
   fprintf(stderr, "For a description of the command line options and\n");
   fprintf(stderr, "how to write jq filters (and why you might want to)\n");
   fprintf(stderr, "see the jq manpage, or the online documentation at\n");
-  fprintf(stderr, "http://stedolan.github.io/jq\n\n");
-  exit(1);
+  fprintf(stderr, "http://stedolan.github.com/jq\n\n");
+  exit(2);
 }
 
 static void die() {
   fprintf(stderr, "Use %s --help for help with command-line options,\n", progname);
-  fprintf(stderr, "or see the jq documentation at http://stedolan.github.io/jq\n");
-  exit(1);
+  fprintf(stderr, "or see the jq documentation at http://stedolan.github.com/jq\n");
+  exit(2);
 }
 
 
@@ -54,22 +54,26 @@ enum {
   ASCII_OUTPUT = 32,
   COLOUR_OUTPUT = 64,
   NO_COLOUR_OUTPUT = 128,
+  SORTED_OUTPUT = 256,
+  UNBUFFERED_OUTPUT = 4096,
 
-  FROM_FILE = 256,
+  FROM_FILE = 512,
+
+  EXIT_STATUS = 8192,
 
   /* debugging only */
   DUMP_DISASM = 2048,
 };
 static int options = 0;
-static struct bytecode* bc;
 
-static void process(jv value, int flags) {
-  jq_state *jq = NULL;
-  jq_init(bc, value, &jq, flags);
+static int process(jq_state *jq, jv value, int flags) {
+  int ret = 14; // No valid results && -e -> exit(4)
+  jq_start(jq, value, flags);
   jv result;
   while (jv_is_valid(result = jq_next(jq))) {
     if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
       fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
+      ret = 0;
       jv_free(result);
     } else {
       int dumpopts;
@@ -80,43 +84,27 @@ static void process(jv value, int flags) {
 #else
       dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0;
 #endif
+      if (options & SORTED_OUTPUT) dumpopts |= JV_PRINT_SORTED;
       if (!(options & COMPACT_OUTPUT)) dumpopts |= JV_PRINT_PRETTY;
       if (options & ASCII_OUTPUT) dumpopts |= JV_PRINT_ASCII;
       if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR;
       if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR;
+      if (jv_get_kind(result) == JV_KIND_FALSE || jv_get_kind(result) == JV_KIND_NULL)
+        ret = 11;
+      else
+        ret = 0;
       jv_dump(result, dumpopts);
     }
     printf("\n");
+    if (options & UNBUFFERED_OUTPUT)
+      fflush(stdout);
   }
   jv_free(result);
-  jq_teardown(&jq);
-}
-
-static jv slurp_file(const char* filename) {
-  FILE* file = fopen(filename, "r");
-  if (!file) {
-    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
-                                             filename,
-                                             strerror(errno)));
-  }
-  jv data = jv_string("");
-  while (!feof(file) && !ferror(file)) {
-    char buf[4096];
-    size_t n = fread(buf, 1, sizeof(buf), file);
-    data = jv_string_concat(data, jv_string_sized(buf, (int)n));
-  }
-  int badread = ferror(file);
-  fclose(file);
-  if (badread) {
-    jv_free(data);
-    return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
-                                             filename));
-  }
-  return data;
+  return ret;
 }
 
 FILE* current_input;
-const char** input_filenames;
+const char** input_filenames = NULL;
 int ninput_files;
 int next_input_idx;
 static int read_more(char* buf, size_t size) {
@@ -144,11 +132,21 @@ static int read_more(char* buf, size_t size) {
 }
 
 int main(int argc, char* argv[]) {
+  jq_state *jq = NULL;
   int ret = 0;
+  int compiled = 0;
+
   if (argc) progname = argv[0];
 
   if (argc > 1 && !strcmp(argv[1], "--run-tests")) {
-    return jq_testsuite(argc - 1, argv + 1);
+    return jq_testsuite(argc, argv);
+  }
+
+  jq = jq_init();
+  if (jq == NULL) {
+    perror("malloc");
+    ret = 2;
+    goto out;
   }
 
   const char* program = 0;
@@ -181,12 +179,18 @@ int main(int argc, char* argv[]) {
       options |= NO_COLOUR_OUTPUT;
     } else if (isoption(argv[i], 'a', "ascii-output")) {
       options |= ASCII_OUTPUT;
+    } else if (isoption(argv[i], 0, "unbuffered")) {
+      options |= UNBUFFERED_OUTPUT;
+    } else if (isoption(argv[i], 'S', "sort-keys")) {
+      options |= SORTED_OUTPUT;
     } else if (isoption(argv[i], 'R', "raw-input")) {
       options |= RAW_INPUT;
     } else if (isoption(argv[i], 'n', "null-input")) {
       options |= PROVIDE_NULL;
     } else if (isoption(argv[i], 'f', "from-file")) {
       options |= FROM_FILE;
+    } else if (isoption(argv[i], 'e', "exit-status")) {
+      options |= EXIT_STATUS;
     } else if (isoption(argv[i], 0, "arg")) {
       if (i >= argc - 2) {
         fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname);
@@ -197,6 +201,27 @@ int main(int argc, char* argv[]) {
       arg = jv_object_set(arg, jv_string("value"), jv_string(argv[i+2]));
       program_arguments = jv_array_append(program_arguments, arg);
       i += 2; // skip the next two arguments
+    } else if (isoption(argv[i], 0, "argfile")) {
+      if (i >= argc - 2) {
+        fprintf(stderr, "%s: --argfile takes two parameters (e.g. -a varname filename)\n", progname);
+        die();
+      }
+      jv arg = jv_object();
+      arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
+      jv data = jv_load_file(argv[i+2], 0);
+      if (!jv_is_valid(data)) {
+        data = jv_invalid_get_msg(data);
+        fprintf(stderr, "%s: Bad JSON in --argfile %s %s: %s\n", progname,
+                argv[i+1], argv[i+2], jv_string_value(data));
+        jv_free(data);
+        ret = 2;
+        goto out;
+      }
+      if (jv_get_kind(data) == JV_KIND_ARRAY && jv_array_length(jv_copy(data)) == 1)
+          data = jv_array_get(data, 0);
+      arg = jv_object_set(arg, jv_string("value"), data);
+      program_arguments = jv_array_append(program_arguments, arg);
+      i += 2; // skip the next two arguments
     } else if (isoption(argv[i],  0,  "debug-dump-disasm")) {
       options |= DUMP_DISASM;
     } else if (isoption(argv[i],  0,  "debug-trace")) {
@@ -204,8 +229,9 @@ int main(int argc, char* argv[]) {
     } else if (isoption(argv[i], 'h', "help")) {
       usage();
     } else if (isoption(argv[i], 'V', "version")) {
-      fprintf(stderr, "jq version %s\n", PACKAGE_VERSION);
-      return 0;
+      printf("jq-%s\n", JQ_VERSION);
+      ret = 0;
+      goto out;
     } else {
       fprintf(stderr, "%s: Unknown option %s\n", progname, argv[i]);
       die();
@@ -220,27 +246,31 @@ int main(int argc, char* argv[]) {
   }
   
   if (options & FROM_FILE) {
-    jv data = slurp_file(program);
+    jv data = jv_load_file(program, 1);
     if (!jv_is_valid(data)) {
       data = jv_invalid_get_msg(data);
       fprintf(stderr, "%s: %s\n", progname, jv_string_value(data));
       jv_free(data);
-      return 1;
+      ret = 2;
+      goto out;
     }
-    bc = jq_compile_args(jv_string_value(data), program_arguments);
+    compiled = jq_compile_args(jq, jv_string_value(data), program_arguments);
     jv_free(data);
   } else {
-    bc = jq_compile_args(program, program_arguments);
+    compiled = jq_compile_args(jq, program, program_arguments);
+  }
+  if (!compiled){
+    ret = 3;
+    goto out;
   }
-  if (!bc) return 1;
 
   if (options & DUMP_DISASM) {
-    dump_disassembly(0, bc);
+    jq_dump_disassembly(jq, 0);
     printf("\n");
   }
 
   if (options & PROVIDE_NULL) {
-    process(jv_null(), jq_flags);
+    ret = process(jq, jv_null(), jq_flags);
   } else {
     jv slurped;
     if (options & SLURP) {
@@ -250,8 +280,7 @@ int main(int argc, char* argv[]) {
         slurped = jv_array();
       }
     }
-    struct jv_parser parser;
-    jv_parser_init(&parser);
+    struct jv_parser* parser = jv_parser_new(0);
     char buf[4096];
     while (read_more(buf, sizeof(buf))) {
       if (options & RAW_INPUT) {
@@ -261,39 +290,43 @@ int main(int argc, char* argv[]) {
             slurped = jv_string_concat(slurped, jv_string(buf));
           } else {
             if (buf[len-1] == '\n') buf[len-1] = 0;
-            process(jv_string(buf), jq_flags);
+            ret = process(jq, jv_string(buf), jq_flags);
           }
         }
       } else {
-        jv_parser_set_buf(&parser, buf, strlen(buf), !feof(stdin));
+        jv_parser_set_buf(parser, buf, strlen(buf), !feof(stdin));
         jv value;
-        while (jv_is_valid((value = jv_parser_next(&parser)))) {
+        while (jv_is_valid((value = jv_parser_next(parser)))) {
           if (options & SLURP) {
             slurped = jv_array_append(slurped, value);
           } else {
-            process(value, jq_flags);
+            ret = process(jq, value, jq_flags);
           }
         }
         if (jv_invalid_has_msg(jv_copy(value))) {
           jv msg = jv_invalid_get_msg(value);
           fprintf(stderr, "parse error: %s\n", jv_string_value(msg));
           jv_free(msg);
-          ret = 1;
+          ret = 4;
           break;
         } else {
           jv_free(value);
         }
       }
     }
-    jv_parser_free(&parser);
+    jv_parser_free(parser);
     if (ret != 0)
       goto out;
     if (options & SLURP) {
-      process(slurped, jq_flags);
+      ret = process(jq, slurped, jq_flags);
     }
   }
 out:
   jv_mem_free(input_filenames);
-  bytecode_free(bc);
+  jq_teardown(&jq);
+  if (ret >= 10 && (options & EXIT_STATUS))
+    return ret - 10;
+  if (ret >= 10)
+    return 0;
   return ret;
 }

opcode.c

@@ -1,31 +0,0 @@
-#include "opcode.h"
-
-// flags, length
-#define NONE 0, 1
-#define CONSTANT OP_HAS_CONSTANT, 2
-#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
-#define BRANCH OP_HAS_BRANCH, 2
-#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
-#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
-#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
-#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
-
-#define OP(name, imm, in, out) \
-  {name, #name, imm, in, out},
-
-static const struct opcode_description opcode_descriptions[] = {
-#include "opcode_list.h"
-};
-
-static const struct opcode_description invalid_opcode_description = {
-  -1, "#INVALID", 0, 0, 0, 0
-};
-
-
-const struct opcode_description* opcode_describe(opcode op) {
-  if ((int)op >= 0 && (int)op < NUM_OPCODES) {
-    return &opcode_descriptions[op];
-  } else {
-    return &invalid_opcode_description;
-  }
-}

opcode.h

@@ -1,41 +0,0 @@
-#ifndef OPCODE_H
-#define OPCODE_H
-#include <assert.h>
-
-typedef enum {
-#define OP(name, imm, in, out) name,
-#include "opcode_list.h"
-#undef OP
-} opcode;
-
-enum {
-  NUM_OPCODES = 
-#define OP(name, imm, in, out) +1
-#include "opcode_list.h"
-#undef OP
-};
-
-enum {
-  OP_HAS_CONSTANT = 2,
-  OP_HAS_VARIABLE = 4,
-  OP_HAS_BRANCH = 8,
-  OP_HAS_CFUNC = 32,
-  OP_HAS_UFUNC = 64,
-  OP_IS_CALL_PSEUDO = 128,
-  OP_HAS_BINDING = 1024,
-};
-struct opcode_description {
-  opcode op;
-  const char* name;
-
-  int flags;
-
-  // length in 16-bit units
-  int length;
-
-  int stack_in, stack_out;
-};
-
-const struct opcode_description* opcode_describe(opcode op);
-
-#endif

opcode_list.h

@@ -6,7 +6,9 @@ OP(LOADV, VARIABLE, 1, 1)
 OP(LOADVN, VARIABLE, 1, 1)
 OP(STOREV, VARIABLE, 1, 0)
 OP(INDEX, NONE,     2, 1)
+OP(INDEX_OPT, NONE,     2, 1)
 OP(EACH,  NONE,     1, 1)
+OP(EACH_OPT,  NONE,     1, 1)
 OP(FORK,  BRANCH,   0, 0)
 OP(JUMP,  BRANCH,   0, 0)
 OP(JUMP_F,BRANCH,   1, 0)

parser.y

@@ -1,16 +1,16 @@
 %{
 #include <stdio.h>
 #include <string.h>
+#include <assert.h>
 #include "compile.h"
 #include "jv_alloc.h"
 #define YYMALLOC jv_mem_alloc
 #define YYFREE jv_mem_free
-
-struct lexer_param;
-
 %}
 %code requires {
 #include "locfile.h"
+struct lexer_param;
+
 #define YYLTYPE location
 #define YYLLOC_DEFAULT(Loc, Rhs, N)             \
   do {                                          \
@@ -47,8 +47,12 @@ struct lexer_param;
 
 %token INVALID_CHARACTER
 %token <literal> IDENT
+%token <literal> FIELD
 %token <literal> LITERAL
 %token <literal> FORMAT
+%token Q "?"
+%token REC ".."
+%token SETMOD "%="
 %token EQ "=="
 %token NEQ "!="
 %token DEFINEDOR "//"
@@ -82,12 +86,12 @@ struct lexer_param;
 %right '|'
 %left ','
 %right "//"
-%nonassoc '=' SETPIPE SETPLUS SETMINUS SETMULT SETDIV SETDEFINEDOR
+%nonassoc '=' SETPIPE SETPLUS SETMINUS SETMULT SETDIV SETMOD SETDEFINEDOR
 %left OR
 %left AND
 %nonassoc NEQ EQ '<' '>' LESSEQ GREATEREQ
 %left '+' '-'
-%left '*' '/'
+%left '*' '/' '%'
 
 
 %type <blk> Exp Term MkDict MkDictPair ExpD ElseBody QQString FuncDef FuncDefs String
@@ -112,25 +116,19 @@ void yyerror(YYLTYPE* loc, block* answer, int* errors,
 int yylex(YYSTYPE* yylval, YYLTYPE* yylloc, block* answer, int* errors, 
           struct locfile* locations, struct lexer_param* lexer_param_ptr) {
   yyscan_t lexer = lexer_param_ptr->lexer;
-  while (1) {
-    int tok = jq_yylex(yylval, yylloc, lexer);
-    if (tok == INVALID_CHARACTER) {
-      FAIL(*yylloc, "Invalid character");
+  int tok = jq_yylex(yylval, yylloc, lexer);
+  if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
+    jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
+    if (jv_get_kind(msg) == JV_KIND_STRING) {
+      FAIL(*yylloc, jv_string_value(msg));
     } else {
-      if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
-        jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
-        if (jv_get_kind(msg) == JV_KIND_STRING) {
-          FAIL(*yylloc, jv_string_value(msg));
-        } else {
-          FAIL(*yylloc, "Invalid literal");
-        }
-        jv_free(msg);
-        jv_free(yylval->literal);
-        yylval->literal = jv_null();
-      }
-      return tok;
+      FAIL(*yylloc, "Invalid literal");
     }
+    jv_free(msg);
+    jv_free(yylval->literal);
+    yylval->literal = jv_null();
   }
+  return tok;
 }
 
 static block gen_dictpair(block k, block v) {
@@ -141,7 +139,11 @@ static block gen_index(block obj, block key) {
   return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX));
 }
 
-static block gen_slice_index(block obj, block start, block end) {
+static block gen_index_opt(block obj, block key) {
+  return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX_OPT));
+}
+
+static block gen_slice_index(block obj, block start, block end, opcode idx_op) {
   block key = BLOCK(gen_subexp(gen_const(jv_object())),
                     gen_subexp(gen_const(jv_string("start"))),
                     gen_subexp(start),
@@ -149,7 +151,7 @@ static block gen_slice_index(block obj, block start, block end) {
                     gen_subexp(gen_const(jv_string("end"))),
                     gen_subexp(end),
                     gen_op_simple(INSERT));
-  return BLOCK(key, obj, gen_op_simple(INDEX));
+  return BLOCK(key, obj, gen_op_simple(idx_op));
 }
 
 static block gen_binop(block a, block b, int op) {
@@ -159,6 +161,7 @@ static block gen_binop(block a, block b, int op) {
   case '-': funcname = "_minus"; break;
   case '*': funcname = "_multiply"; break;
   case '/': funcname = "_divide"; break;
+  case '%': funcname = "_mod"; break;
   case EQ: funcname = "_equal"; break;
   case NEQ: funcname = "_notequal"; break;
   case '<': funcname = "_less"; break;
@@ -176,24 +179,22 @@ static block gen_format(block a, jv fmt) {
 }
 
 static block gen_definedor_assign(block object, block val) {
-  block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
-                         gen_noop(), OP_HAS_VARIABLE);
+  block tmp = gen_op_var_fresh(STOREV, "tmp");
   return BLOCK(gen_op_simple(DUP),
                val, tmp,
                gen_call("_modify", BLOCK(gen_lambda(object),
                                          gen_lambda(gen_definedor(gen_noop(), 
-                                                                  gen_op_var_bound(LOADV, tmp))))));
+                                                                  gen_op_bound(LOADV, tmp))))));
 }
  
 static block gen_update(block object, block val, int optype) {
-  block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
-                         gen_noop(), OP_HAS_VARIABLE);
+  block tmp = gen_op_var_fresh(STOREV, "tmp");
   return BLOCK(gen_op_simple(DUP),
                val,
                tmp,
                gen_call("_modify", BLOCK(gen_lambda(object), 
                                          gen_lambda(gen_binop(gen_noop(),
-                                                              gen_op_var_bound(LOADV, tmp),
+                                                              gen_op_bound(LOADV, tmp),
                                                               optype)))));
 }
 
@@ -234,8 +235,8 @@ Term "as" '$' IDENT '|' Exp {
 "if" Exp "then" Exp ElseBody {
   $$ = gen_cond($2, $4, $5);
 } |
-"if" Exp error {
-  FAIL(@$, "Possibly unterminated 'if' statment");
+"if" Exp "then" error {
+  FAIL(@$, "Possibly unterminated 'if' statement");
   $$ = $2;
 } |
 
@@ -303,10 +304,18 @@ Exp '/' Exp {
   $$ = gen_binop($1, $3, '/');
 } |
 
+Exp '%' Exp {
+  $$ = gen_binop($1, $3, '%');
+} |
+
 Exp "/=" Exp {
   $$ = gen_update($1, $3, '/');
 } |
 
+Exp SETMOD Exp {
+  $$ = gen_update($1, $3, '%');
+} |
+
 Exp "==" Exp {
   $$ = gen_binop($1, $3, EQ);
 } |
@@ -343,7 +352,7 @@ FuncDef:
 
 "def" IDENT '(' IDENT ')' ':' Exp ';' {
   $$ = gen_function(jv_string_value($2), 
-                    gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), 
+                    gen_param(jv_string_value($4)), 
                     $7);
   jv_free($2);
   jv_free($4);
@@ -351,14 +360,73 @@ FuncDef:
 
 "def" IDENT '(' IDENT ';' IDENT ')' ':' Exp ';' {
   $$ = gen_function(jv_string_value($2), 
-                    BLOCK(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), 
-                          gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($6))),
+                    BLOCK(gen_param(jv_string_value($4)), 
+                          gen_param(jv_string_value($6))),
                     $9);
   jv_free($2);
   jv_free($4);
   jv_free($6);
-}
+} |
 
+"def" IDENT '(' IDENT ';' IDENT ';' IDENT ')' ':' Exp ';' {
+  $$ = gen_function(jv_string_value($2), 
+                    BLOCK(gen_param(jv_string_value($4)), 
+                          gen_param(jv_string_value($6)),
+                          gen_param(jv_string_value($8))),
+                    $11);
+  jv_free($2);
+  jv_free($4);
+  jv_free($6);
+  jv_free($8);
+} |
+
+"def" IDENT '(' IDENT ';' IDENT ';' IDENT ';' IDENT ')' ':' Exp ';' {
+  $$ = gen_function(jv_string_value($2), 
+                    BLOCK(gen_param(jv_string_value($4)), 
+                          gen_param(jv_string_value($6)),
+                          gen_param(jv_string_value($8)),
+                          gen_param(jv_string_value($10))),
+                    $13);
+  jv_free($2);
+  jv_free($4);
+  jv_free($6);
+  jv_free($8);
+  jv_free($10);
+} |
+
+"def" IDENT '(' IDENT ';' IDENT ';' IDENT ';' IDENT ';' IDENT ')' ':' Exp ';' {
+  $$ = gen_function(jv_string_value($2), 
+                    BLOCK(gen_param(jv_string_value($4)), 
+                          gen_param(jv_string_value($6)),
+                          gen_param(jv_string_value($8)),
+                          gen_param(jv_string_value($10)),
+                          gen_param(jv_string_value($12))),
+                    $15);
+  jv_free($2);
+  jv_free($4);
+  jv_free($6);
+  jv_free($8);
+  jv_free($10);
+  jv_free($12);
+} |
+
+"def" IDENT '(' IDENT ';' IDENT ';' IDENT ';' IDENT ';' IDENT ';' IDENT ')' ':' Exp ';' {
+  $$ = gen_function(jv_string_value($2), 
+                    BLOCK(gen_param(jv_string_value($4)), 
+                          gen_param(jv_string_value($6)),
+                          gen_param(jv_string_value($8)),
+                          gen_param(jv_string_value($10)),
+                          gen_param(jv_string_value($12)),
+                          gen_param(jv_string_value($14))),
+                    $17);
+  jv_free($2);
+  jv_free($4);
+  jv_free($6);
+  jv_free($8);
+  jv_free($10);
+  jv_free($12);
+  jv_free($14);
+}
 
 
 String:
@@ -396,7 +464,9 @@ ExpD:
 ExpD '|' ExpD { 
   $$ = block_join($1, $3);
 } |
-
+'-' ExpD {
+  $$ = BLOCK($2, gen_call("_negate", gen_noop()));
+} |
 Term {
   $$ = $1;
 }
@@ -406,27 +476,72 @@ Term:
 '.' {
   $$ = gen_noop(); 
 } |
-Term '.' IDENT {
-  $$ = gen_index($1, gen_const($3)); 
+REC {
+  $$ = gen_call("recurse_down", gen_noop());
 } |
-'.' IDENT { 
-  $$ = gen_index(gen_noop(), gen_const($2)); 
+Term FIELD '?' {
+  $$ = gen_index_opt($1, gen_const($2));
 } |
+FIELD '?' { 
+  $$ = gen_index_opt(gen_noop(), gen_const($1)); 
+} |
+Term '.' String '?' {
+  $$ = gen_index_opt($1, $3);
+} |
+'.' String '?' {
+  $$ = gen_index_opt(gen_noop(), $2);
+} |
+Term FIELD {
+  $$ = gen_index($1, gen_const($2));
+} |
+FIELD { 
+  $$ = gen_index(gen_noop(), gen_const($1)); 
+} |
+Term '.' String {
+  $$ = gen_index($1, $3);
+} |
+'.' String {
+  $$ = gen_index(gen_noop(), $2);
+} |
+'.' error {
+  FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
+  $$ = gen_noop();
+} |
+'.' IDENT error {
+  jv_free($2);
+  FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
+  $$ = gen_noop();
+} | 
 /* FIXME: string literals */
+Term '[' Exp ']' '?' {
+  $$ = gen_index_opt($1, $3); 
+} |
 Term '[' Exp ']' {
   $$ = gen_index($1, $3); 
 } |
+Term '[' ']' '?' {
+  $$ = block_join($1, gen_op_simple(EACH_OPT)); 
+} |
 Term '[' ']' {
   $$ = block_join($1, gen_op_simple(EACH)); 
 } |
+Term '[' Exp ':' Exp ']' '?' {
+  $$ = gen_slice_index($1, $3, $5, INDEX_OPT);
+} |
+Term '[' Exp ':' ']' '?' {
+  $$ = gen_slice_index($1, $3, gen_const(jv_null()), INDEX_OPT);
+} |
+Term '[' ':' Exp ']' '?' {
+  $$ = gen_slice_index($1, gen_const(jv_null()), $4, INDEX_OPT);
+} |
 Term '[' Exp ':' Exp ']' {
-  $$ = gen_slice_index($1, $3, $5);
+  $$ = gen_slice_index($1, $3, $5, INDEX);
 } |
 Term '[' Exp ':' ']' {
-  $$ = gen_slice_index($1, $3, gen_const(jv_null()));
+  $$ = gen_slice_index($1, $3, gen_const(jv_null()), INDEX);
 } |
 Term '[' ':' Exp ']' {
-  $$ = gen_slice_index($1, gen_const(jv_null()), $4);
+  $$ = gen_slice_index($1, gen_const(jv_null()), $4, INDEX);
 } |
 LITERAL {
   $$ = gen_const($1); 
@@ -450,7 +565,7 @@ FORMAT {
   $$ = BLOCK(gen_subexp(gen_const(jv_object())), $2, gen_op_simple(POP));
 } |
 '$' IDENT {
-  $$ = gen_location(@$, gen_op_var_unbound(LOADV, jv_string_value($2)));
+  $$ = gen_location(@$, gen_op_unbound(LOADV, jv_string_value($2)));
   jv_free($2);
 } | 
 IDENT {
@@ -467,6 +582,26 @@ IDENT '(' Exp ';' Exp ')' {
   $$ = gen_location(@1, $$);
   jv_free($1);
 } |
+IDENT '(' Exp ';' Exp ';' Exp ')' {
+  $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7)));
+  $$ = gen_location(@1, $$);
+  jv_free($1);
+} |
+IDENT '(' Exp ';' Exp ';' Exp ';' Exp ')' {
+  $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9)));
+  $$ = gen_location(@1, $$);
+  jv_free($1);
+} |
+IDENT '(' Exp ';' Exp ';' Exp ';' Exp ';' Exp ')' {
+  $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9), gen_lambda($11)));
+  $$ = gen_location(@1, $$);
+  jv_free($1);
+} |
+IDENT '(' Exp ';' Exp ';' Exp ';' Exp ';' Exp ';' Exp ')' {
+  $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9), gen_lambda($11), gen_lambda($13)));
+  $$ = gen_location(@1, $$);
+  jv_free($1);
+} |
 '(' error ')' { $$ = gen_noop(); } |
 '[' error ']' { $$ = gen_noop(); } |
 Term '[' error ']' { $$ = $1; } |

scripts/crosscompile

@@ -13,6 +13,11 @@
 set -e
 cd `dirname "$0"`/../build
 
+jobs=-j4
+case "X$1" in
+X-j*) jobs=$1; shift;;
+esac
+
 plat="$1"
 [ -z "$plat" ] && exit 1
 shift
@@ -26,6 +31,7 @@ mkdir install_other
     --prefix="`pwd`/install_other" \
     --bindir="`pwd`/.." \
     "$@"
-make install-binaries
+make "$jobs" install-binaries
 cd ..
-rm -rf tmp
+rm -rf tmp
+

scripts/gen_utf8_tables.py

@@ -6,13 +6,12 @@ mask = lambda n: (1 << n) - 1
 
 
 def print_table(type, name, t):
-    assert len(t) == 256
     print("static const %s %s[] =" % (type, name))
     first = True
     for i in range(0,len(t),16):
         print ((" {" if i == 0 else "  ") +
                ", ".join("0x%02x"%n for n in t[i:i+16]) + 
-               ("," if i + 16 < 256 else "};"))
+               ("," if i + 16 < len(t) else "};"))
 
 
 def utf8info(c):
@@ -30,3 +29,4 @@ print("#define UTF8_CONTINUATION_BYTE ((unsigned char)255)")
 
 print_table("unsigned char", "utf8_coding_length", table(0))
 print_table("unsigned char", "utf8_coding_bits", table(1))
+print_table("int", "utf8_first_codepoint", [0, 0x0, 0x80, 0x800, 0x10000])

scripts/version

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+cd `dirname "$0"`
+git rev-parse --verify -q jq-1.0 > /dev/null 2>&1
+git describe --tags --match 'jq-*' --dirty | sed 's/^jq-//'

setup.sh

@@ -9,12 +9,14 @@
 
 if [ "clean" == "$1" ]; then
   make distclean || echo "continuing..."
-  rm -rf tmp autom4te.cache config
+  find config/m4 -type f \( -name libtool\* -o -name lt\* \) | xargs rm -f
+  find config -name m4 -prune -o -type f -print | xargs rm -f
+  rm -rf tmp autom4te.cache
   rm -f INSTALL Makefile.in aclocal.m4 configure config.h.in ChangeLog
   rm -f jv_utf8_tables.gen.h lexer.c lexer.h parser.c parser.h
 elif [ "superclean" == "$1" ]; then
   # if autoconf errors during distcheck, it leaves files that need chmod'ing
-  ver=$(tr -d '\n' <VERSION)
+  ver=$(scripts/version|tr -d '\n')
   if [ "x${ver}" != "x" ]; then
     if [ -d jq-${ver} ]; then
       chmod -R u+w jq-${ver}
@@ -26,6 +28,7 @@ else
   autoreconf --install
   ./configure --prefix=/opt/junk
   make check
+  [ -d tmp ] && mv tmp tmp-
   mkdir tmp
   make DESTDIR=./tmp install
   make distcheck

tests/all.test

@@ -36,6 +36,10 @@ null
 null
 []
 
+{x: -1}
+null
+{"x": -1}
+
 # The input line starts with a 0xFEFF (byte order mark) codepoint
 # No, there is no reason to have a byte order mark in UTF8 text.
 # But apparently people do, so jq shouldn't break on it.
@@ -75,6 +79,10 @@ null
 "<script>hax</script>"
 "<b>&lt;script&gt;hax&lt;/script&gt;</b>"
 
+[.[]|tojson|fromjson]
+["foo", 1, ["a", 1, "b", 2, {"foo":"bar"}]]
+["foo",1,["a",1,"b",2,{"foo":"bar"}]]
+
 #
 # Dictionary construction syntax
 #
@@ -115,6 +123,29 @@ null
 {"foo": {"bar": 42}, "bar": "badvalue"}
 42
 
+."foo"."bar"
+{"foo": {"bar": 20}}
+20
+
+[.[]|.foo?]
+[1,[2],{"foo":3,"bar":4},{},{"foo":5}]
+[3,null,5]
+
+[.[]|.foo?.bar?]
+[1,[2],[],{"foo":3},{"foo":{"bar":4}},{}]
+[4,null]
+
+[..]
+[1,[[2]],{ "a":[1]}]
+[[1,[[2]],{"a":[1]}],1,[[2]],[2],2,{"a":[1]},[1],1]
+
+[.[]|.[]?]
+[1,null,[],[1,[2,[[3]]]],[{}],[{"a":[1,[2]]}]]
+[1,[2,[[3]]],{},{"a":[1,[2]]}]
+
+[.[]|.[1:3]?]
+[1,null,true,false,"abcdef",{},{"a":1,"b":2},[],[1,2,3,4,5],[1,2]]
+[null,"bc",[],[2,3],[2]]
 
 #
 # Multiple outputs, iteration
@@ -174,6 +205,10 @@ null
 [0,1,2,3,4,5,6]
 [[], [2,3], [0,1,2,3,4], [5,6], [], []]
 
+[.[3:2], .[-5:4], .[:-2], .[-2:], .[3:3][1:], .[10:]]
+"abcdefghi"
+["","","abcdefg","hi","",""]
+
 del(.[2:4],.[0],.[-2:])
 [0,1,2,3,4,5,6,7]
 [1,4,5]
@@ -285,6 +320,14 @@ null
 null
 [2, 8, 10, 14]
 
+25 % 7
+null
+4
+
+49732 % 472
+null
+172
+
 1 + tonumber + ("10" | tonumber)
 4
 15
@@ -332,6 +375,10 @@ def f: (1000,2000); f
 1000
 2000
 
+def f(a;b;c;d;e;f): [a+1,b,c,d,e,f]; f(.[0];.[1];.[0];.[0];.[0];.[0])
+[1,2]
+[2,2,1,1,1,1]
+
 ([1,2] + [4,5])
 [1,2,3]
 [1,2,4,5]
@@ -350,6 +397,36 @@ null
 [5,6]
 [1,2,3]
 
+[.[]|floor]
+[-1.1,1.1,1.9]
+[-2, 1, 1]
+
+[.[]|sqrt]
+[4,9]
+[2,3]
+
+(add / length) as $m | map((. - $m) as $d | $d * $d) | add / length | sqrt
+[2,4,4,4,5,5,7,9]
+2
+
+# Should write a test that calls the -lm function from C (or bc(1)) to
+# check that they match the corresponding jq functions.  However,
+# there's so little template code standing between that it suffices to
+# test a handful of these.  The results were checked by eye against
+# bc(1).
+atan * 4 * 1000000|floor / 1000000
+1
+3.141592
+
+[(3.141592 / 2) * (range(0;20) / 20)|cos * 1000000|floor / 1000000]
+null
+[1,0.996917,0.987688,0.972369,0.951056,0.923879,0.891006,0.85264,0.809017,0.760406,0.707106,0.649448,0.587785,0.522498,0.45399,0.382683,0.309017,0.233445,0.156434,0.078459]
+
+[(3.141592 / 2) * (range(0;20) / 20)|sin * 1000000|floor / 1000000]
+null
+[0,0.078459,0.156434,0.233445,0.309016,0.382683,0.45399,0.522498,0.587785,0.649447,0.707106,0.760405,0.809016,0.85264,0.891006,0.923879,0.951056,0.972369,0.987688,0.996917]
+
+
 def f(x): x | x; f([.], . + [42])
 [1,2,3]
 [[[1,2,3]]]
@@ -400,6 +477,14 @@ path(.)
 42
 []
 
+[paths]
+[1,[[],{"a":2}]]
+[[0],[1],[1,0],[1,1],[1,1,"a"]]
+
+[leaf_paths]
+[1,[[],{"a":2}]]
+[[0],[1,1,"a"]]
+
 ["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
 {"bar": 42, "foo": ["a", "b", "c", "d"]}
 "b"
@@ -448,12 +533,17 @@ null
 {"foo": 42}
 {"foo": 43}
 
-.[] += 2, .[] *= 2, .[] -= 2, .[] /= 2
+.[] += 2, .[] *= 2, .[] -= 2, .[] /= 2, .[] %=2
 [1,3,5]
 [3,5,7]
 [2,6,10]
 [-1,1,3]
 [0.5, 1.5, 2.5]
+[1,1,1]
+
+[.[] % 7]
+[-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7]
+[0,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,0]
 
 .foo += .foo
 {"foo":2}
@@ -545,6 +635,75 @@ def inc(x): x |= .+1; inc(.[].a)
 {}
 [true, true, false]
 
+# string operations
+[.[]|startswith("foo")]
+["fo", "foo", "barfoo", "foobar", "barfoob"]
+[false, true, false, true, false]
+
+[.[]|endswith("foo")]
+["fo", "foo", "barfoo", "foobar", "barfoob"]
+[false, true, true, false, false]
+
+[.[]|ltrimstr("foo")]
+["fo", "foo", "barfoo", "foobar", "afoo"]
+["fo","","barfoo","bar","afoo"]
+
+[.[]|rtrimstr("foo")]
+["fo", "foo", "barfoo", "foobar", "foob"]
+["fo","","bar","foobar","foob"]
+
+.[","]
+"a,bc,def,ghij,klmno"
+[1,4,8,13]
+
+.[", "]
+"a,bc,def,ghij,klmno"
+[]
+
+.[", "]
+"a,b,, c, d,ef, , ghi, jklmn, o"
+[4,7,13,15,20,27]
+
+[(index(","), rindex(",")), indices(",")]
+"a,bc,def,ghij,klmno"
+[1,13,[1,4,8,13]]
+
+indices(1)
+[0,1,1,2,3,4,1,5]
+[1,2,6]
+
+indices([1,2])
+[0,1,2,3,1,4,2,5,1,2,6,7]
+[1,8]
+
+indices([1,2])
+[1]
+[]
+
+indices(", ")
+"a,b, cd,e, fgh, ijkl"
+[3,9,14]
+
+[.[]|split(",")]
+["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
+[["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]]
+
+[.[]|split(", ")]
+["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
+[["a","bc","def","ghij","jklmn","a,b","c,d","e,f"],["a,b,c,d","e,f,g,h"]]
+
+[.[] * 3]
+["a", "ab", "abc"]
+["aaa", "ababab", "abcabcabc"]
+
+[.[] / ","]
+["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
+[["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]]
+
+[.[] / ", "]
+["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
+[["a","bc","def","ghij","jklmn","a,b","c,d","e,f"],["a,b,c,d","e,f,g,h"]]
+
 map(.[1] as $needle | .[0] | contains($needle))
 [[[],[]], [[1,2,3], [1,2]], [[1,2,3], [3,1]], [[1,2,3], [4]], [[1,2,3], [1,4]]]
 [true, true, true, false, false]
@@ -581,6 +740,10 @@ unique
 [1,2,5,3,5,3,1,3]
 [1,2,3,5]
 
+unique
+[]
+[]
+
 [min, max, min_by(.[1]), max_by(.[1]), min_by(.[2]), max_by(.[2])]
 [[4,2,"a"],[3,1,"a"],[2,4,"a"],[1,3,"a"]]
 [[1,3,"a"],[4,2,"a"],[3,1,"a"],[2,4,"a"],[4,2,"a"],[1,3,"a"]]
@@ -600,3 +763,81 @@ unique
 [{a:1}] | .[] | .a=999
 null
 {"a": 999}
+
+to_entries
+{"a": 1, "b": 2}
+[{"key":"a", "value":1}, {"key":"b", "value":2}]
+
+from_entries
+[{"key":"a", "value":1}, {"key":"b", "value":2}]
+{"a": 1, "b": 2}
+
+with_entries(.key |= "KEY_" + .)
+{"a": 1, "b": 2}
+{"KEY_a": 1, "KEY_b": 2}
+
+map(has("foo"))
+[{"foo": 42}, {}]
+[true, false]
+
+map(has(2))
+[[0,1], ["a","b","c"]]
+[false, true]
+
+keys
+[42,3,35]
+[0,1,2]
+
+[][.]
+1000000000000000000
+null
+
+map([1,2][0:.])
+[-1, 1, 2, 3, 1000000000000000000]
+[[1], [1], [1,2], [1,2], [1,2]]
+
+# Test recursive object merge
+
+{"k": {"a": 1, "b": 2}} * .
+{"k": {"a": 0,"c": 3}}
+{"k": {"a": 0, "b": 2, "c": 3}}
+
+{"k": {"a": 1, "b": 2}, "hello": {"x": 1}} * .
+{"k": {"a": 0,"c": 3}, "hello": 1}
+{"k": {"a": 0, "b": 2, "c": 3}, "hello": 1}
+
+{"k": {"a": 1, "b": 2}, "hello": 1} * .
+{"k": {"a": 0,"c": 3}, "hello": {"x": 1}}
+{"k": {"a": 0, "b": 2, "c": 3}, "hello": {"x": 1}}
+
+{"a": {"b": 1}, "c": {"d": 2}, "e": 5} * .
+{"a": {"b": 2}, "c": {"d": 3, "f": 9}}
+{"a": {"b": 2}, "c": {"d": 3, "f": 9}, "e": 5}
+
+[.[]|arrays]
+[1,2,"foo",[],[3,[]],{},true,false,null]
+[[],[3,[]]]
+
+[.[]|objects]
+[1,2,"foo",[],[3,[]],{},true,false,null]
+[{}]
+
+[.[]|iterables]
+[1,2,"foo",[],[3,[]],{},true,false,null]
+[[],[3,[]],{}]
+
+[.[]|scalars]
+[1,2,"foo",[],[3,[]],{},true,false,null]
+[1,2,"foo",true,false,null]
+
+[.[]|values]
+[1,2,"foo",[],[3,[]],{},true,false,null]
+[1,2,"foo",[],[3,[]],{},true,false]
+
+[.[]|booleans]
+[1,2,"foo",[],[3,[]],{},true,false,null]
+[true,false]
+
+[.[]|nulls]
+[1,2,"foo",[],[3,[]],{},true,false,null]
+[null]

tests/run

@@ -1,3 +1,9 @@
 #!/bin/sh
 
-cat $@ | valgrind --error-exitcode=1 -q --leak-check=full ./jq --run-tests
+if which valgrind > /dev/null; then
+    VALGRIND='valgrind --error-exitcode=1 -q --leak-check=full'
+else
+    VALGRIND=
+fi
+
+cat $@ | $VALGRIND ./jq --run-tests