Remove #includes from jv.h

Conflicts:
	Makefile.am

.gitattributes

@@ -0,0 +1,2 @@
+.gitattributes export-ignore
+.gitignore export-ignore

.gitignore

@@ -1,13 +1,41 @@
 *.o
+*.a
+*.lo
+*.la
+*.lai
+*.so
+*.so.*
 *~
+.*.sw[a-p]
+tags
 
-# Autogenerated
-*.gen.*
-
-# Test binaries
-jv_test
-jv_parse
-parsertest*~
+jq
+jq.1
 
 # Something delightfully recursive happens otherwise
-docs/content/2.download/source/*
+docs/content/2.download/source/*
+
+# Autogenerated
+lexer.c
+lexer.h
+parser.c
+parser.h
+
+# Autotools junk
+.libs
+.deps
+libtool
+*.log
+stamp-h1
+config.log
+config.status
+autom4te.cache
+INSTALL
+Makefile
+jq-*.tar.gz
+configure
+aclocal.m4
+Makefile.in
+version.h
+.remake-version-h
+config.cache

AUTHORS

@@ -0,0 +1,6 @@
+Created By:
+Stephen Dolan        <mu@netsoc.tcd.ie>
+
+Contributions by
+Anthony Shortland    <anthony.shortland@me.com>      - rpmbuild target
+Lee Thompson         <stagr.lee@gmail.com>           - autoconf stuff

Makefile

@@ -1,61 +0,0 @@
-CC=gcc -Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function
-prefix=/usr/local
-
-.PHONY: all clean releasedep tarball install uninstall test releasetag
-all: jq
-
-clean:
-	make -Bnd | grep 'Must remake target' | \
-	sed 's/.*`\(.*\)'\''.*/\1/' | grep -v '^all$$' | \
-	xargs rm
-
-
-lexer.gen.c: lexer.l
-	flex -o lexer.gen.c --header-file=lexer.gen.h lexer.l
-lexer.gen.h: lexer.gen.c
-
-parser.gen.c: parser.y lexer.gen.h
-	bison -W -d parser.y -v --report-file=parser.gen.info -o $@
-parser.gen.h: parser.gen.c
-
-jv_utf8_tables.gen.h: gen_utf8_tables.py
-	python $^ > $@
-jv_unicode.c: jv_utf8_tables.gen.h
-
-version.gen.h: VERSION
-	sed 's/.*/#define JQ_VERSION "&"/' $^ > $@
-main.c: version.gen.h
-
-JQ_SRC=parser.gen.c lexer.gen.c opcode.c bytecode.c compile.c execute.c builtin.c jv.c jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c
-
-
-jq_test: $(JQ_SRC) jq_test.c
-	$(CC) -DJQ_DEBUG=1 -o $@ $^
-
-jq: $(JQ_SRC) main.c
-	$(CC) -O -DJQ_DEBUG=0 -o $@ $^
-
-test: jq_test
-	valgrind --error-exitcode=1 -q --leak-check=full ./jq_test >/dev/null
-
-
-releasedep: lexer.gen.c parser.gen.c jv_utf8_tables.gen.h
-
-releasetag:
-	git tag -s "jq-$$(cat VERSION)" -m "jq release $$(cat VERSION)"
-
-docs/content/2.download/source/jq.tgz: jq
-	mkdir -p `dirname $@`
-	tar -czvf $@ `git ls-files; ls *.gen.*`
-
-tarball: docs/content/2.download/source/jq.tgz
-
-install: jq
-	install -d -m 0755 $(prefix)/bin
-	install -m 0755 jq $(prefix)/bin
-
-uninstall:
-	test -d $(prefix)/bin && \
-	cd $(prefix)/bin && \
-	rm -f jq
-

Makefile.am

@@ -0,0 +1,144 @@
+### C source files to be built and distributed.
+
+LIBJQ_INCS = jq_parser.h builtin.h bytecode.h compile.h exec_stack.h	\
+   jv_alloc.h jv_dtoa.h jv_parse.h jv_unicode.h locfile.h		\
+   opcode_list.h parser.y jv_utf8_tables.h lexer.l
+
+LIBJQ_SRC = locfile.c bytecode.c compile.c execute.c builtin.c jv.c	\
+  jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_file.c	\
+  jv_alloc.c jq_test.c ${LIBJQ_INCS}
+
+
+### C build options
+
+AM_CFLAGS = -Wextra -Wall -Wno-missing-field-initializers	\
+  -Wno-unused-parameter -Wno-unused-function
+
+ACLOCAL_AMFLAGS = -I config/m4
+
+### Generating the lexer and parser
+
+# While there is some autoconf macro support for lex/flex, it doesn't support
+# header file creation so we'll use good old make
+BUILT_SOURCES = lexer.h lexer.c parser.h parser.c version.h
+lexer.c: lexer.l
+	$(AM_V_LEX) flex -o lexer.c --header-file=lexer.h $<
+lexer.h: lexer.c
+
+# Tell YACC (bison) autoconf macros that you want a header file created.
+# If the --warnings=all fails, you probably have an old version of bison
+# OSX ships an old bison, so update with homebrew or macports
+AM_YFLAGS = --warnings=all -d
+
+### libjq
+
+lib_LTLIBRARIES = libjq.la
+libjq_la_SOURCES = ${LIBJQ_SRC}
+libjq_la_LIBADD = -lm
+libjq_la_LDFLAGS = -export-symbols-regex '^j[qv]_' -version-info 1:4:0
+
+include_HEADERS = jv.h jq.h
+
+
+### Building the jq binary
+
+# Remake the version.h header file if, and only if, the git ID has changed
+.PHONY: .FORCE
+.FORCE:
+generate_ver = ver="`{ $(srcdir)/scripts/version || echo '$(VERSION)' ; } | sed 's/.*/\#define JQ_VERSION \"&\"/'`"
+.remake-version-h: .FORCE
+	@ $(generate_ver); test "x`cat version.h 2>/dev/null`" = "x$$ver" || touch .remake-version-h
+version.h: .remake-version-h
+	$(AM_V_GEN) $(generate_ver); echo "$$ver" > $@
+
+bin_PROGRAMS = jq
+jq_SOURCES = main.c
+jq_LDFLAGS = -static-libtool-libs
+jq_LDADD = libjq.la -lm
+
+### Tests (make check)
+
+TESTS = tests/all.test
+TEST_LOG_COMPILER = ${srcdir}/tests/run
+
+
+### Building the manpage
+
+# If ENABLE_DOCS is not set, just copy jq.1.default to jq.1
+# The real_docs target succeeds (possibly after building jq.1) only if ENABLE_DOCS is set
+# Thus, making "dist" depend on "real_docs" ensures that we never build a tarball with
+# a stub manpage.
+
+man_MANS = jq.1
+.PHONY: real_docs
+if ENABLE_DOCS
+jq.1: $(srcdir)/docs/content/3.manual/manual.yml
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
+jq.1.default: $(srcdir)/docs/default_manpage.md
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
+real_docs: jq.1
+	if cmp jq.1 $(srcdir)/jq.1.default > /dev/null; then\
+	  rm -f jq.1; $(MAKE) $(AM_MAKEFLAGS) jq.1;\
+	fi
+else
+jq.1: $(srcdir)/jq.1.default
+	$(AM_V_GEN) cp $^ $@
+real_docs:
+	@echo "Ruby dependencies not found, cannot build manpage." > /dev/stderr
+	@echo "Follow the instructions in docs/README.md to install them" > /dev/stderr
+	@echo "and then rerun ./configure" > /dev/stderr
+	false
+endif
+
+
+### Packaging
+
+docs/site.yml: configure.ac
+	sed 's/^jq_version: .*/jq_version: "$(VERSION)"/' $@ > $@.new
+	mv $@.new $@
+
+install-binaries: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) install-exec
+
+# Ensure "make dist" fails when we can't build the real manpage
+dist-hook: real_docs
+
+DOC_FILES = docs/content docs/public docs/templates docs/site.yml	\
+            docs/Gemfile docs/Gemfile.lock docs/Rakefile docs/README.md \
+            docs/default_manpage.md jq.1.default
+
+# setup is only used by distribution developers, not package developers.
+# Still, as a matter of allowing patching, its not a bad idea to distribute 
+# the developer setup script in the tarball.
+EXTRA_DIST = $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)		\
+	gen_utf8_tables.py jq.spec $(DOC_FILES) scripts/version	\
+	parser.h parser.c lexer.h lexer.c libjq.map
+
+# README.md is expected in Github projects, good stuff in it, so we'll 
+# distribute it and install it with the package in the doc directory.
+docdir = ${datadir}/doc/${PACKAGE}
+dist_doc_DATA = README.md COPYING AUTHORS README
+
+RELEASE ?= 1
+rpm: jq
+	@echo "Packaging jq as an RPM ..."
+	mkdir -p rpm/SOURCES rpm/BUILD rpm/BUILDROOT rpm/RPMS rpm/SPECS
+	cp jq-$$(cat VERSION).tar.gz rpm/SOURCES/
+	rpmbuild -tb --define "_topdir ${PWD}/rpm" --define "_prefix /usr" --define "myver $$(cat VERSION)" --define "myrel ${RELEASE}" rpm/SOURCES/jq-$$(cat VERSION).tar.gz
+	find rpm/RPMS/ -name "*.rpm" -exec mv {} ./ \;
+	rm -rf rpm
+
+dist-clean-local:
+	rm -f ${BUILT_SOURCES}
+
+# Not sure why this doesn't get cleaned up automatically, guess
+# automake used to man pages which are hand coded?
+# 'make clean' doesn't delete the manpage if it can't be rebuilt
+.PHONY: clean-local-docs
+clean-local-docs:
+if ENABLE_DOCS
+	rm -f jq.1
+endif 
+
+clean-local: clean-local-docs
+	rm -f version.h .remake-version-h

README

@@ -0,0 +1 @@
+README.md

README.md

@@ -4,11 +4,25 @@ jq
 jq is a command-line JSON processor.
 
 If you want to learn to use jq, read the documentation at
-[http://stedolan.github.com/jq](http://stedolan.github.com/jq). This
+[http://stedolan.github.io/jq](http://stedolan.github.io/jq). This
 documentation is generated from the docs/ folder of this repository.
 
 If you want to hack on jq, feel free, but be warned that its internals
-are not well-documented at the moment. Bring a hard hat and a shovel.
+are not well-documented at the moment. Bring a hard hat and a
+shovel. Also, read the wiki: http://github.com/stedolan/jq/wiki
 
-You can find some basic build instructions at
-[http://stedolan.github.com/jq/download](http://stedolan.github.com/jq/download)
+If you're building directly from the latest git, you'll need flex and
+bison installed. To build, run:
+
+    autoreconf -i
+    ./configure
+    make
+
+After make finishes, you'll be able to use `./jq`. You can also
+install it using:
+
+    sudo make install
+
+If you're not using the latest git version but instead building a
+released tarball (available on the website), then you won't need to
+run `autoreconf` (and shouldn't), and you won't need flex or bison.

VERSION

@@ -1 +0,0 @@
-1.1

build/.gitignore

@@ -0,0 +1 @@
+*

builtin.c

@@ -1,51 +1,73 @@
+#include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 #include "builtin.h"
 #include "compile.h"
-#include "parser.h"
+#include "jq_parser.h"
+#include "bytecode.h"
 #include "locfile.h"
+#include "jv_unicode.h"
 
-enum {
-  CMP_OP_LESS,
-  CMP_OP_GREATER,
-  CMP_OP_LESSEQ,
-  CMP_OP_GREATEREQ
-} _cmp_op;
 
-static void f_plus(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
-  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    output[0] = jv_number(jv_number_value(a) + 
-                          jv_number_value(b));
-  } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
-    output[0] = jv_string_concat(a, b);
-  } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
-    output[0] = jv_array_concat(a, b);
-  } else if (jv_get_kind(a) == JV_KIND_OBJECT && jv_get_kind(b) == JV_KIND_OBJECT) {
-    output[0] = jv_object_merge(a, b);
-  } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to add %s and %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
+static jv type_error(jv bad, const char* msg) {
+  jv err = jv_invalid_with_msg(jv_string_fmt("%s %s",
+                                             jv_kind_name(jv_get_kind(bad)),
+                                             msg));
+  jv_free(bad);
+  return err;
+}
+
+static jv type_error2(jv bad1, jv bad2, const char* msg) {
+  jv err = jv_invalid_with_msg(jv_string_fmt("%s and %s %s",
+                                             jv_kind_name(jv_get_kind(bad1)),
+                                             jv_kind_name(jv_get_kind(bad2)),
+                                             msg));
+  jv_free(bad1);
+  jv_free(bad2);
+  return err;
+}
+
+static jv f_plus(jv input, jv a, jv b) {
+  jv_free(input);
+  if (jv_get_kind(a) == JV_KIND_NULL) {
     jv_free(a);
+    return b;
+  } else if (jv_get_kind(b) == JV_KIND_NULL) {
     jv_free(b);
+    return a;
+  } else if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
+    return jv_number(jv_number_value(a) + 
+                     jv_number_value(b));
+  } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
+    return jv_string_concat(a, b);
+  } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
+    return jv_array_concat(a, b);
+  } else if (jv_get_kind(a) == JV_KIND_OBJECT && jv_get_kind(b) == JV_KIND_OBJECT) {
+    return jv_object_merge(a, b);
+  } else {
+    return type_error2(a, b, "cannot be added");
   }
 }
 
-static void f_minus(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
+static jv f_negate(jv input) {
+  if (jv_get_kind(input) != JV_KIND_NUMBER) {
+    return type_error(input, "cannot be negated");
+  }
+  jv ret = jv_number(-jv_number_value(input));
+  jv_free(input);
+  return ret;
+}
+
+static jv f_minus(jv input, jv a, jv b) {
+  jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    output[0] = jv_number(jv_number_value(a) - jv_number_value(b));
+    return jv_number(jv_number_value(a) - jv_number_value(b));
   } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
     jv out = jv_array();
-    for (int i=0; i<jv_array_length(jv_copy(a)); i++) {
-      jv x = jv_array_get(jv_copy(a), i);
+    jv_array_foreach(a, i, x) {
       int include = 1;
-      for (int j=0; j<jv_array_length(jv_copy(b)); j++) {
-        if (jv_equal(jv_copy(x), jv_array_get(jv_copy(b), j))) {
+      jv_array_foreach(b, j, y) {
+        if (jv_equal(jv_copy(x), y)) {
           include = 0;
           break;
         }
@@ -56,268 +78,515 @@ static void f_minus(jv input[], jv output[]) {
     }
     jv_free(a);
     jv_free(b);
-    output[0] = out;
+    return out;
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to subtract %s and %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
+    return type_error2(a, b, "cannot be subtracted");
   }
 }
 
-static void f_multiply(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
+static jv f_multiply(jv input, jv a, jv b) {
+  jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    output[0] = jv_number(jv_number_value(a) * jv_number_value(b));
+    return jv_number(jv_number_value(a) * jv_number_value(b));
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to multiply %s and %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
+    return type_error2(a, b, "cannot be multiplied");
   }  
 }
 
-static void f_divide(jv input[], jv output[]) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
+static jv f_divide(jv input, jv a, jv b) {
+  jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    output[0] = jv_number(jv_number_value(a) / jv_number_value(b));
+    return jv_number(jv_number_value(a) / jv_number_value(b));
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to divide %s by %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
+    return type_error2(a, b, "cannot be divided");
   }  
 }
 
-static void f_add(jv input[], jv output[]) {
-  jv array = input[0];
-  if (jv_get_kind(array) != JV_KIND_ARRAY) {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot add elements of an %s",
-                                                  jv_kind_name(jv_get_kind(array))));
-  } else if (jv_array_length(jv_copy(array)) == 0) {
-    output[0] = jv_null();
+static jv f_equal(jv input, jv a, jv b) {
+  jv_free(input);
+  return jv_bool(jv_equal(a, b));
+}
+
+static jv f_notequal(jv input, jv a, jv b) {
+  jv_free(input);
+  return jv_bool(!jv_equal(a, b));
+}
+
+enum cmp_op {
+  CMP_OP_LESS,
+  CMP_OP_GREATER,
+  CMP_OP_LESSEQ,
+  CMP_OP_GREATEREQ
+};
+
+static jv order_cmp(jv input, jv a, jv b, enum cmp_op op) {
+  jv_free(input);
+  int r = jv_cmp(a, b);
+  return jv_bool((op == CMP_OP_LESS && r < 0) ||
+                 (op == CMP_OP_LESSEQ && r <= 0) ||
+                 (op == CMP_OP_GREATEREQ && r >= 0) ||
+                 (op == CMP_OP_GREATER && r > 0));
+}
+
+static jv f_less(jv input, jv a, jv b) {
+  return order_cmp(input, a, b, CMP_OP_LESS);
+}
+
+static jv f_greater(jv input, jv a, jv b) {
+  return order_cmp(input, a, b, CMP_OP_GREATER);
+}
+
+static jv f_lesseq(jv input, jv a, jv b) {
+  return order_cmp(input, a, b, CMP_OP_LESSEQ);
+}
+
+static jv f_greatereq(jv input, jv a, jv b) {
+  return order_cmp(input, a, b, CMP_OP_GREATEREQ);
+}
+
+static jv f_contains(jv a, jv b) {
+  if (jv_get_kind(a) == jv_get_kind(b)) {
+    return jv_bool(jv_contains(a, b));
   } else {
-    jv sum = jv_array_get(jv_copy(array), 0);
-    for (int i = 1; i < jv_array_length(jv_copy(array)); i++) {
-      jv x = jv_array_get(jv_copy(array), i);
-      jv add_args[] = {jv_null(), x, sum};
-      f_plus(add_args, &sum);
+    return type_error2(a, b, "cannot have their containment checked");
+  }
+}
+
+static jv f_tonumber(jv input) {
+  if (jv_get_kind(input) == JV_KIND_NUMBER) {
+    return input;
+  }
+  if (jv_get_kind(input) == JV_KIND_STRING) {
+    jv parsed = jv_parse(jv_string_value(input));
+    if (!jv_is_valid(parsed) || jv_get_kind(parsed) == JV_KIND_NUMBER) {
+      jv_free(input);
+      return parsed;
     }
-    output[0] = sum;
   }
-  jv_free(array);
+  return type_error(input, "cannot be parsed as a number");
 }
 
-static void f_equal(jv input[], jv output[]) {
-  jv_free(input[0]);
-  output[0] = jv_bool(jv_equal(input[2], input[1]));
-}
-
-static void order_cmp(jv input[], jv output[], int op) {
-  jv_free(input[0]);
-  jv a = input[2];
-  jv b = input[1];
-  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    double da = jv_number_value(a);
-    double db = jv_number_value(b);
-    output[0] = jv_bool((op == CMP_OP_LESS && da < db) ||
-                        (op == CMP_OP_LESSEQ && da <= db) ||
-                        (op == CMP_OP_GREATEREQ && da >= db) ||
-                        (op == CMP_OP_GREATER && da > db));
+static jv f_length(jv input) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY) {
+    return jv_number(jv_array_length(input));
+  } else if (jv_get_kind(input) == JV_KIND_OBJECT) {
+    return jv_number(jv_object_length(input));
+  } else if (jv_get_kind(input) == JV_KIND_STRING) {
+    return jv_number(jv_string_length_codepoints(input));
+  } else if (jv_get_kind(input) == JV_KIND_NULL) {
+    jv_free(input);
+    return jv_number(0);
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to compare order of %s wrt %s",
-                                                  jv_kind_name(jv_get_kind(a)),
-                                                  jv_kind_name(jv_get_kind(b))));
-    jv_free(a);
-    jv_free(b);
+    return type_error(input, "has no length");
   }
 }
 
-static void f_less(jv input[], jv output[]) {
-  order_cmp(input, output, CMP_OP_LESS);
+static jv f_tostring(jv input) {
+  if (jv_get_kind(input) == JV_KIND_STRING) {
+    return input;
+  } else {
+    return jv_dump_string(input, 0);
+  }
 }
 
-static void f_greater(jv input[], jv output[]) {
-  order_cmp(input, output, CMP_OP_GREATER);
-}
+#define CHARS_ALPHANUM "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
 
-static void f_lesseq(jv input[], jv output[]) {
-  order_cmp(input, output, CMP_OP_LESSEQ);
-}
+static jv escape_string(jv input, const char* escapings) {
 
-static void f_greatereq(jv input[], jv output[]) {
-  order_cmp(input, output, CMP_OP_GREATEREQ);
-}
+  assert(jv_get_kind(input) == JV_KIND_STRING);
+  const char* lookup[128] = {0};
+  const char* p = escapings;
+  while (*p) {
+    lookup[(int)*p] = p+1;
+    p++;
+    p += strlen(p);
+    p++;
+  }
 
-static void f_tonumber(jv input[], jv output[]) {
-  if (jv_get_kind(input[0]) == JV_KIND_NUMBER) {
-    output[0] = input[0];
-  } else if (jv_get_kind(input[0]) == JV_KIND_STRING) {
-    jv parsed = jv_parse(jv_string_value(input[0]));
-    if (!jv_is_valid(parsed)) {
-      jv_free(input[0]);
-      output[0] = parsed;
-    } else if (jv_get_kind(parsed) != JV_KIND_NUMBER) {
-      output[0] = jv_invalid_with_msg(jv_string_fmt("'%s' is not a number",
-                                                    jv_string_value(input[0])));
-      jv_free(input[0]);
+  jv ret = jv_string("");
+  const char* i = jv_string_value(input);
+  const char* end = i + jv_string_length_bytes(jv_copy(input));
+  const char* cstart;
+  int c = 0;
+  while ((i = jvp_utf8_next((cstart = i), end, &c))) {
+    assert(c != -1);
+    if (c < 128 && lookup[c]) {
+      ret = jv_string_append_str(ret, lookup[c]);
     } else {
-      jv_free(input[0]);
-      output[0] = parsed;
+      ret = jv_string_append_buf(ret, cstart, i - cstart);
     }
-  } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot parse %s as a number",
-                                                  jv_kind_name(jv_get_kind(input[0]))));
-    jv_free(input[0]);
-    return;
   }
+  jv_free(input);
+  return ret;
+
 }
 
-static void f_length(jv input[], jv output[]) {
-  if (jv_get_kind(input[0]) == JV_KIND_ARRAY) {
-    output[0] = jv_number(jv_array_length(input[0]));
-  } else if (jv_get_kind(input[0]) == JV_KIND_OBJECT) {
-    output[0] = jv_number(jv_object_length(input[0]));
-  } else if (jv_get_kind(input[0]) == JV_KIND_STRING) {
-    output[0] = jv_number(jv_string_length(input[0]));
-  } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot get the length of %s",
-                                                  jv_kind_name(jv_get_kind(input[0]))));
-    jv_free(input[0]);
+static jv f_format(jv input, jv fmt) {
+  if (jv_get_kind(fmt) != JV_KIND_STRING) {
+    jv_free(input);
+    return type_error(fmt, "is not a valid format");
   }
-}
-
-static void f_tostring(jv input[], jv output[]) {
-  if (jv_get_kind(input[0]) == JV_KIND_STRING) {
-    output[0] = input[0];
-  } else {
-    output[0] = jv_dump_string(input[0], 0);
-  }
-}
-
-static int string_cmp(const void* pa, const void* pb){
-  const jv* a = pa;
-  const jv* b = pb;
-  int lena = jv_string_length(jv_copy(*a));
-  int lenb = jv_string_length(jv_copy(*b));
-  int minlen = lena < lenb ? lena : lenb;
-  int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen);
-  if (r == 0) r = lena - lenb;
-  return r;
-}
-
-static void f_keys(jv input[], jv output[]) {
-  if (jv_get_kind(input[0]) == JV_KIND_OBJECT) {
-    int nkeys = jv_object_length(jv_copy(input[0]));
-    jv* keys = malloc(sizeof(jv) * nkeys);
-    int kidx = 0;
-    jv_object_foreach(i, input[0]) {
-      keys[kidx++] = jv_object_iter_key(input[0], i);
+  const char* fmt_s = jv_string_value(fmt);
+  if (!strcmp(fmt_s, "json")) {
+    jv_free(fmt);
+    return jv_dump_string(input, 0);
+  } else if (!strcmp(fmt_s, "text")) {
+    jv_free(fmt);
+    return f_tostring(input);
+  } else if (!strcmp(fmt_s, "csv")) {
+    jv_free(fmt);
+    if (jv_get_kind(input) != JV_KIND_ARRAY)
+      return type_error(input, "cannot be csv-formatted, only array");
+    jv line = jv_string("");
+    jv_array_foreach(input, i, x) {
+      if (i) line = jv_string_append_str(line, ",");
+      switch (jv_get_kind(x)) {
+      case JV_KIND_NULL:
+        /* null rendered as empty string */
+        jv_free(x);
+        break;
+      case JV_KIND_TRUE:
+      case JV_KIND_FALSE:
+        line = jv_string_concat(line, jv_dump_string(x, 0));
+        break;
+      case JV_KIND_NUMBER:
+        if (jv_number_value(x) != jv_number_value(x)) {
+          /* NaN, render as empty string */
+          jv_free(x);
+        } else {
+          line = jv_string_concat(line, jv_dump_string(x, 0));
+        }
+        break;
+      case JV_KIND_STRING: {
+        line = jv_string_append_str(line, "\"");
+        line = jv_string_concat(line, escape_string(x, "\"\"\"\0"));
+        line = jv_string_append_str(line, "\"");
+        break;
+      }
+      default:
+        jv_free(input);
+        jv_free(line);
+        return type_error(x, "is not valid in a csv row");
+      }
     }
-    qsort(keys, nkeys, sizeof(jv), string_cmp);
-    output[0] = jv_array_sized(nkeys);
-    for (int i = 0; i<nkeys; i++) {
-      output[0] = jv_array_append(output[0], keys[i]);
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "html")) {
+    jv_free(fmt);
+    return escape_string(f_tostring(input), "&&amp;\0<&lt;\0>&gt;\0'&apos;\0\"&quot;\0");
+  } else if (!strcmp(fmt_s, "uri")) {
+    jv_free(fmt);
+    input = f_tostring(input);
+
+    int unreserved[128] = {0};
+    const char* p = CHARS_ALPHANUM "-_.!~*'()";
+    while (*p) unreserved[(int)*p++] = 1;
+
+    jv line = jv_string("");
+    const char* s = jv_string_value(input);
+    for (int i=0; i<jv_string_length_bytes(jv_copy(input)); i++) {
+      unsigned ch = (unsigned)(unsigned char)*s;
+      if (ch < 128 && unreserved[ch]) {
+        line = jv_string_append_buf(line, s, 1);
+      } else {
+        line = jv_string_concat(line, jv_string_fmt("%%%02x", ch));
+      }
+      s++;
     }
-    free(keys);
-    jv_free(input[0]);
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "sh")) {
+    jv_free(fmt);
+    if (jv_get_kind(input) != JV_KIND_ARRAY)
+      input = jv_array_set(jv_array(), 0, input);
+    jv line = jv_string("");
+    jv_array_foreach(input, i, x) {
+      if (i) line = jv_string_append_str(line, " ");
+      switch (jv_get_kind(x)) {
+      case JV_KIND_NULL:
+      case JV_KIND_TRUE:
+      case JV_KIND_FALSE:
+      case JV_KIND_NUMBER:
+        line = jv_string_concat(line, jv_dump_string(x, 0));
+        break;
+
+      case JV_KIND_STRING: {
+        line = jv_string_append_str(line, "'");
+        line = jv_string_concat(line, escape_string(x, "''\\''\0"));
+        line = jv_string_append_str(line, "'");
+        break;
+      }
+
+      default:
+        jv_free(input);
+        jv_free(line);
+        return type_error(x, "can not be escaped for shell");
+      }
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "base64")) {
+    jv_free(fmt);
+    input = f_tostring(input);
+    jv line = jv_string("");
+    const char b64[64 + 1] = CHARS_ALPHANUM "+/";
+    const char* data = jv_string_value(input);
+    int len = jv_string_length_bytes(jv_copy(input));
+    for (int i=0; i<len; i+=3) {
+      uint32_t code = 0;
+      int n = len - i >= 3 ? 3 : len-i;
+      for (int j=0; j<3; j++) {
+        code <<= 8;
+        code |= j < n ? (unsigned)data[i+j] : 0;
+      }
+      char buf[4];
+      for (int j=0; j<4; j++) {
+        buf[j] = b64[(code >> (18 - j*6)) & 0x3f];
+      }
+      if (n < 3) buf[3] = '=';
+      if (n < 2) buf[2] = '=';
+      line = jv_string_append_buf(line, buf, sizeof(buf));
+    }
+    jv_free(input);
+    return line;
   } else {
-    output[0] = jv_invalid_with_msg(jv_string_fmt("'keys' only supports object, not %s",
-                                                  jv_kind_name(jv_get_kind(input[0]))));
-    jv_free(input[0]);
+    jv_free(input);
+    return jv_invalid_with_msg(jv_string_concat(fmt, jv_string(" is not a valid format")));
   }
 }
 
-static void f_type(jv input[], jv output[]) {
-  output[0] = jv_string(jv_kind_name(jv_get_kind(input[0])));
-  jv_free(input[0]);
+static jv f_keys(jv input) {
+  if (jv_get_kind(input) == JV_KIND_OBJECT || jv_get_kind(input) == JV_KIND_ARRAY) {
+    return jv_keys(input);
+  } else {
+    return type_error(input, "has no keys");
+  }
 }
 
-static block j_empty() {
-  return gen_op_block_defn(CLOSURE_CREATE, "empty", gen_op_simple(BACKTRACK));
+static jv f_sort(jv input){
+  if (jv_get_kind(input) == JV_KIND_ARRAY) {
+    return jv_sort(input, jv_copy(input));
+  } else {
+    return type_error(input, "cannot be sorted, as it is not an array");
+  }
 }
 
-static block j_false() {
-  return gen_op_block_defn(CLOSURE_CREATE, "false",
-                           block_join(gen_op_const(LOADK, jv_false()),
-                                      gen_op_simple(RET)));
+static jv f_sort_by_impl(jv input, jv keys) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY && 
+      jv_get_kind(keys) == JV_KIND_ARRAY &&
+      jv_array_length(jv_copy(input)) == jv_array_length(jv_copy(keys))) {
+    return jv_sort(input, keys);
+  } else {
+    return type_error2(input, keys, "cannot be sorted, as they are not both arrays");
+  }
 }
 
-static block j_true() {
-  return gen_op_block_defn(CLOSURE_CREATE, "true",
-                           block_join(gen_op_const(LOADK, jv_true()),
-                                      gen_op_simple(RET)));
+static jv f_group_by_impl(jv input, jv keys) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY && 
+      jv_get_kind(keys) == JV_KIND_ARRAY &&
+      jv_array_length(jv_copy(input)) == jv_array_length(jv_copy(keys))) {
+    return jv_group(input, keys);
+  } else {
+    return type_error2(input, keys, "cannot be sorted, as they are not both arrays");
+  }
 }
 
-static block j_null() {
-  return gen_op_block_defn(CLOSURE_CREATE, "null",
-                           block_join(gen_op_const(LOADK, jv_null()),
-                                      gen_op_simple(RET)));
+static jv minmax_by(jv values, jv keys, int is_min) {
+  if (jv_get_kind(values) != JV_KIND_ARRAY)
+    return type_error2(values, keys, "cannot be iterated over");
+  if (jv_get_kind(keys) != JV_KIND_ARRAY)
+    return type_error2(values, keys, "cannot be iterated over");
+  if (jv_array_length(jv_copy(values)) != jv_array_length(jv_copy(keys)))
+    return type_error2(values, keys, "have wrong length");
+
+  if (jv_array_length(jv_copy(values)) == 0) {
+    jv_free(values);
+    jv_free(keys);
+    return jv_null();
+  }
+  jv ret = jv_array_get(jv_copy(values), 0);
+  jv retkey = jv_array_get(jv_copy(keys), 0);
+  for (int i=1; i<jv_array_length(jv_copy(values)); i++) {
+    jv item = jv_array_get(jv_copy(keys), i);
+    int cmp = jv_cmp(jv_copy(item), jv_copy(retkey));
+    if ((cmp < 0) == (is_min == 1)) {
+      jv_free(retkey);
+      retkey = item;
+      jv_free(ret);
+      ret = jv_array_get(jv_copy(values), i);
+    } else {
+      jv_free(item);
+    }
+  }
+  jv_free(values);
+  jv_free(keys);
+  jv_free(retkey);
+  return ret;
 }
 
-static block j_not() {
-  return gen_op_block_defn(CLOSURE_CREATE, "not",
-                           block_join(gen_condbranch(gen_op_const(LOADK, jv_false()),
-                                                     gen_op_const(LOADK, jv_true())),
-                                      gen_op_simple(RET)));
+static jv f_min(jv x) {
+  return minmax_by(x, jv_copy(x), 1);
 }
 
-static struct cfunction function_list[] = {
-  {f_plus, "_plus", CALL_BUILTIN_3_1},
-  {f_minus, "_minus", CALL_BUILTIN_3_1},
-  {f_multiply, "_multiply", CALL_BUILTIN_3_1},
-  {f_divide, "_divide", CALL_BUILTIN_3_1},
-  {f_tonumber, "tonumber", CALL_BUILTIN_1_1},
-  {f_tostring, "tostring", CALL_BUILTIN_1_1},
-  {f_keys, "keys", CALL_BUILTIN_1_1},
-  {f_equal, "_equal", CALL_BUILTIN_3_1},
-  {f_less, "_less", CALL_BUILTIN_3_1},
-  {f_greater, "_greater", CALL_BUILTIN_3_1},
-  {f_lesseq, "_lesseq", CALL_BUILTIN_3_1},
-  {f_greatereq, "_greatereq", CALL_BUILTIN_3_1},
-  {f_length, "length", CALL_BUILTIN_1_1},
-  {f_type, "type", CALL_BUILTIN_1_1},
-  {f_add, "add", CALL_BUILTIN_1_1},
+static jv f_max(jv x) {
+  return minmax_by(x, jv_copy(x), 0);
+}
+
+static jv f_min_by_impl(jv x, jv y) {
+  return minmax_by(x, y, 1);
+}
+
+static jv f_max_by_impl(jv x, jv y) {
+  return minmax_by(x, y, 0);
+}
+
+
+static jv f_type(jv input) {
+  jv out = jv_string(jv_kind_name(jv_get_kind(input)));
+  jv_free(input);
+  return out;
+}
+
+static jv f_error(jv input, jv msg) {
+  jv_free(input);
+  msg = f_tostring(msg);
+  return jv_invalid_with_msg(msg);
+}
+
+static const struct cfunction function_list[] = {
+  {(cfunction_ptr)f_plus, "_plus", 3},
+  {(cfunction_ptr)f_negate, "_negate", 1},
+  {(cfunction_ptr)f_minus, "_minus", 3},
+  {(cfunction_ptr)f_multiply, "_multiply", 3},
+  {(cfunction_ptr)f_divide, "_divide", 3},
+  {(cfunction_ptr)f_tonumber, "tonumber", 1},
+  {(cfunction_ptr)f_tostring, "tostring", 1},
+  {(cfunction_ptr)f_keys, "keys", 1},
+  {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking
+  {(cfunction_ptr)jv_getpath, "getpath", 2},
+  {(cfunction_ptr)jv_delpaths, "delpaths", 2},
+  {(cfunction_ptr)jv_has, "has", 2},
+  {(cfunction_ptr)f_equal, "_equal", 3},
+  {(cfunction_ptr)f_notequal, "_notequal", 3},
+  {(cfunction_ptr)f_less, "_less", 3},
+  {(cfunction_ptr)f_greater, "_greater", 3},
+  {(cfunction_ptr)f_lesseq, "_lesseq", 3},
+  {(cfunction_ptr)f_greatereq, "_greatereq", 3},
+  {(cfunction_ptr)f_contains, "contains", 2},
+  {(cfunction_ptr)f_length, "length", 1},
+  {(cfunction_ptr)f_type, "type", 1},
+  {(cfunction_ptr)f_sort, "sort", 1},
+  {(cfunction_ptr)f_sort_by_impl, "_sort_by_impl", 2},
+  {(cfunction_ptr)f_group_by_impl, "_group_by_impl", 2},
+  {(cfunction_ptr)f_min, "min", 1},
+  {(cfunction_ptr)f_max, "max", 1},
+  {(cfunction_ptr)f_min_by_impl, "_min_by_impl", 2},
+  {(cfunction_ptr)f_max_by_impl, "_max_by_impl", 2},
+  {(cfunction_ptr)f_error, "error", 2},
+  {(cfunction_ptr)f_format, "format", 2},
 };
 
-static struct symbol_table cbuiltins = {function_list, sizeof(function_list)/sizeof(function_list[0])};
+struct bytecoded_builtin { const char* name; block code; };
+static block bind_bytecoded_builtins(block b) {
+  block builtins = gen_noop();
+  {
+    struct bytecoded_builtin builtin_defs[] = {
+      {"empty", gen_op_simple(BACKTRACK)},
+      {"false", gen_const(jv_false())},
+      {"true", gen_const(jv_true())},
+      {"null", gen_const(jv_null())},
+      {"not", gen_condbranch(gen_const(jv_false()),
+                             gen_const(jv_true()))}
+    };
+    for (unsigned i=0; i<sizeof(builtin_defs)/sizeof(builtin_defs[0]); i++) {
+      builtins = BLOCK(builtins, gen_function(builtin_defs[i].name, gen_noop(),
+                                              builtin_defs[i].code));
+    }
+  }
+  {
+    struct bytecoded_builtin builtin_def_1arg[] = {
+      {"path", BLOCK(gen_op_simple(PATH_BEGIN), 
+                     gen_call("arg", gen_noop()),
+                     gen_op_simple(PATH_END))},
+    };
+    for (unsigned i=0; i<sizeof(builtin_def_1arg)/sizeof(builtin_def_1arg[0]); i++) {
+      builtins = BLOCK(builtins, gen_function(builtin_def_1arg[i].name,
+                                              gen_param("arg"),
+                                              builtin_def_1arg[i].code));
+    }
+  }
+  {
+    block rangevar = gen_op_var_fresh(STOREV, "rangevar");
+    block init = BLOCK(gen_op_simple(DUP), gen_call("start", gen_noop()), rangevar);
+    block range = BLOCK(init, 
+                        gen_call("end", gen_noop()),
+                        gen_op_bound(RANGE, rangevar));
+    builtins = BLOCK(builtins, gen_function("range",
+                                            BLOCK(gen_param("start"), gen_param("end")),
+                                            range));
+  }
+  
+  return block_bind_referenced(builtins, b, OP_IS_CALL_PSEUDO);
+}
 
-typedef block (*bytecoded_builtin)();
-static bytecoded_builtin bytecoded_builtins[] = {
-  j_empty,
-  j_false,
-  j_true,
-  j_null,
-  j_not,
-};
-
-static const char* jq_builtins[] = {
+static const char* const jq_builtins[] = {
   "def map(f): [.[] | f];",
   "def select(f): if f then . else empty end;",
+  "def sort_by(f): _sort_by_impl(map([f]));",
+  "def group_by(f): _group_by_impl(map([f]));",
+  "def unique: group_by(.) | map(.[0]);",
+  "def max_by(f): _max_by_impl(map([f]));",
+  "def min_by(f): _min_by_impl(map([f]));",
+  "def add: reduce .[] as $x (null; . + $x);",
+  "def del(f): delpaths([path(f)]);",
+  "def _assign(paths; value): value as $v | reduce path(paths) as $p (.; setpath($p; $v));",
+  "def _modify(paths; update): reduce path(paths) as $p (.; setpath($p; getpath($p) | update));",
+  "def recurse(f): ., (f | select(. != null) | recurse(f));",
+  "def to_entries: [keys[] as $k | {key: $k, value: .[$k]}];",
+  "def from_entries: map({(.key): .value}) | add;",
+  "def with_entries(f): to_entries | map(f) | from_entries;",
+  "def reverse: [.[length - 1 - range(0;length)]];",
 };
 
 
-block builtins_bind(block b) {
-  for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) {
-    struct locfile src;
-    locfile_init(&src, jq_builtins[i], strlen(jq_builtins[i]));
-    block funcs;
-    int nerrors = jq_parse_library(&src, &funcs);
-    assert(!nerrors);
-    b = block_bind(funcs, b, OP_IS_CALL_PSEUDO);
-    locfile_free(&src);
+int builtins_bind_one(block* bb, const char* code) {
+  struct locfile src;
+  locfile_init(&src, code, strlen(code));
+  block funcs;
+  int nerrors = jq_parse_library(&src, &funcs);
+  if (nerrors == 0) {
+    *bb = block_bind_referenced(funcs, *bb, OP_IS_CALL_PSEUDO);
   }
-  block builtins = gen_noop();
-  for (unsigned i=0; i<sizeof(bytecoded_builtins)/sizeof(bytecoded_builtins[0]); i++) {
-    block_append(&builtins, bytecoded_builtins[i]());
+  locfile_free(&src);
+  return nerrors;
+}
+
+int slurp_lib(block* bb) {
+  int nerrors = 0;
+  char* home = getenv("HOME");
+  if (home) {    // silently ignore no $HOME
+    jv filename = jv_string_append_str(jv_string(home), "/.jq");
+    jv data = jv_load_file(jv_string_value(filename), 1);
+    if (jv_is_valid(data)) {
+      nerrors = builtins_bind_one(bb, jv_string_value(data) );
+    }
+    jv_free(filename);
+    jv_free(data);
   }
-  b = block_bind(builtins, b, OP_IS_CALL_PSEUDO);
-  return gen_cbinding(&cbuiltins, b);
+  return nerrors;
+}
+
+int builtins_bind(block* bb) {
+  int nerrors = slurp_lib(bb);
+  if (nerrors) {
+    block_free(*bb);
+    return nerrors;
+  }
+  for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) {
+    nerrors = builtins_bind_one(bb, jq_builtins[i]);
+    assert(!nerrors);
+  }
+  *bb = bind_bytecoded_builtins(*bb);
+  *bb = gen_cbinding(function_list, sizeof(function_list)/sizeof(function_list[0]), *bb);
+  return nerrors;
 }

builtin.h

@@ -4,6 +4,6 @@
 #include "bytecode.h"
 #include "compile.h"
 
-block builtins_bind(block);
+int builtins_bind(block*);
 
 #endif

bytecode.c

@@ -3,28 +3,48 @@
 #include <stdlib.h>
 
 #include "bytecode.h"
-#include "opcode.h"
+#include "jv_alloc.h"
+
+// flags, length
+#define NONE 0, 1
+#define CONSTANT OP_HAS_CONSTANT, 2
+#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
+#define BRANCH OP_HAS_BRANCH, 2
+#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
+#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
+#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
+#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
+
+#define OP(name, imm, in, out) \
+  {name, #name, imm, in, out},
+
+static const struct opcode_description opcode_descriptions[] = {
+#include "opcode_list.h"
+};
+
+static const struct opcode_description invalid_opcode_description = {
+  -1, "#INVALID", 0, 0, 0, 0
+};
+
+
+const struct opcode_description* opcode_describe(opcode op) {
+  if ((int)op >= 0 && (int)op < NUM_OPCODES) {
+    return &opcode_descriptions[op];
+  } else {
+    return &invalid_opcode_description;
+  }
+}
+
 
 static int bytecode_operation_length(uint16_t* codeptr) {
-  if (opcode_describe(*codeptr)->flags & OP_HAS_VARIABLE_LENGTH_ARGLIST) {
-    return 2 + codeptr[1] * 2;
-  } else {
-    return opcode_length(*codeptr);
+  int length = opcode_describe(*codeptr)->length;
+  if (*codeptr == CALL_JQ) {
+    length += codeptr[1] * 2;
   }
+  return length;
 }
 
-void dump_disassembly(int indent, struct bytecode* bc) {
-  if (bc->nclosures > 0) {
-    printf("%*s[params: %d]\n", indent, "", bc->nclosures);
-  }
-  dump_code(indent, bc);
-  for (int i=0; i<bc->nsubfunctions; i++) {
-    printf("%*ssubfn[%d]:\n", indent, "", i);
-    dump_disassembly(indent+2, bc->subfunctions[i]);
-  }
-}
-
-void dump_code(int indent, struct bytecode* bc) {
+static void dump_code(int indent, struct bytecode* bc) {
   int pc = 0;
   while (pc < bc->codelen) {
     printf("%*s", indent, "");
@@ -34,6 +54,43 @@ void dump_code(int indent, struct bytecode* bc) {
   }
 }
 
+static void symbol_table_free(struct symbol_table* syms) {
+  jv_mem_free(syms->cfunctions);
+  jv_free(syms->cfunc_names);
+  jv_mem_free(syms);
+}
+
+void dump_disassembly(int indent, struct bytecode* bc) {
+  if (bc->nclosures > 0) {
+    printf("%*s[params: ", indent, "");
+    jv params = jv_object_get(jv_copy(bc->debuginfo), jv_string("params"));
+    for (int i=0; i<bc->nclosures; i++) {
+      if (i) printf(", ");
+      jv name = jv_array_get(jv_copy(params), i);
+      printf("%s", jv_string_value(name));
+      jv_free(name);
+    }
+    jv_free(params);
+    printf("]\n");
+  }
+  dump_code(indent, bc);
+  for (int i=0; i<bc->nsubfunctions; i++) {
+    struct bytecode* subfn = bc->subfunctions[i];
+    jv name = jv_object_get(jv_copy(subfn->debuginfo), jv_string("name"));
+    printf("%*s%s:%d:\n", indent, "", jv_string_value(name), i);
+    jv_free(name);
+    dump_disassembly(indent+2, subfn);
+  }
+}
+
+static struct bytecode* getlevel(struct bytecode* bc, int level) {
+  while (level > 0) {
+    bc = bc->parent;
+    level--;
+  }
+  return bc;
+}
+
 void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
   int pc = codeptr - bc->code;
   printf("%04d ", pc);
@@ -41,19 +98,32 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
   printf("%s", op->name);
   if (op->length > 1) {
     uint16_t imm = bc->code[pc++];
-    if (op->flags & OP_HAS_VARIABLE_LENGTH_ARGLIST) {
-      for (int i=0; i<imm; i++) {
+    if (op->op == CALL_JQ) {
+      for (int i=0; i<imm+1; i++) {
         uint16_t level = bc->code[pc++];
         uint16_t idx = bc->code[pc++];
+        jv name;
         if (idx & ARG_NEWCLOSURE) {
-          printf(" subfn[%d]", idx & ~ARG_NEWCLOSURE);
+          idx &= ~ARG_NEWCLOSURE;
+          name = jv_object_get(jv_copy(getlevel(bc,level)->subfunctions[idx]->debuginfo), 
+                               jv_string("name"));
         } else {
-          printf(" param[%d]", idx);
+          name = jv_array_get(jv_object_get(jv_copy(getlevel(bc,level)->debuginfo), 
+                                            jv_string("params")), idx);
         }
+        printf(" %s:%d",
+               jv_string_value(name),
+               idx);
+        jv_free(name);
         if (level) {
           printf("^%d", level);
         }
       }
+    } else if (op->op == CALL_BUILTIN) {
+      int func = bc->code[pc++];
+      jv name = jv_array_get(jv_copy(bc->globals->cfunc_names), func);
+      printf(" %s", jv_string_value(name));
+      jv_free(name);
     } else if (op->flags & OP_HAS_BRANCH) {
       printf(" %04d", pc + imm);
     } else if (op->flags & OP_HAS_CONSTANT) {
@@ -61,7 +131,11 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
       jv_dump(jv_array_get(jv_copy(bc->constants), imm), 0);
     } else if (op->flags & OP_HAS_VARIABLE) {
       uint16_t v = bc->code[pc++];
-      printf(" v%d", v);
+      jv name = jv_array_get(jv_object_get(jv_copy(getlevel(bc,imm)->debuginfo), jv_string("locals")), v);
+      printf(" $%s:%d", 
+             jv_string_value(name), 
+             v);
+      jv_free(name);
       if (imm) {
         printf("^%d", imm);
       }
@@ -71,18 +145,16 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
   }  
 }
 
-void symbol_table_free(struct symbol_table* syms) {
-  free(syms->cfunctions);
-  free(syms);
-}
-
 void bytecode_free(struct bytecode* bc) {
-  free(bc->code);
+  if (!bc)
+    return;
+  jv_mem_free(bc->code);
   jv_free(bc->constants);
   for (int i=0; i<bc->nsubfunctions; i++)
     bytecode_free(bc->subfunctions[i]);
   if (!bc->parent)
     symbol_table_free(bc->globals);
-  free(bc->subfunctions);
-  free(bc);
+  jv_mem_free(bc->subfunctions);
+  jv_free(bc->debuginfo);
+  jv_mem_free(bc);
 }

bytecode.h

@@ -3,20 +3,56 @@
 #include <stdint.h>
 
 #include "jv.h"
-#include "opcode.h"
 
-typedef void (*cfunction_ptr)(jv input[], jv output[]);
+typedef enum {
+#define OP(name, imm, in, out) name,
+#include "opcode_list.h"
+#undef OP
+} opcode;
 
+enum {
+  NUM_OPCODES = 
+#define OP(name, imm, in, out) +1
+#include "opcode_list.h"
+#undef OP
+};
+
+enum {
+  OP_HAS_CONSTANT = 2,
+  OP_HAS_VARIABLE = 4,
+  OP_HAS_BRANCH = 8,
+  OP_HAS_CFUNC = 32,
+  OP_HAS_UFUNC = 64,
+  OP_IS_CALL_PSEUDO = 128,
+  OP_HAS_BINDING = 1024,
+};
+struct opcode_description {
+  opcode op;
+  const char* name;
+
+  int flags;
+
+  // length in 16-bit units
+  int length;
+
+  int stack_in, stack_out;
+};
+
+const struct opcode_description* opcode_describe(opcode op);
+
+
+#define MAX_CFUNCTION_ARGS 10
+typedef void (*cfunction_ptr)();
 struct cfunction {
   cfunction_ptr fptr;
   const char* name;
-  opcode callop;
+  int nargs;
 };
 
-#define MAX_CFUNCTION_ARGS 10
 struct symbol_table {
   struct cfunction* cfunctions;
   int ncfunctions;
+  jv cfunc_names;
 };
 
 // The bytecode format matters in:
@@ -40,13 +76,13 @@ struct bytecode {
   int nsubfunctions;
 
   struct bytecode* parent;
+
+  jv debuginfo;
 };
 
 void dump_disassembly(int, struct bytecode* code);
-void dump_code(int, struct bytecode* code);
 void dump_operation(struct bytecode* bc, uint16_t* op);
 
-void symbol_table_free(struct symbol_table* syms);
 void bytecode_free(struct bytecode* bc);
 
 #endif

compile.c

@@ -1,36 +1,54 @@
+#define _GNU_SOURCE // for strdup
 #include <assert.h>
 #include <string.h>
 #include <stdlib.h>
-#include "opcode.h"
 #include "compile.h"
+#include "bytecode.h"
 #include "locfile.h"
+#include "jv_alloc.h"
 
+/*
+  The intermediate representation for jq filters is as a sequence of
+  struct inst, which form a doubly-linked list via the next and prev
+  pointers. 
+
+  A "block" represents a sequence of "struct inst", which may be
+  empty.
+
+  Blocks are generated by the parser bottom-up, so may have free
+  variables (refer to things not defined). See inst.bound_by and
+  inst.symbol.
+ */
 struct inst {
   struct inst* next;
   struct inst* prev;
 
   opcode op;
   
-  union {
+  struct {
     uint16_t intval;
     struct inst* target;
     jv constant;
-    struct cfunction* cfunc;
+    const struct cfunction* cfunc;
   } imm;
 
   location source;
 
   // Binding
-  // An instruction requiring binding (for parameters/variables)
+  // An instruction requiring binding (for parameters/variables/functions)
   // is in one of three states:
-  //   bound_by = NULL  - Unbound free variable
-  //   bound_by = self  - This instruction binds a variable
-  //   bound_by = other - Uses variable bound by other instruction
-  // The immediate field is generally not meaningful until instructions
-  // are bound, and even then only for instructions which bind.
+  //   inst->bound_by = NULL  - Unbound free variable
+  //   inst->bound_by = inst  - This instruction binds a variable
+  //   inst->bound_by = other - Uses variable bound by other instruction
+  // Unbound instructions (references to other things that may or may not 
+  // exist) are created by "gen_foo_unbound", and bindings are created by
+  // block_bind(definition, body), which binds all instructions in
+  // body which are unboudn and refer to "definition" by name.
   struct inst* bound_by;
   char* symbol;
-  block subfn;
+
+  block subfn;   // used by CLOSURE_CREATE (body of function)
+  block arglist; // used by CLOSURE_CREATE (formals) and CALL_JQ (arguments)
 
   // This instruction is compiled as part of which function?
   // (only used during block_compile)
@@ -40,26 +58,26 @@ struct inst {
 };
 
 static inst* inst_new(opcode op) {
-  inst* i = malloc(sizeof(inst));
+  inst* i = jv_mem_alloc(sizeof(inst));
   i->next = i->prev = 0;
   i->op = op;
   i->bytecode_pos = -1;
   i->bound_by = 0;
   i->symbol = 0;
   i->subfn = gen_noop();
+  i->arglist = gen_noop();
   i->source = UNKNOWN_LOCATION;
   return i;
 }
 
 static void inst_free(struct inst* i) {
-  free(i->symbol);
-  if (opcode_describe(i->op)->flags & OP_HAS_BLOCK) {
-    block_free(i->subfn);
-  }
+  jv_mem_free(i->symbol);
+  block_free(i->subfn);
+  block_free(i->arglist);
   if (opcode_describe(i->op)->flags & OP_HAS_CONSTANT) {
     jv_free(i->imm.constant);
   }
-  free(i);
+  jv_mem_free(i);
 }
 
 static block inst_block(inst* i) {
@@ -106,9 +124,9 @@ block gen_op_simple(opcode op) {
 }
 
 
-block gen_op_const(opcode op, jv constant) {
-  assert(opcode_describe(op)->flags & OP_HAS_CONSTANT);
-  inst* i = inst_new(op);
+block gen_const(jv constant) {
+  assert(opcode_describe(LOADK)->flags & OP_HAS_CONSTANT);
+  inst* i = inst_new(LOADK);
   i->imm.constant = constant;
   return inst_block(i);
 }
@@ -134,83 +152,26 @@ void inst_set_target(block b, block target) {
   b.first->imm.target = target.last;
 }
 
-block gen_op_var_unbound(opcode op, const char* name) {
+block gen_op_unbound(opcode op, const char* name) {
+  assert(opcode_describe(op)->flags & OP_HAS_BINDING);
+  inst* i = inst_new(op);
+  i->symbol = strdup(name);
+  return inst_block(i);
+}
+
+block gen_op_var_fresh(opcode op, const char* name) {
   assert(opcode_describe(op)->flags & OP_HAS_VARIABLE);
-  inst* i = inst_new(op);
-  i->symbol = strdup(name);
-  return inst_block(i);
+  return block_bind(gen_op_unbound(op, name),
+                    gen_noop(), OP_HAS_VARIABLE);
 }
 
-block gen_op_var_bound(opcode op, block binder) {
+block gen_op_bound(opcode op, block binder) {
   assert(block_is_single(binder));
-  block b = gen_op_var_unbound(op, binder.first->symbol);
+  block b = gen_op_unbound(op, binder.first->symbol);
   b.first->bound_by = binder.first;
   return b;
 }
 
-block gen_op_symbol(opcode op, const char* sym) {
-  assert(opcode_describe(op)->flags & OP_HAS_SYMBOL);
-  inst* i = inst_new(op);
-  i->symbol = strdup(sym);
-  return inst_block(i);
-}
-
-block gen_op_block_defn(opcode op, const char* name, block block) {
-  assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO);
-  assert(opcode_describe(op)->flags & OP_HAS_BLOCK);
-  inst* i = inst_new(op);
-  i->subfn = block;
-  i->symbol = strdup(name);
-  return inst_block(i);
-}
-
-static void block_bind_subblock(block binder, block body, int bindflags);
-block gen_op_block_defn_rec(opcode op, const char* name, block blk) {
-  block b = gen_op_block_defn(op, name, blk);
-  block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING);
-  return b;
-}
-
-block gen_op_block_unbound(opcode op, const char* name) {
-  assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO);
-  inst* i = inst_new(op);
-  i->symbol = strdup(name);
-  return inst_block(i);
-}
-
-block gen_op_block_bound(opcode op, block binder) {
-  assert(block_is_single(binder));
-  block b = gen_op_block_unbound(op, binder.first->symbol);
-  b.first->bound_by = binder.first;
-  return b;
-}
-
-block gen_op_call(opcode op, block arglist) {
-  assert(opcode_describe(op)->flags & OP_HAS_VARIABLE_LENGTH_ARGLIST);
-  inst* i = inst_new(op);
-  block prelude = gen_noop();
-  block call = inst_block(i);
-  int nargs = 0;
-  inst* curr = 0;
-  while ((curr = block_take(&arglist))) {
-    assert(opcode_describe(curr->op)->flags & OP_IS_CALL_PSEUDO);
-    block bcurr = inst_block(curr);
-    switch (curr->op) {
-    default: assert(0 && "Unknown type of parameter"); break;
-    case CLOSURE_REF:
-      block_append(&call, bcurr);
-      break;
-    case CLOSURE_CREATE:
-      block_append(&prelude, bcurr);
-      block_append(&call, gen_op_block_bound(CLOSURE_REF, bcurr));
-      break;
-    }
-    nargs++;
-  }
-  assert(nargs < 100); //FIXME
-  i->imm.intval = nargs;
-  return block_join(prelude, call);
-}
 
 static void inst_join(inst* a, inst* b) {
   assert(a && b);
@@ -247,13 +208,14 @@ int block_has_only_binders(block binders, int bindflags) {
   return 1;
 }
 
-static void block_bind_subblock(block binder, block body, int bindflags) {
+static int block_bind_subblock(block binder, block body, int bindflags) {
   assert(block_is_single(binder));
   assert((opcode_describe(binder.first->op)->flags & bindflags) == bindflags);
   assert(binder.first->symbol);
   assert(binder.first->bound_by == 0 || binder.first->bound_by == binder.first);
 
   binder.first->bound_by = binder.first;
+  int nrefs = 0;
   for (inst* i = body.first; i; i = i->next) {
     int flags = opcode_describe(i->op)->flags;
     if ((flags & bindflags) == bindflags &&
@@ -261,175 +223,189 @@ static void block_bind_subblock(block binder, block body, int bindflags) {
         !strcmp(i->symbol, binder.first->symbol)) {
       // bind this instruction
       i->bound_by = binder.first;
+      nrefs++;
     }
-    if (flags & OP_HAS_BLOCK) {
-      // binding recurses into closures
-      block_bind_subblock(binder, i->subfn, bindflags);
-    }
+    // binding recurses into closures
+    nrefs += block_bind_subblock(binder, i->subfn, bindflags);
+    // binding recurses into argument list
+    nrefs += block_bind_subblock(binder, i->arglist, bindflags);
   }
+  return nrefs;
 }
 
-block block_bind(block binder, block body, int bindflags) {
+static void block_bind_each(block binder, block body, int bindflags) {
   assert(block_has_only_binders(binder, bindflags));
   bindflags |= OP_HAS_BINDING;
   for (inst* curr = binder.first; curr; curr = curr->next) {
     block_bind_subblock(inst_block(curr), body, bindflags);
   }
+}
+
+block block_bind(block binder, block body, int bindflags) {
+  block_bind_each(binder, body, bindflags);
   return block_join(binder, body);
 }
 
+block block_bind_referenced(block binder, block body, int bindflags) {
+  assert(block_has_only_binders(binder, bindflags));
+  bindflags |= OP_HAS_BINDING;
+  block refd = gen_noop();
+  for (inst* curr; (curr = block_take(&binder));) {
+    block b = inst_block(curr);
+    if (block_bind_subblock(b, body, bindflags)) {
+      refd = BLOCK(refd, b);
+    } else {
+      block_free(b);
+    }
+  }
+  return block_join(refd, body);
+}
+
+block gen_function(const char* name, block formals, block body) {
+  block_bind_each(formals, body, OP_IS_CALL_PSEUDO);
+  inst* i = inst_new(CLOSURE_CREATE);
+  i->subfn = body;
+  i->symbol = strdup(name);
+  i->arglist = formals;
+  block b = inst_block(i);
+  block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING);
+  return b;
+}
+
+block gen_param(const char* name) {
+  return gen_op_unbound(CLOSURE_PARAM, name);
+}
+
+block gen_lambda(block body) {
+  return gen_function("@lambda", gen_noop(), body);
+}
+
+block gen_call(const char* name, block args) {
+  block b = gen_op_unbound(CALL_JQ, name);
+  b.first->arglist = args;
+  return b;
+}
+
+
 
 block gen_subexp(block a) {
-  block c = gen_noop();
-  block_append(&c, gen_op_simple(DUP));
-  block_append(&c, a);
-  block_append(&c, gen_op_simple(SWAP));
-  return c;
+  return BLOCK(gen_op_simple(SUBEXP_BEGIN), a, gen_op_simple(SUBEXP_END));
 }
 
 block gen_both(block a, block b) {
-  block c = gen_noop();
   block jump = gen_op_targetlater(JUMP);
-  block fork = gen_op_targetlater(FORK);
-  block_append(&c, fork);
-  block_append(&c, a);
-  block_append(&c, jump);
-  inst_set_target(fork, c);
-  block_append(&c, b);
+  block fork = gen_op_target(FORK, jump);
+  block c = BLOCK(fork, a, jump, b);
   inst_set_target(jump, c);
   return c;
 }
 
 
 block gen_collect(block expr) {
-  block c = gen_noop();
-  block_append(&c, gen_op_simple(DUP));
-  block_append(&c, gen_op_const(LOADK, jv_array()));
-  block array_var = block_bind(gen_op_var_unbound(STOREV, "collect"),
-                               gen_noop(), OP_HAS_VARIABLE);
-  block_append(&c, array_var);
+  block array_var = gen_op_var_fresh(STOREV, "collect");
+  block c = BLOCK(gen_op_simple(DUP), gen_const(jv_array()), array_var);
 
-  block tail = {0};
-  block_append(&tail, gen_op_simple(DUP));
-  block_append(&tail, gen_op_var_bound(LOADV, array_var));
-  block_append(&tail, gen_op_simple(SWAP));
-  block_append(&tail, gen_op_simple(APPEND));
-  block_append(&tail, gen_op_var_bound(STOREV, array_var));
-  block_append(&tail, gen_op_simple(BACKTRACK));
+  block tail = BLOCK(gen_op_bound(APPEND, array_var),
+                     gen_op_simple(BACKTRACK));
 
-  block_append(&c, gen_op_target(FORK, tail));
-  block_append(&c, expr);
-  block_append(&c, tail);
-
-  block_append(&c, gen_op_var_bound(LOADV, array_var));
-
-  return c;
+  return BLOCK(c,
+               gen_op_target(FORK, tail),
+               expr, 
+               tail,
+               gen_op_bound(LOADVN, array_var));
 }
 
-block gen_assign(block expr) {
-  block c = gen_noop();
-  block_append(&c, gen_op_simple(DUP));
-  block result_var = block_bind(gen_op_var_unbound(STOREV, "result"),
-                                gen_noop(), OP_HAS_VARIABLE);
-  block_append(&c, result_var);
-
-  block loop = gen_noop();
-  block_append(&loop, gen_op_simple(DUP));
-  block_append(&loop, expr);
-  block_append(&loop, gen_op_var_bound(ASSIGN, result_var));
-  block_append(&loop, gen_op_simple(BACKTRACK));
-
-  block_append(&c, gen_op_target(FORK, loop));
-  block_append(&c, loop);
-  block_append(&c, gen_op_var_bound(LOADV, result_var));
-  return c;
+block gen_reduce(const char* varname, block source, block init, block body) {
+  block res_var = gen_op_var_fresh(STOREV, "reduce");
+  block loop = BLOCK(gen_op_simple(DUP),
+                     source,
+                     block_bind(gen_op_unbound(STOREV, varname),
+                                BLOCK(gen_op_bound(LOADVN, res_var),
+                                      body,
+                                      gen_op_bound(STOREV, res_var)),
+                                OP_HAS_VARIABLE),
+                     gen_op_simple(BACKTRACK));
+  return BLOCK(gen_op_simple(DUP),
+               init,
+               res_var,
+               gen_op_target(FORK, loop),
+               loop,
+               gen_op_bound(LOADVN, res_var));
 }
 
 block gen_definedor(block a, block b) {
   // var found := false
-  block c = gen_op_simple(DUP);
-  block_append(&c, gen_op_const(LOADK, jv_false()));
-  block found_var = block_bind(gen_op_var_unbound(STOREV, "found"),
-                               gen_noop(), OP_HAS_VARIABLE);
-  block_append(&c, found_var);
+  block found_var = gen_op_var_fresh(STOREV, "found");
+  block init = BLOCK(gen_op_simple(DUP), gen_const(jv_false()), found_var);
 
   // if found, backtrack. Otherwise execute b
-  block tail = gen_op_simple(DUP);
-  block_append(&tail, gen_op_var_bound(LOADV, found_var));
   block backtrack = gen_op_simple(BACKTRACK);
-  block_append(&tail, gen_op_target(JUMP_F, backtrack));
-  block_append(&tail, backtrack);
-  block_append(&tail, gen_op_simple(POP));
-  block_append(&tail, b);
+  block tail = BLOCK(gen_op_simple(DUP),
+                     gen_op_bound(LOADV, found_var),
+                     gen_op_target(JUMP_F, backtrack),
+                     backtrack,
+                     gen_op_simple(POP),
+                     b);
 
   // try again
   block if_notfound = gen_op_simple(BACKTRACK);
 
   // found := true, produce result
-  block if_found = gen_op_simple(DUP);
-  block_append(&if_found, gen_op_const(LOADK, jv_true()));
-  block_append(&if_found, gen_op_var_bound(STOREV, found_var));
-  block_append(&if_found, gen_op_target(JUMP, tail));
+  block if_found = BLOCK(gen_op_simple(DUP),
+                         gen_const(jv_true()),
+                         gen_op_bound(STOREV, found_var),
+                         gen_op_target(JUMP, tail));
 
-  block_append(&c, gen_op_target(FORK, if_notfound));
-  block_append(&c, a);
-  block_append(&c, gen_op_target(JUMP_F, if_found));
-  block_append(&c, if_found);
-  block_append(&c, if_notfound);
-  block_append(&c, tail);
-
-  return c;
+  return BLOCK(init,
+               gen_op_target(FORK, if_notfound),
+               a,
+               gen_op_target(JUMP_F, if_found),
+               if_found,
+               if_notfound,
+               tail);
 }
 
 block gen_condbranch(block iftrue, block iffalse) {
-  block b = gen_noop();
-  block_append(&iftrue, gen_op_target(JUMP, iffalse));
-  block_append(&b, gen_op_target(JUMP_F, iftrue));
-  block_append(&b, iftrue);
-  block_append(&b, iffalse);
-  return b;
+  iftrue = BLOCK(iftrue, gen_op_target(JUMP, iffalse));
+  return BLOCK(gen_op_target(JUMP_F, iftrue), iftrue, iffalse);
 }
 
 block gen_and(block a, block b) {
   // a and b = if a then (if b then true else false) else false
-  block code = gen_op_simple(DUP);
-  block_append(&code, a);
-
-  block if_a_true = gen_op_simple(POP);
-  block_append(&if_a_true, b);
-  block_append(&if_a_true, gen_condbranch(gen_op_const(LOADK, jv_true()),
-                                          gen_op_const(LOADK, jv_false())));
-  block_append(&code, gen_condbranch(if_a_true,
-                                     block_join(gen_op_simple(POP), gen_op_const(LOADK, jv_false()))));
-  return code;
+  return BLOCK(gen_op_simple(DUP), a, 
+               gen_condbranch(BLOCK(gen_op_simple(POP),
+                                    b,
+                                    gen_condbranch(gen_const(jv_true()),
+                                                   gen_const(jv_false()))),
+                              BLOCK(gen_op_simple(POP), gen_const(jv_false()))));
 }
 
 block gen_or(block a, block b) {
   // a or b = if a then true else (if b then true else false)
-  block code = gen_op_simple(DUP);
-  block_append(&code, a);
+  return BLOCK(gen_op_simple(DUP), a,
+               gen_condbranch(BLOCK(gen_op_simple(POP), gen_const(jv_true())),
+                              BLOCK(gen_op_simple(POP),
+                                    b,
+                                    gen_condbranch(gen_const(jv_true()),
+                                                   gen_const(jv_false())))));
+}
 
-  block if_a_false = gen_op_simple(POP);
-  block_append(&if_a_false, b);
-  block_append(&if_a_false, gen_condbranch(gen_op_const(LOADK, jv_true()),
-                                           gen_op_const(LOADK, jv_false())));
-  block_append(&code, gen_condbranch(block_join(gen_op_simple(POP), gen_op_const(LOADK, jv_true())),
-                                     if_a_false));
-  return code;
+block gen_var_binding(block var, const char* name, block body) {
+  return BLOCK(gen_op_simple(DUP), var,
+               block_bind(gen_op_unbound(STOREV, name),
+                          body, OP_HAS_VARIABLE));
 }
 
 block gen_cond(block cond, block iftrue, block iffalse) {
-  block b = gen_op_simple(DUP);
-  block_append(&b, cond);
-  block_append(&b, gen_condbranch(block_join(gen_op_simple(POP), iftrue),
-                                  block_join(gen_op_simple(POP), iffalse)));
-  return b;
+  return BLOCK(gen_op_simple(DUP), cond, 
+               gen_condbranch(BLOCK(gen_op_simple(POP), iftrue),
+                              BLOCK(gen_op_simple(POP), iffalse)));
 }
 
-block gen_cbinding(struct symbol_table* t, block code) {
-  for (int cfunc=0; cfunc<t->ncfunctions; cfunc++) {
+block gen_cbinding(const struct cfunction* cfunctions, int ncfunctions, block code) {
+  for (int cfunc=0; cfunc<ncfunctions; cfunc++) {
     inst* i = inst_new(CLOSURE_CREATE_C);
-    i->imm.cfunc = &t->cfunctions[cfunc];
+    i->imm.cfunc = &cfunctions[cfunc];
     i->symbol = strdup(i->imm.cfunc->name);
     code = block_bind(inst_block(i), code, OP_IS_CALL_PSEUDO);
   }
@@ -451,66 +427,169 @@ static int count_cfunctions(block b) {
   int n = 0;
   for (inst* i = b.first; i; i = i->next) {
     if (i->op == CLOSURE_CREATE_C) n++;
-    if (opcode_describe(i->op)->flags & OP_HAS_BLOCK)
-      n += count_cfunctions(i->subfn);
+    n += count_cfunctions(i->subfn);
   }
   return n;
 }
 
+
+// Expands call instructions into a calling sequence
+static int expand_call_arglist(struct locfile* locations, block* b) {
+  int errors = 0;
+  block ret = gen_noop();
+  for (inst* curr; (curr = block_take(b));) {
+    if (opcode_describe(curr->op)->flags & OP_HAS_BINDING) {
+      if (!curr->bound_by) {
+        locfile_locate(locations, curr->source, "error: %s is not defined", curr->symbol);
+        errors++;
+        // don't process this instruction if it's not well-defined
+        ret = BLOCK(ret, inst_block(curr));
+        continue;
+      }
+    }
+
+    block prelude = gen_noop();
+    if (curr->op == CALL_JQ) {
+      int actual_args = 0, desired_args = 0;
+      // We expand the argument list as a series of instructions
+      switch (curr->bound_by->op) {
+      default: assert(0 && "Unknown function type"); break;
+      case CLOSURE_CREATE: 
+      case CLOSURE_PARAM: {
+        block callargs = gen_noop();
+        for (inst* i; (i = block_take(&curr->arglist));) {
+          assert(opcode_describe(i->op)->flags & OP_IS_CALL_PSEUDO);
+          block b = inst_block(i);
+          switch (i->op) {
+          default: assert(0 && "Unknown type of parameter"); break;
+          case CLOSURE_REF:
+            block_append(&callargs, b);
+            break;
+          case CLOSURE_CREATE:
+            block_append(&prelude, b);
+            block_append(&callargs, gen_op_bound(CLOSURE_REF, b));
+            break;
+          }
+          actual_args++;
+        }
+        curr->imm.intval = actual_args;
+        curr->arglist = callargs;
+
+        if (curr->bound_by->op == CLOSURE_CREATE) {
+          for (inst* i = curr->bound_by->arglist.first; i; i = i->next) {
+            assert(i->op == CLOSURE_PARAM);
+            desired_args++;
+          }
+        }
+        break;
+      }
+
+      case CLOSURE_CREATE_C: {
+        for (inst* i; (i = block_take(&curr->arglist)); ) {
+          assert(i->op == CLOSURE_CREATE); // FIXME
+          block body = i->subfn;
+          i->subfn = gen_noop();
+          inst_free(i);
+          // arguments should be pushed in reverse order, prepend them to prelude
+          errors += expand_call_arglist(locations, &body);
+          prelude = BLOCK(gen_subexp(body), prelude);
+          actual_args++;
+        }
+        assert(curr->op == CALL_JQ);
+        curr->op = CALL_BUILTIN;
+        curr->imm.intval = actual_args + 1 /* include the implicit input in arg count */;
+        assert(curr->bound_by->op == CLOSURE_CREATE_C);
+        desired_args = curr->bound_by->imm.cfunc->nargs - 1;
+        assert(!curr->arglist.first);
+        break;
+      }
+      }
+
+      if (actual_args != desired_args) {
+        locfile_locate(locations, curr->source, 
+                       "error: %s arguments to %s (expected %d but got %d)",
+                       actual_args > desired_args ? "too many" : "too few",
+                       curr->symbol, desired_args, actual_args);
+        errors++;
+      }
+
+    }
+    ret = BLOCK(ret, prelude, inst_block(curr));
+  }
+  *b = ret;
+  return errors;
+}
+
 static int compile(struct locfile* locations, struct bytecode* bc, block b) {
   int errors = 0;
   int pos = 0;
   int var_frame_idx = 0;
   bc->nsubfunctions = 0;
-  bc->nclosures = 0;
+  errors += expand_call_arglist(locations, &b);
+  b = BLOCK(b, gen_op_simple(RET));
+  jv localnames = jv_array();
   for (inst* curr = b.first; curr; curr = curr->next) {
     if (!curr->next) assert(curr == b.last);
-    pos += opcode_length(curr->op);
+    int length = opcode_describe(curr->op)->length;
+    if (curr->op == CALL_JQ) {
+      for (inst* arg = curr->arglist.first; arg; arg = arg->next) {
+        length += 2;
+      }
+    }
+    pos += length;
     curr->bytecode_pos = pos;
     curr->compiled = bc;
 
-    int opflags = opcode_describe(curr->op)->flags;
-    if (opflags & OP_HAS_BINDING) {
-      if (!curr->bound_by) {
-        locfile_locate(locations, curr->source, "error: %s is not defined", curr->symbol);
-        errors++;
-      }
-    }
-    if ((opflags & OP_HAS_VARIABLE) &&
+    assert(curr->op != CLOSURE_REF && curr->op != CLOSURE_PARAM);
+
+    if ((opcode_describe(curr->op)->flags & OP_HAS_VARIABLE) &&
         curr->bound_by == curr) {
       curr->imm.intval = var_frame_idx++;
+      localnames = jv_array_append(localnames, jv_string(curr->symbol));
     }
-    if (opflags & OP_HAS_BLOCK) {
+
+    if (curr->op == CLOSURE_CREATE) {
       assert(curr->bound_by == curr);
       curr->imm.intval = bc->nsubfunctions++;
     }
-    if (curr->op == CLOSURE_PARAM) {
-      assert(curr->bound_by == curr);
-      curr->imm.intval = bc->nclosures++;
-    }
     if (curr->op == CLOSURE_CREATE_C) {
       assert(curr->bound_by == curr);
       int idx = bc->globals->ncfunctions++;
+      bc->globals->cfunc_names = jv_array_append(bc->globals->cfunc_names,
+                                                 jv_string(curr->symbol));
       bc->globals->cfunctions[idx] = *curr->imm.cfunc;
       curr->imm.intval = idx;
     }
   }
+  bc->debuginfo = jv_object_set(bc->debuginfo, jv_string("locals"), localnames);
   if (bc->nsubfunctions) {
-    bc->subfunctions = malloc(sizeof(struct bytecode*) * bc->nsubfunctions);
+    bc->subfunctions = jv_mem_alloc(sizeof(struct bytecode*) * bc->nsubfunctions);
     for (inst* curr = b.first; curr; curr = curr->next) {
-      if (!(opcode_describe(curr->op)->flags & OP_HAS_BLOCK))
-        continue;
-      struct bytecode* subfn = malloc(sizeof(struct bytecode));
-      bc->subfunctions[curr->imm.intval] = subfn;
-      subfn->globals = bc->globals;
-      subfn->parent = bc;
-      errors += compile(locations, subfn, curr->subfn);
+      if (curr->op == CLOSURE_CREATE) {
+        struct bytecode* subfn = jv_mem_alloc(sizeof(struct bytecode));
+        bc->subfunctions[curr->imm.intval] = subfn;
+        subfn->globals = bc->globals;
+        subfn->parent = bc;
+        subfn->nclosures = 0;
+        subfn->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_string(curr->symbol));
+        jv params = jv_array();
+        for (inst* param = curr->arglist.first; param; param = param->next) {
+          assert(param->op == CLOSURE_PARAM);
+          assert(param->bound_by == param);
+          param->imm.intval = subfn->nclosures++;
+          param->compiled = subfn;
+          params = jv_array_append(params, jv_string(param->symbol));
+        }
+        subfn->debuginfo = jv_object_set(subfn->debuginfo, jv_string("params"), params);
+        errors += compile(locations, subfn, curr->subfn);
+        curr->subfn = gen_noop();
+      }
     }
   } else {
     bc->subfunctions = 0;
   }
   bc->codelen = pos;
-  uint16_t* code = malloc(sizeof(uint16_t) * bc->codelen);
+  uint16_t* code = jv_mem_alloc(sizeof(uint16_t) * bc->codelen);
   bc->code = code;
   pos = 0;
   jv constant_pool = jv_array();
@@ -519,80 +598,58 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
     const struct opcode_description* op = opcode_describe(curr->op);
     if (op->length == 0)
       continue;
-    uint16_t* opcode_rewrite = &code[pos];
     code[pos++] = curr->op;
-    int opflags = op->flags;
-    assert(!(op->flags & OP_IS_CALL_PSEUDO));
-    if (opflags & OP_HAS_VARIABLE_LENGTH_ARGLIST) {
-      int nargs = curr->imm.intval;
-      assert(nargs > 0);
-      code[pos++] = (uint16_t)nargs;
-      int desired_params = 0;
-      for (int i=0; i<nargs; i++) {
-        curr = curr->next;
-        assert(curr && opcode_describe(curr->op)->flags & OP_IS_CALL_PSEUDO);
-        code[pos++] = nesting_level(bc, curr->bound_by);
-        switch (curr->bound_by->op) {
-        default: assert(0 && "Unknown type of argument");
-        case CLOSURE_CREATE:
-          code[pos++] = curr->bound_by->imm.intval | ARG_NEWCLOSURE;
-          if (i == 0) {
-            inst* i = curr->bound_by;
-            desired_params = i->compiled->subfunctions[i->imm.intval]->nclosures;
-          }
-          break;
-        case CLOSURE_PARAM:
-          code[pos++] = curr->bound_by->imm.intval;
-          if (i == 0) desired_params = 0;
-          break;
-        case CLOSURE_CREATE_C:
-          code[pos++] = curr->bound_by->imm.intval;
-          *opcode_rewrite = bc->globals->cfunctions[curr->bound_by->imm.intval].callop;
-          if (i == 0) desired_params = 0;
-          break;
-        }
+    assert(curr->op != CLOSURE_REF && curr->op != CLOSURE_PARAM);
+    if (curr->op == CALL_BUILTIN) {
+      assert(curr->bound_by->op == CLOSURE_CREATE_C);
+      assert(!curr->arglist.first);
+      code[pos++] = (uint16_t)curr->imm.intval;
+      code[pos++] = curr->bound_by->imm.intval;
+    } else if (curr->op == CALL_JQ) {
+      assert(curr->bound_by->op == CLOSURE_CREATE ||
+             curr->bound_by->op == CLOSURE_PARAM);
+      code[pos++] = (uint16_t)curr->imm.intval;
+      code[pos++] = nesting_level(bc, curr->bound_by);
+      code[pos++] = curr->bound_by->imm.intval | 
+        (curr->bound_by->op == CLOSURE_CREATE ? ARG_NEWCLOSURE : 0);
+      for (inst* arg = curr->arglist.first; arg; arg = arg->next) {
+        assert(arg->op == CLOSURE_REF && arg->bound_by->op == CLOSURE_CREATE);
+        code[pos++] = nesting_level(bc, arg->bound_by);
+        code[pos++] = arg->bound_by->imm.intval | ARG_NEWCLOSURE;
       }
-      assert(nargs - 1 == desired_params);
-    } else if (opflags & OP_HAS_CONSTANT) {
+    } else if (op->flags & OP_HAS_CONSTANT) {
       code[pos++] = jv_array_length(jv_copy(constant_pool));
       constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));
-    } else if (opflags & OP_HAS_VARIABLE) {
+    } else if (op->flags & OP_HAS_VARIABLE) {
       code[pos++] = nesting_level(bc, curr->bound_by);
       uint16_t var = (uint16_t)curr->bound_by->imm.intval;
       code[pos++] = var;
       if (var > maxvar) maxvar = var;
-    } else if (opflags & OP_HAS_BRANCH) {
+    } else if (op->flags & OP_HAS_BRANCH) {
       assert(curr->imm.target->bytecode_pos != -1);
       assert(curr->imm.target->bytecode_pos > pos); // only forward branches
       code[pos] = curr->imm.target->bytecode_pos - (pos + 1);
       pos++;
-    } else if (opflags & OP_HAS_CFUNC) {
-      assert(curr->symbol);
-      int found = 0;
-      for (int i=0; i<bc->globals->ncfunctions; i++) {
-        if (!strcmp(curr->symbol, bc->globals->cfunctions[i].name)) {
-          code[pos++] = i;
-          found = 1;
-          break;
-        }
-      }
-      assert(found);
     } else if (op->length > 1) {
       assert(0 && "codegen not implemented for this operation");
     }
   }
   bc->constants = constant_pool;
   bc->nlocals = maxvar + 2; // FIXME: frames of size zero?
+  block_free(b);
   return errors;
 }
 
 int block_compile(block b, struct locfile* locations, struct bytecode** out) {
-  struct bytecode* bc = malloc(sizeof(struct bytecode));
+  struct bytecode* bc = jv_mem_alloc(sizeof(struct bytecode));
   bc->parent = 0;
-  bc->globals = malloc(sizeof(struct symbol_table));
+  bc->nclosures = 0;
+  bc->globals = jv_mem_alloc(sizeof(struct symbol_table));
   int ncfunc = count_cfunctions(b);
   bc->globals->ncfunctions = 0;
-  bc->globals->cfunctions = malloc(sizeof(struct cfunction) * ncfunc);
+  bc->globals->cfunctions = jv_mem_alloc(sizeof(struct cfunction) * ncfunc);
+  bc->globals->cfunc_names = jv_array();
+  bc->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_null());
   int nerrors = compile(locations, bc, b);
   assert(bc->globals->ncfunctions == ncfunc);
   if (nerrors > 0) {

compile.h

@@ -1,8 +1,8 @@
 #ifndef COMPILE_H
 #define COMPILE_H
 #include <stdint.h>
+#include "jv.h"
 #include "bytecode.h"
-#include "opcode.h"
 #include "locfile.h"
 
 struct inst;
@@ -18,33 +18,36 @@ block gen_location(location, block);
 
 block gen_noop();
 block gen_op_simple(opcode op);
-block gen_op_const(opcode op, jv constant);
+block gen_const(jv constant);
 block gen_op_target(opcode op, block target);
-block gen_op_var_unbound(opcode op, const char* name);
-block gen_op_var_bound(opcode op, block binder);
-block gen_op_block_defn(opcode op, const char* name, block block);
-block gen_op_block_defn_rec(opcode op, const char* name, block block);
-block gen_op_block_unbound(opcode op, const char* name);
-block gen_op_call(opcode op, block arglist);
-block gen_op_symbol(opcode op, const char* name);
+block gen_op_unbound(opcode op, const char* name);
+block gen_op_bound(opcode op, block binder);
+block gen_op_var_fresh(opcode op, const char* name);
 
+block gen_function(const char* name, block formals, block body);
+block gen_param(const char* name);
+block gen_lambda(block body);
+block gen_call(const char* name, block body);
 block gen_subexp(block a);
 block gen_both(block a, block b);
 block gen_collect(block expr);
-block gen_assign(block expr);
+block gen_reduce(const char* varname, block source, block init, block body);
 block gen_definedor(block a, block b);
 block gen_condbranch(block iftrue, block iffalse);
 block gen_and(block a, block b);
 block gen_or(block a, block b);
 
+block gen_var_binding(block var, const char* name, block body);
+
 block gen_cond(block cond, block iftrue, block iffalse);
 
-block gen_cbinding(struct symbol_table* functions, block b);
+block gen_cbinding(const struct cfunction* functions, int nfunctions, block b);
 
 void block_append(block* b, block b2);
 block block_join(block a, block b);
 int block_has_only_binders(block, int bindflags);
 block block_bind(block binder, block body, int bindflags);
+block block_bind_referenced(block binder, block body, int bindflags);
 
 int block_compile(block, struct locfile*, struct bytecode**);
 
@@ -52,4 +55,20 @@ void block_free(block);
 
 
 
+// Here's some horrible preprocessor gunk so that code
+// sequences can be contructed as BLOCK(block1, block2, block3)
+
+#define BLOCK_1(b1) (b1)
+#define BLOCK_2(b1,b2) (block_join((b1),(b2)))
+#define BLOCK_3(b1,b2,b3) (block_join(BLOCK_2(b1,b2),(b3)))
+#define BLOCK_4(b1,b2,b3,b4) (block_join(BLOCK_3(b1,b2,b3),(b4)))
+#define BLOCK_5(b1,b2,b3,b4,b5) (block_join(BLOCK_4(b1,b2,b3,b4),(b5)))
+#define BLOCK_6(b1,b2,b3,b4,b5,b6) (block_join(BLOCK_5(b1,b2,b3,b4,b5),(b6)))
+#define BLOCK_7(b1,b2,b3,b4,b5,b6,b7) (block_join(BLOCK_6(b1,b2,b3,b4,b5,b6),(b7)))
+
+#define BLOCK_IDX(_1,_2,_3,_4,_5,_6,_7,NAME,...) NAME
+#define BLOCK(...) \
+  BLOCK_IDX(__VA_ARGS__, BLOCK_7, BLOCK_6, BLOCK_5, BLOCK_4, BLOCK_3, BLOCK_2, BLOCK_1)(__VA_ARGS__)
+
+
 #endif

config/.gitignore

@@ -0,0 +1,8 @@
+config.guess
+config.sub
+compile
+depcomp
+install-sh
+ltmain.sh
+missing
+ylwrap

config/m4/.gitignore

@@ -0,0 +1,2 @@
+libtool.m4
+lt*.m4

configure.ac

@@ -0,0 +1,74 @@
+AC_INIT([jq], [1.3], [mu@netsoc.tcd.ie],
+             [jq], [http://stedolan.github.com/jq/])
+
+dnl Created autoconf implementation thompson@dtosolutions, 26NOV12
+AC_PREREQ([2.61])
+AC_CONFIG_AUX_DIR([config])
+AM_INIT_AUTOMAKE([parallel-tests foreign])
+AM_SILENT_RULES([yes])
+AC_PROG_CC
+AC_PROG_CC_STDC
+AC_PROG_CPP_WERROR
+AC_PROG_YACC
+AC_OBJEXT
+AC_EXEEXT
+LT_INIT([shared static win32-dll])
+AM_PROG_CC_C_O
+
+dnl couldn't use AM_PROG_LEX as it doesn't support header files like the
+dnl AC_PROG_YACC macros...
+
+dnl
+dnl these program checks should probably be deleted
+dnl
+
+AC_PROG_LEX
+if test "x$LEX" != xflex; then
+  LEX="$SHELL $missing_dir/missing flex"
+fi
+
+
+dnl Check for valgrind
+AC_CHECK_PROGS(valgrind_cmd, valgrind)
+if test "x$valgrind_cmd" = "x" ; then
+    AC_MSG_WARN([valgrind is required to test jq.])
+fi
+
+
+dnl Don't attempt to build docs if there's no Ruby lying around
+AC_ARG_ENABLE([docs],
+   AC_HELP_STRING([--disable-docs], [don't build docs]))
+
+AS_IF([test "x$enable_docs" != "xno"],[
+   AC_CHECK_PROGS(bundle_cmd, bundle)
+
+   AC_CACHE_CHECK([for Ruby dependencies], [jq_cv_ruby_deps],
+     [jq_cv_ruby_deps=yes;
+        AS_IF([test "x$bundle_cmd" = "x" || \
+        ! bmsg="`cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null`"],[
+           AC_MSG_WARN([$bmsg])
+           cat <<EOF
+*****************************************************************
+*  Ruby dependencies for building jq documentation not found.   *
+*  You can still build, install and hack on jq, but the manpage *
+*  will not be rebuilt and some of the tests won't run.         *
+*  See docs/README.md for how to install the docs dependencies. *
+*****************************************************************
+EOF
+           jq_cv_ruby_deps=no
+     ])])
+
+   if test "x$jq_cv_ruby_deps" != "xyes"; then
+     enable_docs=no
+   fi
+])
+
+AM_CONDITIONAL([ENABLE_DOCS], [test "x$enable_docs" != xno])
+AC_SUBST([BUNDLER], ["$bundle_cmd"])
+
+AC_CONFIG_MACRO_DIR([config/m4])
+AM_INIT_AUTOMAKE([-Wall])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+
+

docs/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem "rake"
+gem "bonsai"
+gem "maruku"
+gem "ronn"

docs/Gemfile.lock

@@ -0,0 +1,63 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    POpen4 (0.1.4)
+      Platform (>= 0.4.0)
+      open4
+    Platform (0.4.0)
+    activesupport (3.2.12)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    addressable (2.3.4)
+    bonsai (1.4.8)
+      activesupport (>= 3.0.3)
+      builder (>= 3.0.0)
+      i18n (>= 0.5.0)
+      launchy (>= 0.3.7)
+      liquid (>= 2.2.2)
+      maruku (>= 0.6.0)
+      rack
+      sass
+      sinatra (>= 1.0)
+      tilt (>= 1.3)
+      watch (>= 0.1.0)
+      yui-compressor
+    builder (3.2.0)
+    hpricot (0.8.6)
+    i18n (0.6.4)
+    launchy (2.3.0)
+      addressable (~> 2.3)
+    liquid (2.5.0)
+    maruku (0.6.1)
+      syntax (>= 1.0.0)
+    multi_json (1.7.3)
+    mustache (0.99.4)
+    open4 (1.3.0)
+    rack (1.5.2)
+    rack-protection (1.5.0)
+      rack
+    rake (10.0.3)
+    rdiscount (2.0.7.2)
+    ronn (0.7.3)
+      hpricot (>= 0.8.2)
+      mustache (>= 0.7.0)
+      rdiscount (>= 1.5.8)
+    sass (3.2.8)
+    sinatra (1.4.2)
+      rack (~> 1.5, >= 1.5.2)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    syntax (1.0.0)
+    tilt (1.4.0)
+    watch (0.1.0)
+    yui-compressor (0.9.6)
+      POpen4 (>= 0.1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bonsai
+  maruku
+  rake
+  ronn

docs/README.md

@@ -0,0 +1,25 @@
+Documentation
+=============
+
+The jq website, manpages and some of the tests are generated from this
+directory. The directory holds a [Bonsai](http://tinytree.info)
+website, and the manual is a YAML file in `content/3.manual`.
+
+To build the documentation (including building the jq manpage), you'll
+need a working Ruby setup. The easiest way to get one is to install
+RVM and Ruby 1.9.3 like so:
+
+    \curl -L https://get.rvm.io | bash -s stable --ruby=1.9.3
+
+After that finishes installing, you'll need to make sure RVM is on
+your path by doing `source $HOME/.rvm/scripts/rvm`, or just opening a
+new shell. See <http://rvm.io> for more info on RVM.
+
+Once RVM is installed, you can install all the dependencies for jq's
+documentation build by running this from the `docs` directory:
+
+    bundle install
+
+When bundle manages to install the dependencies, rerun `./configure`
+in the jq root directory and then the Makefile will be able to
+generate the jq manpage.

docs/Rakefile

@@ -1,7 +1,11 @@
+require 'yaml'
 require 'bonsai'
 require 'liquid'
 require 'maruku'
 require 'json'
+require 'ronn'
+require 'tempfile'
+require 'yaml'
 
 module ExtraFilters
   def markdownify(input)
@@ -38,11 +42,13 @@ task :serve do
     
     server = fork {        
       app = Rack::Builder.app {
-        use Bonsai::StaticPassThrough, :root => Bonsai.root_dir + "/output", :urls => ["/"]
+        map "/jq" do
+          use Bonsai::StaticPassThrough, :root => Bonsai.root_dir + "/output", :urls => ["/"]
+        end
         run Bonsai::DevelopmentServer
       }
       Rack::Handler.default.run(app, :Port => 5000) do
-        Launchy.open("http://localhost:5000/")
+        Launchy.open("http://localhost:5000/jq/")
       end
     }    
     Watch.new("{content,templates,public}/**/*") { Bonsai::Exporter.process! }
@@ -57,3 +63,84 @@ task :build do
   Bonsai.root_dir = Dir.pwd
   Bonsai::Exporter.publish!
 end
+
+$BINARIES = {
+  :osx32 => "--host=i686-apple-darwin10 CFLAGS='-m32 -g -O'",
+  :osx64 => "--host=i686-apple-darwin10 CFLAGS='-m64 -g -O'",
+  :win32 => "--host=i686-w64-mingw32 CFLAGS='-g -O'",
+  :win64 => "--host=x86_64-w64-mingw32 CFLAGS='-g -O'",
+  :linux32 => "--host=x86_64-linux-gnu CFLAGS='-m32 -g -O'",
+  :linux64 => "--host=x86_64-linux-gnu CFLAGS='-m64 -g -O'"
+}
+
+$BINARIES.each do |name, args|
+  file "../build/#{name}" do |t|
+    sh "MAKEFLAGS=-j4 ../scripts/crosscompile #{name} #{args}"
+  end
+  task :binaries => ["../build/#{name}"]
+end
+
+task :binaries => ["output/download"] do
+  $BINARIES.each do |name, args|
+    FileUtils.cp_r "../build/#{name}", "output/download/"
+  end
+end
+
+def load_manual
+  YAML::ENGINE.yamler = 'syck'
+  YAML::load(File.open("content/3.manual/manual.yml"))
+end
+
+task :manpage do
+  Tempfile.open "manpage" do |f|
+    manual = load_manual
+    f.puts manual['manpage_intro']
+    f.puts manual['body']
+    manual['sections'].each do |section|
+      
+      f.puts "## #{section['title'].upcase}\n"
+      f.puts section['body']
+      f.puts ""
+      (section['entries'] || []).each do |entry|
+        f.puts "### #{entry['title']}\n"
+        f.puts entry['body']
+        f.puts ""
+        (entry['examples'] || []).each do |example|
+          f.puts "    jq '#{example['program']}'"
+          f.puts "       #{example['input']}"
+          f.puts "    => #{example['output'].join(", ")}"
+          f.puts
+        end
+      end
+      f.puts ""
+    end
+    f.puts manual['manpage_epilogue']
+    f.close
+    puts Ronn::Document.new(f.path).convert('roff').gsub(/<\/?code>/,"")
+  end  
+end
+
+task :manpage_default => ["default_manpage.md"] do
+  puts Ronn::Document.new("default_manpage.md").convert('roff').gsub(/<\/?code>/,"")
+end
+
+task :mantests do
+  load_manual['sections'].each do |section|
+    (section['entries'] || []).each do |entry|
+      (entry['examples'] || []).each do |example|
+        puts example['program'].gsub("\n", " ")
+        puts example['input']
+        example['output'].each do |s| puts s end
+        puts
+      end
+    end
+  end
+end
+
+directory "output/download/source"
+task :tarball => ["output/download/source"] do
+  sh "cd ..; ./configure && make dist && make distclean"
+  sh "cp ../jq-*.tar.gz output/download/source"
+end
+
+task :dist => [:build, :binaries, :tarball]

docs/content/1.tutorial/default.yml

@@ -2,7 +2,7 @@ headline: Tutorial
 body:
   - text: |
 
-      Twitter have a JSON API, so let's play with that. This URL gets
+      Twitter has a JSON API, so let's play with that. This URL gets
       us the last 5 tweets about JSON:
       
   - command: "curl 'http://search.twitter.com/search.json?q=json&rpp=5&include_entities=true'"
@@ -237,5 +237,5 @@ body:
       - - -
 
       Here endeth the tutorial! There's lots more to play with, go
-      read [the manual](../manual) if you're interested, and [download
-      jq](../download) if you haven't already.
+      read [the manual](../manual/) if you're interested, and [download
+      jq](../download/) if you haven't already.

docs/content/2.download/default.yml

@@ -5,53 +5,86 @@ body:
 
       jq is written in C and has no runtime dependencies, so it should be
       possible to build it for nearly any platform. Prebuilt binaries are
-      available for Linux (64-bit x86) and OS X.
-    
-       * [Download binary for 64-bit Linux](linux_x86_64/jq)
-       * [Download binary for OS X](osx_64/jq)
-       * [Download source](source/jq.tgz)
+      available for Linux, OS X and Windows.
 
-      The binaries should just run, but you may need to make them
-      executable first using:
+      The binaries should just run, but on OS X and Linux you may need
+      to make them executable first using `chmod +x jq`.
 
-          chmod +x jq
-       
       jq is licensed under the MIT license. For all of the gory
       details, read the file `COPYING` in the source distribution.
 
-      Hacking on jq
-      =============
 
-      If you want to work on jq, grab the source from
-      [https://github.com/stedolan/jq](https://github.com/stedolan/jq).
-        
+      ### Linux
+      
+       * Binaries for [64-bit](linux64/jq) or [32-bit](linux32/jq) systems.
+
+       * For Arch users, a PKGBUILD for jq-1.1 is in the
+         [AUR](https://aur.archlinux.org/packages.php?ID=63849), as
+         well as a PKGBUILD for the HEAD of master
+         ([jq-git](https://aur.archlinux.org/packages.php?ID=63850)).
+         You can install them using
+         [Yaourt](https://wiki.archlinux.org/index.php/Yaourt) or
+         manually by following instructions on [Arch Linux's
+         Wiki](https://wiki.archlinux.org/index.php/Arch_UseRepository).
+
+       * jq 1.2 is [in Debian](http://packages.debian.org/jq)
+
+      ### OS X
+
+       * `brew install jq` using [homebrew](http://mxcl.github.com/homebrew/)
+
+       * Or, grab prebuilt [64-bit binaries](osx64/jq) or [32-bit
+         binaries](osx32/jq)
+
+      ### Windows
+
+       * Executables for [64-bit](win64/jq.exe) or [32-bit](win32/jq.exe)
+
+      ### From source on Linux or OS X
+
+       * [Source tarball for jq $JQ_VERSION](source/jq-$JQ_VERSION.tar.gz)
+
+      You can build it using the usual `./configure && make && sudo
+      make install` rigmarole.
+
+      If you're interested in using the lastest development version, try:
+
+          git clone https://github.com/stedolan/jq.git
+          cd jq
+          autoreconf
+          ./configure
+          make
+          sudo make install
 
       To build it from a git clone, you'll need to install a few
       packages first:
 
        * [Flex](http://www.gnu.org/software/flex/)
        * [Bison](http://www.gnu.org/software/bison/)
-       * [Python](http://www.python.org)
        * [GCC](http://gcc.gnu.org)
        * [Make](http://www.gnu.org/software/make)
-
-      Most of these aren't necessary if you're just trying to compile
-      jq from the released tarball - that version has the
-      non-platform-specific build steps already done, so you'll only
-      need a C compiler and `make` to finish it off.
+       * [Autotools](http://www.gnu.org/software/automake)
 
       For Linux systems, these will all be in your system's package
       manager, and if you do development on the machine they're most
-      likely already installed. I have no idea how to get these
-      installed on OS X, you're on your own there.
+      likely already installed.
+
+      On OS X, these are all included in Apple's command line tools, which
+      can be installed from [Xcode](http://developer.apple.com/technologies/tools/).
+      However, you may find that you need a newer version of Bison than the one provided
+      by Apple. This can be found in [Homebrew](http://mxcl.github.com/homebrew/)
+      or [MacPorts](http://macports.org/).
 
       `flex` and `bison` are used to generate the lexer and parser for
       jq, and some python scripts generate the UTF8 encoding tables
       needed for JSON parsing.
    
-      Building the documentation
-      --
+      #### Building the documentation
 
       jq's documentation is compiled into static HTML using
       [Bonsai](http://www.tinytree.info). To view the documentation
-      locally, run `rake serve` from the docs/ subdirectory.
+      locally, run `rake serve` from the docs/ subdirectory. You'll
+      need a few Ruby dependencies, which can be installed by
+      following the instructions in `docs/README.md`.
+
+

docs/content/3.manual/manual.yml

@@ -26,6 +26,41 @@ body: |
 
   But that's getting ahead of ourselves. :) Let's start with something
   simpler:
+
+manpage_intro: |
+  jq(1) -- Command-line JSON processor
+  ====================================
+
+  ## SYNOPSIS
+
+  `jq` [<options>...] <filter> [<files>...]
+
+  `jq` can transform JSON in various ways, by selecting, iterating,
+  reducing and otherwise mangling JSON documents. For instance,
+  running the command `jq 'map(.price) | add'` will take an array of
+  JSON objects as input and return the sum of their "price" fields.
+
+  By default, `jq` reads a stream of JSON objects (whitespace
+  separated) from `stdin`. One or more <files> may be specified, in
+  which case `jq` will read input from those instead.
+  
+  The <options> are described in the [INVOKING JQ] section, they
+  mostly concern input and output formatting. The <filter> is written
+  in the jq language and specifies how to transform the input
+  document.
+
+  ## FILTERS
+
+manpage_epilogue: |
+  ## BUGS
+
+  Presumably. Report them or discuss them at:
+
+      https://github.com/stedolan/jq/issues
+
+  ## AUTHOR
+
+  Stephen Dolan `<mu@netsoc.tcd.ie>`
   
 sections:
   - title: Invoking jq
@@ -40,46 +75,70 @@ sections:
       You can affect how jq reads and writes its input and output
       using some command-line options:
 
-      * `--slurp`/`-s` 
-
+      * `--slurp`/`-s`:
+  
         Instead of running the filter for each JSON object in the
         input, read the entire input stream into a large array and run
         the filter just once.
       
-      * `--raw-input`/`-R` 
+      * `--raw-input`/`-R`:
       
         Don't parse the input as JSON. Instead, each line of text is
         passed to the filter as a string. If combined with `--slurp`,
         then the entire input is passed to the filter as a single long
         string.
-
-      * `--null-input`/`-n`
-
+  
+      * `--null-input`/`-n`:
+  
         Don't read any input at all! Instead, the filter is run once
         using `null` as the input. This is useful when using jq as a
         simple calculator or to construct JSON data from scratch.
-
-      * `--compact-output` / `-c`
-
+  
+      * `--compact-output` / `-c`:
+  
         By default, jq pretty-prints JSON output. Using this option
         will result in more compact output by instead putting each
         JSON object on a single line.
-
-      * `--ascii-output` / `-a`
-
+  
+      * `--colour-output` / `-C` and `--monochrome-output` / `-M`:
+      
+        By default, jq outputs colored JSON if writing to a
+        terminal. You can force it to produce color even if writing to
+        a pipe or a file using `-C`, and disable color with `-M`.
+  
+      * `--ascii-output` / `-a`:
+  
         jq usually outputs non-ASCII Unicode codepoints as UTF-8, even
         if the input specified them as escape sequences (like
         "\u03bc"). Using this option, you can force jq to produce pure
         ASCII output with every non-ASCII character replaced with the
         equivalent escape sequence.
 
-      * `--raw-output` / `-r`
+      * `--sort-keys` / `-S`:
+        
+        Output the fields of each object with the keys in sorted order.
 
+      * `--raw-output` / `-r`:
+  
         With this option, if the filter's result is a string then it
         will be written directly to standard output rather than being
         formatted as a JSON string with quotes. This can be useful for
         making jq filters talk to non-JSON-based systems.
 
+      * `--arg name value`:
+
+        This option passes a value to the jq program as a predefined
+        variable. If you run jq with `--arg foo bar`, then `$foo` is
+        available in the program and has the value `"bar"`.
+
+      * `--argfile name filename`:
+
+        This option passes the first value from the named file as a
+        value to the jq program as a predefined variable. If you run jq
+        with `--argfile foo bar`, then `$foo` is available in the
+        program and has the value resulting from parsing the content of
+        the file named `bar`.
+
   - title: Basic filters
     entries:
       - title: "`.`"
@@ -101,10 +160,13 @@ sections:
       - title: "`.foo`"
         body: |
           
-          The simplest *useful* filter is .foo. When given a
+          The simplest *useful* filter is `.foo`. When given a
           JSON object (aka dictionary or hash) as input, it produces
-          the value at the key "foo", or null if there\'s none present.
+          the value at the key "foo", or null if there's none present.
 
+          If the key contains special characters, you need to surround
+          it with double quotes like this: `."foo$"`.
+          
         examples:
           - program: '.foo'
             input: '{"foo": 42, "bar": "less interesting data"}'
@@ -112,16 +174,26 @@ sections:
           - program: '.foo'
             input: '{"notfoo": true, "alsonotfoo": false}'
             output: ['null']
+          - program: '."foo"'
+            input: '{"foo": 42}'
+            output: [42]
 
-      - title: "`.[foo]`"
+      - title: "`.[foo]`, `.[2]`, `.[10:15]`"
         body: |
           
           You can also look up fields of an object using syntax like
           `.["foo"]` (.foo above is a shorthand version of this). This
           one works for arrays as well, if the key is an
-          integer. Arrays are zero-based (like javascript), so .[2]
+          integer. Arrays are zero-based (like javascript), so `.[2]`
           returns the third element of the array.
 
+          The `.[10:15]` syntax can be used to return a subarray of an
+          array. The array returned by `.[10:15]` will be of length 5,
+          containing the elements from index 10 (inclusive) to index
+          15 (exclusive). Either index may be negative (in which case
+          it counts backwards from the end of the array), or omitted
+          (in which case it refers to the start or end of the array).
+
         examples:
           - program: '.[0]'
             input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
@@ -131,18 +203,33 @@ sections:
             input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
             output: ['null']
 
+          - program: '.[2:4]'
+            input: '["a","b","c","d","e"]'
+            output: ['["c", "d"]']
+          
+          - program: '.[:3]'
+            input: '["a","b","c","d","e"]'
+            output: ['["a", "b", "c"]']
+
+          - program: '.[-2:]'
+            input: '["a","b","c","d","e"]'
+            output: ['["d", "e"]']
+
       - title: "`.[]`"
         body: |
           
           If you use the `.[foo]` syntax, but omit the index
           entirely, it will return *all* of the elements of an
           array. Running `.[]` with the input `[1,2,3]` will produce the
-          numbers as three seperate results, rather than as a single
+          numbers as three separate results, rather than as a single
           array.
 
+          You can also use this on an object, and it will return all
+          the values of the object.
+
         examples:
           - program: '.[]'
-            input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]'
+            input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
             output:
               - '{"name":"JSON", "good":true}'
               - '{"name":"XML", "good":false}'
@@ -151,6 +238,10 @@ sections:
             input: '[]'
             output: []
 
+          - program: '.[]'
+            input: '{"a": 1, "b": 1}'
+            output: ['1', '1']
+
       - title: "`,`"
         body: |
           
@@ -172,13 +263,13 @@ sections:
             
           - program: '.[4,2]'
             input: '["a","b","c","d","e"]'
-            output: ['"d"', '"c"']
+            output: ['"e"', '"c"']
           
       - title: "`|`"
         body: |
           The | operator combines two filters by feeding the output(s) of
-          the one on the left into the input of the one on the right. It\'s
-          pretty much the same as the Unix shell\'s pipe, if you\'re used to
+          the one on the left into the input of the one on the right. It's
+          pretty much the same as the Unix shell's pipe, if you're used to
           that. 
 
           If the one on the left produces multiple results, the one on
@@ -188,7 +279,7 @@ sections:
 
         examples:
           - program: '.[] | .name'
-            input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]'
+            input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
             output: ['"JSON"', '"XML"']
 
   - title: Types and Values
@@ -216,8 +307,8 @@ sections:
           in `[.foo, .bar, .baz]`) or to "collect" all the results of a
           filter into an array (as in `[.items[].name]`)
         
-          Once you understand the "," operator, you can look at jq\'s array
-          syntax in a different light: the expression [1,2,3] is not using a
+          Once you understand the "," operator, you can look at jq's array
+          syntax in a different light: the expression `[1,2,3]` is not using a
           built-in syntax for comma-separated arrays, but is instead applying
           the `[]` operator (collect results) to the expression 1,2,3 (which
           produces three different results).
@@ -238,8 +329,8 @@ sections:
           
           If the keys are "sensible" (all alphabetic characters), then
           the quotes can be left off. The value can be any expression
-          (although you may need to wrap it in parentheses if it\'s a
-          complicated one), which gets applied to the {} expression\'s
+          (although you may need to wrap it in parentheses if it's a
+          complicated one), which gets applied to the {} expression's
           input (remember, all filters have an input and an
           output).
           
@@ -253,10 +344,10 @@ sections:
           
               {user: .user, title: .title}
           
-          Because that\'s so common, there\'s a shortcut syntax: `{user, title}`.
+          Because that's so common, there's a shortcut syntax: `{user, title}`.
           
           If one of the expressions produces multiple results,
-          multiple dictionaries will be produced. If the input\'s
+          multiple dictionaries will be produced. If the input's
           
               {"user":"stedolan","titles":["JQ Primer", "More JQ"]}
           
@@ -315,14 +406,23 @@ sections:
               the key-value pairs from both objects into a single
               combined object. If both objects contain a value for the
               same key, the object on the right of the `+` wins.
+          
+          `null` can be added to any value, and returns the other
+          value unchanged.
 
         examples:
           - program: '.a + 1'
             input: '{"a": 7}'
-            output: '{"a": 8}'
+            output: ['8']
           - program: '.a + .b'
             input: '{"a": [1,2], "b": [3,4]}'
             output: ['[1,2,3,4]']
+          - program: '.a + null'
+            input: '{"a": 1}'
+            output: ['1']
+          - program: '.a + 1'
+            input: '{}'
+            output: ['1']
           - program: '{a: 1} + {b: 2} + {c: 3} + {a: 42}'
             input: 'null'
             output: ['{"a": 42, "b": 2, "c": 3}']
@@ -373,6 +473,103 @@ sections:
             input: '[[1,2], "string", {"a":2}, null]'
             output: [2, 6, 1, 0]
 
+      - title: `keys`
+        body: |
+          
+          The builtin function `keys`, when given an object, returns
+          its keys in an array.
+
+          The keys are sorted "alphabetically", by unicode codepoint
+          order. This is not an order that makes particular sense in
+          any particular language, but you can count on it being the
+          same for any two objects with the same set of keys,
+          regardless of locale settings.
+
+          When `keys` is given an array, it returns the valid indices
+          for that array: the integers from 0 to length-1.
+
+        examples:
+          - program: 'keys'
+            input: '{"abc": 1, "abcd": 2, "Foo": 3}'
+            output: ['["Foo", "abc", "abcd"]']
+          - program: 'keys'
+            input: '[42,3,35]'
+            output: ['[0,1,2]']
+
+      - title: `has`
+        body: |
+          
+          The builtin function `has` returns whether the input object
+          has the given key, or the input array has an element at the
+          given index.
+
+          `has($key)` has the same effect as checking whether `$key`
+          is a member of the array returned by `keys`, although `has`
+          will be faster.
+
+        examples:
+          - program: 'map(has("foo"))'
+            input: '[{"foo": 42}, {}]'
+            output: ['[true, false]']
+          - program: 'map(has(2))'
+            input: '[[0,1], ["a","b","c"]]'
+            output: ['[false, true]']
+
+      - title: `to_entries`, `from_entries`, `with_entries`
+        body: |
+          
+          These functions convert between an object and an array of
+          key-value pairs. If `to_entries` is passed an object, then
+          for each `k: v` entry in the input, the output array
+          includes `{"key": k, "value": v}`. 
+
+          `from_entries` does the opposite conversion, and
+          `with_entries(foo)` is a shorthand for `to_entries |
+          map(foo) | from_entries`, useful for doing some operation to
+          all keys and values of an object.
+
+        examples:
+          - program: 'to_entries'
+            input: '{"a": 1, "b": 2}'
+            output: ['[{"key":"a", "value":1}, {"key":"b", "value":2}]']
+          - program: 'from_entries'
+            input: '[{"key":"a", "value":1}, {"key":"b", "value":2}]'
+            output: ['{"a": 1, "b": 2}']
+          - program: 'with_entries(.key |= "KEY_" + .)'
+            input: '{"a": 1, "b": 2}'
+            output: ['{"KEY_a": 1, "KEY_b": 2}']
+
+
+      - title: `select`
+        body: |
+          
+          The function `select(foo)` produces its input unchanged if
+          `foo` returns true for that input, and produces no output
+          otherwise.
+
+          It's useful for filtering lists: `[1,2,3] | map(select(. >= 2))`
+          will give you `[3]`.
+
+        examples:
+          - program: 'map(select(. >= 2))'
+            input: '[1,5,3,0,7]'
+            output: ['[5,3,7]']
+
+      - title: `empty`
+        body: |
+          
+          `empty` returns no results. None at all. Not even `null`.
+
+          It's useful on occasion. You'll know if you need it :)
+
+        examples:
+          - program: '1, empty, 2'
+            input: 'null'
+            output: [1, 2]
+          - program: '[1,2,empty,3]'
+            input: 'null'
+            output: ['[1,2,3]']
+
       - title: `map(x)`
         body: |
 
@@ -402,13 +599,29 @@ sections:
         examples:
           - program: add
             input: '["a","b","c"]'
-            output: ["abc"]
+            output: ['"abc"']
           - program: add
             input: '[1, 2, 3]'
             output: [6]
           - program: add
             input: '[]'
             output: ["null"]
+
+      - title: `range`
+        body: |
+          
+          The `range` function produces a range of numbers. `range(4;10)`
+          produces 6 numbers, from 4 (inclusive) to 10 (exclusive). The numbers
+          are produced as separate outputs. Use `[range(4;10)]` to get a range as 
+          an array.
+
+        examples:
+          - program: 'range(2;4)'
+            input: 'null'
+            output: ['2', '3']
+          - program: '[range(2;4)]'
+            input: 'null'
+            output: ['[2,3]']
             
       - title: `tonumber`
         body: |
@@ -420,7 +633,7 @@ sections:
         examples:
           - program: '.[] | tonumber'
             input: '[1, "1"]'
-            output: [1,1]
+            output: [1, 1]
 
       - title: `tostring`
         body: |
@@ -434,31 +647,249 @@ sections:
             input: '[1, "1", [1]]'
             output: ['"1"', '"1"', '"[1]"']
 
-      - title: "String interpolation - `@(text)`"
+      - title: `type`
         body: |
-
-          jq supports an alternative syntax for strings. Instead of
-          "foo", you can write `@(foo)`. When using this syntax,
-          `%(expression)` may be used to insert the value of
-          `expression` into the string (converted with `tostring`).
-
-          String interpolation does not occur for normal double-quoted
-          strings (like `"foo"`) in order to be fully compatible with
-          JSON.
-
-          All of the usual JSON escapes (`\n`, `\r` and the like) work
-          inside `@()`-quoted strings, as well as `\%` and `\)` if
-          those characters are needed literally.
+          
+          The `type` function returns the type of its argument as a
+          string, which is one of null, boolean, number, string, array
+          or object.
 
         examples:
-          - program: '@(The input was %(.), which is one less than %(.+1))'
+          - program: 'map(type)'
+            input: '[0, false, [], {}, null, "hello"]'
+            output: ['["number", "boolean", "array", "object", "null", "string"]']
+
+      - title: `sort, sort_by`
+        body: |
+          
+          The `sort` functions sorts its input, which must be an
+          array. Values are sorted in the following order:
+
+          * `null`
+          * `false`
+          * `true`
+          * numbers
+          * strings, in alphabetical order (by unicode codepoint value)
+          * arrays, in lexical order
+          * objects
+
+          The ordering for objects is a little complex: first they're
+          compared by comparing their sets of keys (as arrays in
+          sorted order), and if their keys are equal then the values
+          are compared key by key.
+
+          `sort_by` may be used to sort by a particular field of an
+          object, or by applying any jq filter. `sort_by(foo)`
+          compares two elements by comparing the result of `foo` on
+          each element.
+
+        examples:
+          - program: 'sort'
+            input: '[8,3,null,6]'
+            output: ['[null,3,6,8]']
+          - program: 'sort_by(.foo)'
+            input: '[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}]'
+            output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}]']
+
+      - title: `group_by`
+        body: |
+          
+          `group_by(.foo)` takes as input an array, groups the
+          elements having the same `.foo` field into separate arrays,
+          and produces all of these arrays as elements of a larger
+          array, sorted by the value of the `.foo` field.
+
+          Any jq expression, not just a field access, may be used in
+          place of `.foo`. The sorting order is the same as described
+          in the `sort` function above.
+
+        examples:
+          - program: 'group_by(.foo)'
+            input: '[{"foo":1, "bar":10}, {"foo":3, "bar":100}, {"foo":1, "bar":1}]'
+            output: ['[[{"foo":1, "bar":10}, {"foo":1, "bar":1}], [{"foo":3, "bar":100}]]']
+
+      - title: `min`, `max`, `min_by`, `max_by`
+        body: |
+          
+          Find the minimum or maximum element of the input array. The
+          `_by` versions allow you to specify a particular field or
+          property to examine, e.g. `min_by(.foo)` finds the object
+          with the smallest `foo` field.
+
+        examples:
+          - program: 'min'
+            input: '[5,4,2,7]'
+            output: ['2']
+          - program: 'max_by(.foo)'
+            input: '[{"foo":1, "bar":14}, {"foo":2, "bar":3}]'
+            output: ['{"foo":2, "bar":3}']
+
+      - title: `unique`
+        body: |
+          
+          The `unique` function takes as input an array and produces
+          an array of the same elements, in sorted order, with
+          duplicates removed.
+
+        examples:
+          - program: 'unique'
+            input: '[1,2,5,3,5,3,1,3]'
+            output: ['[1,2,3,5]']
+
+      - title: `reverse`
+        body: |
+          
+          This function reverses an array.
+
+        examples:
+          - program: 'reverse'
+            input: '[1,2,3,4]'
+            output: ['[4,3,2,1]']
+
+      - title: `contains`
+        body: |
+
+          The filter `contains(b)` will produce true if b is
+          completely contained within the input. A string B is
+          contained in a string A if B is a substring of A. An array B
+          is contained in an array A is all elements in B are
+          contained in any element in A. An object B is contained in
+          object A if all of the values in B are contained in the
+          value in A with the same key. All other types are assumed to
+          be contained in each other if they are equal.
+
+        examples:
+          - program: 'contains("bar")'
+            input: '"foobar"'
+            output: ['true']
+          - program: 'contains(["baz", "bar"])'
+            input: '["foobar", "foobaz", "blarp"]'
+            output: ['true']
+          - program: 'contains(["bazzzzz", "bar"])'
+            input: '["foobar", "foobaz", "blarp"]'
+            output: ['false']
+          - program: 'contains({foo: 12, bar: [{barp: 12}]})'
+            input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
+            output: ['true']
+          - program: 'contains({foo: 12, bar: [{barp: 15}]})'
+            input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
+            output: ['false']
+
+      - title: `recurse`
+        body: |
+          
+          The `recurse` function allows you to search through a
+          recursive structure, and extract interesting data from all
+          levels. Suppose your input represents a filesystem:
+
+              {"name": "/", "children": [
+                {"name": "/bin", "children": [
+                  {"name": "/bin/ls", "children": []},
+                  {"name": "/bin/sh", "children": []}]},
+                {"name": "/home", "children": [
+                  {"name": "/home/stephen", "children": [
+                    {"name": "/home/stephen/jq", "children": []}]}]}]}
+          
+          Now suppose you want to extract all of the filenames
+          present. You need to retrieve `.name`, `.children[].name`,
+          `.children[].children[].name`, and so on. You can do this
+          with:
+
+              recurse(.children[]) | .name
+
+        examples:
+          - program: 'recurse(.foo[])'
+            input: '{"foo":[{"foo": []}, {"foo":[{"foo":[]}]}]}'
+            output: 
+              - '{"foo":[{"foo":[]},{"foo":[{"foo":[]}]}]}'
+              - '{"foo":[]}'
+              - '{"foo":[{"foo":[]}]}'
+              - '{"foo":[]}'
+
+
+      - title: "String interpolation - `\(foo)`"
+        body: |
+
+          Inside a string, you can put an expression inside parens
+          after a backslash. Whatever the expression returns will be
+          interpolated into the string.
+
+        examples:
+          - program: '"The input was \(.), which is one less than \(.+1)"'
             input: '42'
             output: ['"The input was 42, which is one less than 43"']
           
-              
+      - title: "Format strings and escaping"
+        body: |
 
-              
+          The `@foo` syntax is used to format and escape strings,
+          which is useful for building URLs, documents in a language
+          like HTML or XML, and so forth. `@foo` can be used as a
+          filter on its own, the possible escapings are:
 
+          * `@text`:
+
+            Calls `tostring`, see that function for details.
+
+          * `@json`:
+
+            Serialises the input as JSON.
+
+          * `@html`:
+
+            Applies HTML/XML escaping, by mapping the characters
+            `<>&'"` to their entity equivalents `&lt;`, `&gt;`,
+            `&amp;`, `&apos;`, `&quot;`.
+
+          * `@uri`:
+
+            Applies percent-encoding, by mapping all reserved URI
+            characters to a `%xx` sequence.
+
+          * `@csv`:
+            
+            The input must be an array, and it is rendered as CSV
+            with double quotes for strings, and quotes escaped by
+            repetition.
+
+          * `@sh`:
+            
+            The input is escaped suitable for use in a command-line
+            for a POSIX shell. If the input is an array, the output
+            will be a series of space-separated strings.
+            
+          * `@base64`:
+
+            The input is converted to base64 as specified by RFC 4648.
+
+          This syntax can be combined with string interpolation in a
+          useful way. You can follow a `@foo` token with a string
+          literal. The contents of the string literal will *not* be
+          escaped. However, all interpolations made inside that string
+          literal will be escaped. For instance,
+
+              @uri "http://www.google.com/search?q=\(.search)"
+
+          will produce the following output for the input
+          `{"search":"jq!"}`:
+
+              http://www.google.com/search?q=jq%21
+
+          Note that the slashes, question mark, etc. in the URL are
+          not escaped, as they were part of the string literal.
+
+        examples:
+          - program: '@html'
+            input: '"This works if x < y"'
+            output: ['"This works if x &lt; y"']
+
+#          - program: '@html "<span>Anonymous said: \(.)</span>"'
+#            input: '"<script>alert(\"lol hax\");</script>"'
+#            output: ["<span>Anonymous said: &lt;script&gt;alert(&quot;lol hax&quot;);&lt;/script&gt;</span>"]
+
+          - program: '@sh "echo \(.)"'
+            input: "\"O'Hara's Ale\""
+            output: ["\"echo 'O'\\\\''Hara'\\\\''s Ale'\""]
         
   - title: Conditionals and Comparisons
     entries:
@@ -477,8 +908,7 @@ sections:
         examples:
           - program: '.[] == 1'
             input: '[1, 1.0, "1", "banana"]'
-            output: ['[true, true, false, false]']
-          
+            output: ['true', 'true', 'false', 'false']
       - title: if-then-else
         body: |
 
@@ -491,8 +921,8 @@ sections:
           means that you'll sometimes have to be more explicit about
           the condition you want: you can't test whether, e.g. a
           string is empty using `if .name then A else B end`, you'll
-          need something more like 'if (.name | count) > 0 then A else
-          B end' instead.
+          need something more like `if (.name | count) > 0 then A else
+          B end` instead.
           
           If the condition A produces multiple results, it is
           considered "true" if any of those results is not false or
@@ -511,6 +941,21 @@ sections:
               end
             input: 2
             output: ['"many"']
+      
+      - title: `>, >=, <=, <`
+        body: |
+          
+          The comparison operators `>`, `>=`, `<=`, `<` return whether
+          their left argument is greater than, greater than or equal
+          to, less than or equal to or less than their right argument
+          (respectively).
+
+          The ordering is the same as that described for `sort`, above.
+
+        examples:
+          - program: '. < 5'
+            input: 2
+            output: ['true']
 
       - title: and/or/not
         body: |
@@ -541,9 +986,12 @@ sections:
           - program: '(true, false) or false'
             input: 'null'
             output: ['true', 'false']
-          - program: '(true, false) and (true, false)'
+#          - program: '(true, false) and (true, false)'
+#            input: 'null'
+#            output: ['true', 'false', 'false', 'false']
+          - program: '(true, true) and (true, false)'
             input: 'null'
-            output: ['true', 'false', 'false', 'false']
+            output: ['true', 'false', 'true', 'false']
           - program: '[true, false | not]'
             input: 'null'
             output: ['[false, true]']
@@ -569,7 +1017,7 @@ sections:
             input: '{}'
             output: [42]
           
-  - title: Variables and Functions
+  - title: Advanced features
     body: |
       Variables are an absolute necessity in most programming languages, but
       they're relegated to an "advanced feature" in jq.
@@ -586,6 +1034,10 @@ sections:
       (many jq functions such as `map` and `find` are in fact written
       in jq).
 
+      Finally, jq has a `reduce` operation, which is very powerful but a
+      bit tricky. Again, it's mostly used internally, to define some
+      useful bits of jq's standard library.
+
     entries:
       - title: Variables
         body: |
@@ -600,8 +1052,8 @@ sections:
           For instance, calculating the average value of an array of numbers
           requires a few variables in most languages - at least one to hold the
           array, perhaps one for each element or for a loop counter. In jq, it's
-          simply `add / length` - the `sum` expression is given the array and
-          produces its sum, and the `count` expression is given the array and
+          simply `add / length` - the `add` expression is given the array and
+          produces its sum, and the `length` expression is given the array and
           produces its length.
           
           So, there's generally a cleaner way to solve most problems in jq that
@@ -636,10 +1088,11 @@ sections:
           
               .realnames as $names | .posts[] | {title, author: $names[.author]}
           
-          The expression "foo as $x" runs foo, puts the result in $x,
-          and returns the original input. Apart from the side-effect
-          of binding the variable, it has the same effect as ".".
-          
+          The expression `exp as $x | ...` means: for each value of expression
+          `exp`, run the rest of the pipeline with the entire original input, and
+          with `$x` set to that value.  Thus `as` functions as something of a 
+          foreach loop.
+
           Variables are scoped over the rest of the expression that defines
           them, so 
           
@@ -678,18 +1131,41 @@ sections:
           If you want the value-argument behaviour for defining simple
           functions, you can just use a variable:
 
-              def addvalue(f): f as $value | map(, + $value);
+              def addvalue(f): f as $value | map(. + $value);
 
           With that definition, `addvalue(.foo)` will add the current
           input's `.foo` field to each element of the array.
 
         examples:
-          - program: 'def addvalue(f): map(. + [$value]); addvalue(.[0])'
+          - program: 'def addvalue(f): . + [f]; map(addvalue(.[0]))'
             input: '[[1,2],[10,20]]'
             output: ['[[1,2,1], [10,20,10]]']
-          - program: 'def addvalue(f): f as $x | map (. + $x); addvalue(.[0])'
+          - program: 'def addvalue(f): f as $x | map(. + $x); addvalue(.[0])'
             input: '[[1,2],[10,20]]'
-            output: ['[[1,2,[1,2]], [10,20,[1,2]]]']
+            output: ['[[1,2,1,2], [10,20,1,2]]']
+
+      - title: Reduce
+        body: |
+        
+          The `reduce` syntax in jq allows you to combine all of the
+          results of an expression by accumulating them into a single
+          answer. As an example, we'll pass `[3,2,1]` to this expression:
+
+              reduce .[] as $item (0; . + $item)
+          
+          For each result that `.[]` produces, `. + $item` is run to
+          accumulate a running total, starting from 0. In this
+          example, `.[]` produces the results 3, 2, and 1, so the
+          effect is similar to running something like this:
+
+              0 | (3 as $item | . + $item) | 
+                  (2 as $item | . + $item) |
+                  (1 as $item | . + $item)
+
+        examples:
+          - program: 'reduce .[] as $item (0; . + $item)'
+            input: '[10,2,5,3]'
+            output: ['20']
       
 
   - title: Assignment
@@ -801,4 +1277,5 @@ sections:
           "stedolan" wrote, and we can comment on each of them in the same way
           that we did before:
           
-              (.posts[] | select(.author == "stedolan") | .comments) |= . + ["terrible."]
+              (.posts[] | select(.author == "stedolan") | .comments) |=
+                  . + ["terrible."]

docs/content/index/index.yml

@@ -23,5 +23,12 @@ body3: |
   shorter and simpler than you'd expect.
   
 tail: |
-  Go read the [tutorial](tutorial) for more, or the [manual](manual)
-  for *way* more.
+  Go read the [tutorial](tutorial/) for more, or the [manual](manual/)
+  for *way* more.
+
+news:
+  - date: 2013-05-19
+    body: |
+      
+      jq 1.3 (finally) released! Get it on the [download](download/) page.
+

docs/default_manpage.md

@@ -0,0 +1,22 @@
+jq(1) -- Command-line JSON processor
+====================================
+
+## DESCRIPTION
+
+`jq` can transform JSON in various ways, by selecting, iterating,
+reducing and otherwise mangling JSON documents. 
+
+This version of `jq` was built without a manual, so this manpage is a
+stub. For full documentation of the `jq` language, see:
+
+    http://stedolan.github.com/jq
+    
+## BUGS
+
+Presumably. Report them or discuss them at:
+
+    https://github.com/stedolan/jq/issues
+
+## AUTHOR
+
+Stephen Dolan `<mu@netsoc.tcd.ie>`

docs/public/css/base.scss

@@ -89,4 +89,11 @@ h3 code {
 
 footer {
     text-align: center;
+}
+
+#news {
+    font-size: 12pt;
+    .date {
+          font-style: italic;
+    }
 }

docs/site.yml

@@ -1,10 +1,12 @@
 # The key value pairs found below are available within the templates.
 
+:url: http://stedolan.github.io/jq
+
+# This line is modified by the Makefile. To change the version number,
+# edit the Autoconf version number at the top of configure.ac
+jq_version: "1.3"
 
-jq_version: 1.1
-# Next line is needed for publishing to github pages
 root: '/jq'
-# root: ''
 footer: |
   
   This website is made with [Bonsai](http://www.tinytree.info) and
@@ -13,4 +15,4 @@ footer: |
 
   jq is licensed under the MIT license (code) and the
   [CC-BY-3.0](http://creativecommons.org/licenses/by/3.0/) license
-  (docs).
+  (docs).

docs/templates/default.liquid

@@ -10,7 +10,7 @@
         <h1>{{headline}}</h1>
         {% for item in body %}
           {% if item.text %}
-            {{ item.text | markdownify }}
+            {{ item.text | replace: '$JQ_VERSION', jq_version | markdownify }}
           {% endif %}
         
           {% if item.command %}

docs/templates/index.liquid

@@ -16,9 +16,9 @@
                 <span class="caret"></span>
               </a>
               <ul class="dropdown-menu">
-                <li><a href="{{root}}/download/linux_x86_64/jq">Linux (x86_64)</a></li>
-                <li><a href="{{root}}/download/osx_64/jq">OS X (64-bit)</a></li>
-                <li><a href="{{root}}/download">Other platforms and source</a></li>
+                <li><a href="{{root}}/download/linux64/jq">Linux (64-bit)</a></li>
+                <li><a href="{{root}}/download/osx64/jq">OS X (64-bit)</a></li>
+                <li><a href="{{root}}/download/">Other platforms and source</a></li>
               </ul>
             </div>
 
@@ -35,6 +35,16 @@
       <div class="row" style="text-align:center; margin-top: 30px">
         {{tail | markdownify}}
       </div>
+      <div class="row">
+        <h2>News</h2>
+        <div id="news">
+          <ul>
+            {% for item in news %}
+            <li><span class="date">{{ item.date | date: '%m %B %Y' }}</span>{{item.body | markdownify}}</li>
+            {% endfor %}
+          </ul>
+        </div>
+      </div>
     </div>
     {% include "shared/footer" %}    
   </body>

exec_stack.h

@@ -0,0 +1,82 @@
+#ifndef EXEC_STACK_H
+#define EXEC_STACK_H
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct determine_alignment {
+  char x;
+  union { int i; double d; uint64_t u64; size_t sz; void* ptr; } u;
+};
+enum {ALIGNMENT = offsetof(struct determine_alignment, u)};
+
+static size_t align_round_up(size_t sz) {
+  return ((sz + (ALIGNMENT - 1)) / ALIGNMENT) * ALIGNMENT;
+}
+
+typedef int stack_ptr;
+
+struct stack {
+  char* mem_end; // one-past-the-end of allocated region
+  stack_ptr bound;
+  stack_ptr limit; // 0 - stack is empty
+};
+
+static void stack_init(struct stack* s) {
+  s->mem_end = 0;
+  s->bound = ALIGNMENT;
+  s->limit = 0;
+}
+
+static void stack_reset(struct stack* s) {
+  assert(s->limit == 0 && "stack freed while not empty");
+  char* mem_start = s->mem_end - ( -s->bound + ALIGNMENT);
+  free(mem_start);
+  stack_init(s);
+}
+
+static int stack_pop_will_free(struct stack* s, stack_ptr p) {
+  return p == s->limit;
+}
+
+static void* stack_block(struct stack* s, stack_ptr p) {
+  return (void*)(s->mem_end + p);
+}
+
+static stack_ptr* stack_block_next(struct stack* s, stack_ptr p) {
+  return &((stack_ptr*)stack_block(s, p))[-1];
+}
+
+static void stack_reallocate(struct stack* s, size_t sz) {
+  int old_mem_length = -(s->bound) + ALIGNMENT;
+  char* old_mem_start = s->mem_end - old_mem_length;
+  
+  int new_mem_length = align_round_up((old_mem_length + sz + 256) * 2);
+  char* new_mem_start = realloc(old_mem_start, new_mem_length);
+  memmove(new_mem_start + (new_mem_length - old_mem_length),
+            new_mem_start, old_mem_length);
+  s->mem_end = new_mem_start + new_mem_length;
+  s->bound = -(new_mem_length - ALIGNMENT);
+}
+
+static stack_ptr stack_push_block(struct stack* s, stack_ptr p, size_t sz) {
+  int alloc_sz = align_round_up(sz) + ALIGNMENT;
+  stack_ptr r = s->limit - alloc_sz;
+  if (r < s->bound) {
+    stack_reallocate(s, alloc_sz);
+  }
+  s->limit = r;
+  *stack_block_next(s, r) = p;
+  return r;
+}
+
+static stack_ptr stack_pop_block(struct stack* s, stack_ptr p, size_t sz) {
+  stack_ptr r = *stack_block_next(s, p);
+  if (p == s->limit) {
+    int alloc_sz = align_round_up(sz) + ALIGNMENT;
+    s->limit += alloc_sz;
+  }
+  return r;
+}
+#endif

execute.h

@@ -1,13 +0,0 @@
-#ifndef EXECUTE_H
-#define EXECUTE_H
-#include "bytecode.h"
-
-
-struct bytecode* jq_compile(const char* str);
-
-void jq_init(struct bytecode* bc, jv value);
-jv jq_next();
-void jq_teardown();
-
-
-#endif

forkable_stack.h

@@ -1,162 +0,0 @@
-#ifndef FORKABLE_STACK_H
-#define FORKABLE_STACK_H
-#include <stdlib.h>
-#include <stddef.h>
-#include <assert.h>
-#include <string.h>
-
-struct forkable_stack_header {
-  /* distance in bytes between this header and the header
-     of the next object on the stack */
-  int next_delta;
-};
-
-#define FORKABLE_STACK_HEADER struct forkable_stack_header fk_header_
-
-struct forkable_stack {
-  // Stack grows down from stk+length
-
-  char* stk;
-
-  // stk+length is just past end of allocated area
-  // stk = malloc(length)
-  int length;
-
-  // stk+pos is header of top object, or stk+length if empty
-  int pos;
-
-  // everything after-or-including stk+savedlimit must be preserved
-  int savedlimit;
-};
-
-static void forkable_stack_check(struct forkable_stack* s) {
-  assert(s->stk);
-  assert(s->length > 0);
-  assert(s->pos >= 0 && s->pos <= s->length);
-  assert(s->savedlimit >= 0 && s->savedlimit <= s->length);
-}
-
-static int forkable_stack_empty(struct forkable_stack* s) {
-  return s->pos == s->length;
-}
-
-static void forkable_stack_init(struct forkable_stack* s, size_t sz) {
-  s->stk = malloc(sz);
-  s->length = sz;
-  s->pos = s->length;
-  s->savedlimit = s->length;
-  forkable_stack_check(s);
-}
-
-static void forkable_stack_free(struct forkable_stack* s) {
-  assert(forkable_stack_empty(s));
-  assert(s->savedlimit == s->length);
-  free(s->stk);
-  s->stk = 0;
-}
-
-static void* forkable_stack_push(struct forkable_stack* s, size_t sz_size) {
-  assert(sz_size > 0 && sz_size % sizeof(struct forkable_stack_header) == 0);
-  int size = (int)sz_size;
-  forkable_stack_check(s);
-  int curr = s->pos < s->savedlimit ? s->pos : s->savedlimit;
-  if (curr - size < 0) {
-    //assert(0);
-    int oldlen = s->length;
-    s->length = (size + oldlen + 1024) * 2;
-    s->stk = realloc(s->stk, s->length);
-    int shift = s->length - oldlen;
-    memmove(s->stk + shift, s->stk, oldlen);
-    s->pos += shift;
-    s->savedlimit += shift;
-    curr += shift;
-  }
-  int newpos = curr - size;
-  assert(newpos < s->pos);
-  void* ret = (void*)(s->stk + newpos);
-  ((struct forkable_stack_header*)ret)->next_delta = s->pos - newpos;
-  s->pos = newpos;
-  forkable_stack_check(s);
-  return ret;
-}
-
-static void* forkable_stack_peek(struct forkable_stack* s) {
-  assert(!forkable_stack_empty(s));
-  forkable_stack_check(s);
-  return (void*)(s->stk + s->pos);
-}
-
-static void* forkable_stack_peek_next(struct forkable_stack* s, void* top) {
-  forkable_stack_check(s);
-  struct forkable_stack_header* elem = top;
-  int elempos = (char*)top - s->stk;
-  assert(elempos >= 0 && elempos < s->length);
-  assert(elempos + elem->next_delta <= s->length);
-  if (elempos + elem->next_delta < s->length) {
-    return (void*)(s->stk + elempos + elem->next_delta);
-  } else {
-    return 0;
-  }
-}
-
-// Returns 1 if the next forkable_stack_pop will permanently remove an
-// object from the stack (i.e. the top object was not saved with a fork)
-static int forkable_stack_pop_will_free(struct forkable_stack* s) {
-  return s->pos < s->savedlimit ? 1 : 0;
-}
-
-static void forkable_stack_pop(struct forkable_stack* s) {
-  forkable_stack_check(s);
-  struct forkable_stack_header* elem = forkable_stack_peek(s);
-  s->pos += elem->next_delta;
-}
-
-
-
-struct forkable_stack_state {
-  int prevpos, prevlimit;
-};
-
-static void forkable_stack_save(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  state->prevpos = s->pos;
-  state->prevlimit = s->savedlimit;
-  if (s->pos < s->savedlimit) s->savedlimit = s->pos;
-}
-
-static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  int curr_pos = s->pos;
-  s->pos = state->prevpos;
-  state->prevpos = curr_pos;
-
-  int curr_limit = s->savedlimit;
-  if (curr_pos < curr_limit) s->savedlimit = curr_pos;
-  //state->prevlimit = curr_limit; FIXME clean up
-  forkable_stack_check(s);
-}
-
-static void forkable_stack_restore(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  assert(s->savedlimit <= state->prevpos);
-  assert(s->savedlimit <= state->prevlimit);
-  s->pos = state->prevpos;
-  s->savedlimit = state->prevlimit;
-  forkable_stack_check(s);
-}
-
-typedef int stack_idx;
-
-static stack_idx forkable_stack_to_idx(struct forkable_stack* s, void* ptr) {
-  char* item = ptr;
-  int pos = item - s->stk;
-  assert(pos >= 0 && pos < s->length);
-  return s->length - pos;
-}
-
-static void* forkable_stack_from_idx(struct forkable_stack* s, stack_idx idx) {
-  assert(idx >= 1 && idx <= s->length);
-  return &s->stk[s->length - idx];
-}
-
-#endif

frame_layout.h

@@ -1,118 +0,0 @@
-#ifndef FRAME_LAYOUT_H
-#include "forkable_stack.h"
-#include "bytecode.h"
-
-struct closure {
-  struct bytecode* bc;
-  stack_idx env;
-};
-
-struct continuation {
-  struct bytecode* bc;
-  stack_idx env;
-  uint16_t* retaddr;
-  // FIXME: probably not necessary as a separate field
-  int is_backtrack_frame;
-};
-
-typedef union frame_elem {
-  FORKABLE_STACK_HEADER;
-  struct continuation cont;
-  struct closure closure;
-  jv jsonval;
-} *frame_ptr;
-
-/*
- * Frame layout
- * fr[0] - FORKABLE_STACK_HEADER (next pointer)
- * fr[1] - self (used to store return addresses, etc)
- * fr[2...nclosures+2] - closure params
- * fr[nclosures+2..nclosures+nlocals+2] - local variables
- */
-
-static int frame_size(struct bytecode* bc) {
-  return sizeof(union frame_elem) * (bc->nclosures + bc->nlocals + 2);
-}
-
-static struct continuation* frame_self(frame_ptr fr) {
-  return &fr[1].cont;
-}
-
-static struct closure* frame_closure_arg(frame_ptr fr, int closure) {
-  assert(closure >= 0);
-  assert(closure < frame_self(fr)->bc->nclosures);
-  return &fr[2+closure].closure;
-}
-
-static jv* frame_local_var(frame_ptr fr, int var) {
-  assert(var >= 0);
-  assert(var < frame_self(fr)->bc->nlocals);
-  return &fr[2 + frame_self(fr)->bc->nclosures + var].jsonval;
-}
-
-
-static frame_ptr frame_current(struct forkable_stack* stk) {
-  frame_ptr fp = forkable_stack_peek(stk);
-  if (forkable_stack_peek_next(stk, fp)) {
-    struct bytecode* bc = frame_self(forkable_stack_peek_next(stk, fp))->bc;
-    assert(frame_self(fp)->retaddr >= bc->code && frame_self(fp)->retaddr < bc->code + bc->codelen);
-  } else {
-    assert(frame_self(fp)->retaddr == 0);
-  }
-  return fp;
-}
-
-static struct bytecode* frame_current_bytecode(struct forkable_stack* stk) {
-  return frame_self(frame_current(stk))->bc;
-}
-static uint16_t** frame_current_retaddr(struct forkable_stack* stk) {
-  return &frame_self(frame_current(stk))->retaddr;
-}
-
-static frame_ptr frame_get_parent(struct forkable_stack* stk, frame_ptr fr) {
-  return forkable_stack_from_idx(stk, frame_self(fr)->env);
-}
-
-static frame_ptr frame_get_level(struct forkable_stack* stk, frame_ptr fr, int level) {
-  for (int i=0; i<level; i++) {
-    fr = frame_get_parent(stk, fr);
-  }
-  return fr;
-}
-
-static frame_ptr frame_push(struct forkable_stack* stk, struct closure cl, uint16_t* retaddr) {
-  frame_ptr fp = forkable_stack_push(stk, frame_size(cl.bc));
-  struct continuation* cc = frame_self(fp);
-  cc->bc = cl.bc;
-  cc->env = cl.env;
-  cc->retaddr = retaddr;
-  cc->is_backtrack_frame = 0;
-  for (int i=0; i<cl.bc->nlocals; i++) {
-    *frame_local_var(fp, i) = jv_invalid();
-  }
-  return fp;
-}
-
-static frame_ptr frame_push_backtrack(struct forkable_stack* stk, uint16_t* retaddr) {
-  struct continuation cc = *frame_self(frame_current(stk));
-  frame_ptr fp = forkable_stack_push(stk, sizeof(union frame_elem) * 2);
-  assert(!cc.is_backtrack_frame);
-  cc.is_backtrack_frame = 1;
-  cc.retaddr = retaddr;
-  *frame_self(fp) = cc;
-  return fp;
-}
-
-static void frame_pop(struct forkable_stack* stk) {
-  frame_ptr fp = frame_current(stk);
-  if (forkable_stack_pop_will_free(stk) && 
-      !frame_self(fp)->is_backtrack_frame) {
-    int nlocals = frame_self(fp)->bc->nlocals;
-    for (int i=0; i<nlocals; i++) {
-      jv_free(*frame_local_var(fp, i));
-    }
-  }
-  forkable_stack_pop(stk);
-}
-
-#endif

jq.1.default

@@ -0,0 +1,39 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "JQ" "1" "May 2013" "" ""
+.
+.SH "NAME"
+\fBjq\fR \- Command\-line JSON processor
+.
+.SH "DESCRIPTION"
+\fBjq\fR can transform JSON in various ways, by selecting, iterating, reducing and otherwise mangling JSON documents\.
+.
+.P
+This version of \fBjq\fR was built without a manual, so this manpage is a stub\. For full documentation of the \fBjq\fR language, see:
+.
+.IP "" 4
+.
+.nf
+
+http://stedolan\.github\.com/jq
+.
+.fi
+.
+.IP "" 0
+.
+.SH "BUGS"
+Presumably\. Report them or discuss them at:
+.
+.IP "" 4
+.
+.nf
+
+https://github\.com/stedolan/jq/issues
+.
+.fi
+.
+.IP "" 0
+.
+.SH "AUTHOR"
+Stephen Dolan \fB<mu@netsoc\.tcd\.ie>\fR

jq.h

@@ -0,0 +1,18 @@
+#ifndef _JQ_H_
+#define _JQ_H_
+
+#include <jv.h>
+
+enum {JQ_DEBUG_TRACE = 1};
+
+typedef struct jq_state jq_state;
+jq_state *jq_init(void);
+void jq_set_nomem_handler(jq_state *, void (*)(void *), void *);
+int jq_compile(jq_state *, const char* str);
+int jq_compile_args(jq_state *, const char* str, jv args);
+void jq_dump_disassembly(jq_state *, int);
+void jq_start(jq_state *, jv value, int flags);
+jv jq_next(jq_state *);
+void jq_teardown(jq_state **);
+
+#endif /* !_JQ_H_ */

jq.spec

@@ -0,0 +1,65 @@
+# This is spec file maintained by developers of JQ, not by a OS distro.
+# Your OS of choice will likely ignore this RPM spec file.
+Summary: Command-line JSON processor
+Name: jq
+Version: %{myver}
+Release: %{myrel}%{?dist}
+Source0: jq-%{myver}.tar.gz
+URL: https://github.com/stedolan/jq
+License: BSD
+AutoReqProv: no
+#BuildPrereq: autoconf, libtool, automake, flex, bison, python
+
+Group: Applications/System
+# Requires:
+
+# Disables debug packages and stripping of binaries:
+%global _enable_debug_package 0
+%global debug_package %{nil}
+%global __os_install_post %{nil}
+
+# Crank up the compression
+%define _binary_payload w7.lzdio
+
+%description
+jq is a command-line JSON processor
+
+%prep
+%setup
+
+%build
+echo "Building in: \"$(pwd)\""
+%if "%{devbuild}" == "yes"
+./configure --prefix=%{_prefix} --enable-devel
+%else
+./configure --prefix=%{_prefix}
+%endif
+make
+
+%install
+echo "Installing to: \"%{buildroot}\""
+make install DESTDIR=%{buildroot}
+
+%clean
+rm -rf %{buildroot}
+
+%files
+%defattr(-,root,root)
+%{_bindir}/jq
+%if "%{devbuild}" == "yes"
+%{_libexecdir}/%{name}/jq_test
+%{_libexecdir}/%{name}/testdata
+%endif
+%{_datadir}/doc/%{name}/AUTHORS
+%{_datadir}/doc/%{name}/COPYING
+%{_datadir}/doc/%{name}/INSTALL
+%{_datadir}/doc/%{name}/NEWS
+%{_datadir}/doc/%{name}/README
+%{_datadir}/doc/%{name}/README.md
+%{_datadir}/man/man1/jq.1
+
+%changelog
+
+%pre
+
+%post

jq_parser.h

@@ -1,5 +1,7 @@
-#ifndef PARSER_H
-#define PARSER_H
+#ifndef JQ_PARSER_H
+#define JQ_PARSER_H
+#include "locfile.h"
+#include "compile.h"
 
 int jq_parse(struct locfile* source, block* answer);
 int jq_parse_library(struct locfile* locations, block* answer);

jq_test.c

@@ -3,19 +3,26 @@
 #include <string.h>
 #include <stdlib.h>
 #include "jv.h"
-#include "execute.h"
+#include "jq.h"
 
 static void jv_test();
 static void run_jq_tests();
 
-int main() {
+
+int jq_testsuite(int argc, char* argv[]) {
+  FILE *testdata = stdin;
   jv_test();
-  run_jq_tests();
+  if (argc > 2) {
+    testdata = fopen(argv[2], "r");
+    if (!testdata) {
+      perror("fopen");
+      exit(1);
+    }
+  }
+  run_jq_tests(testdata);
+  return 0;
 }
 
-
-
-
 static int skipline(const char* buf) {
   int p = 0;
   while (buf[p] == ' ' || buf[p] == '\t') p++;
@@ -23,46 +30,51 @@ static int skipline(const char* buf) {
   return 0;
 }
 
-static void run_jq_tests() {
-  FILE* testdata = fopen("testdata","r");
+static void run_jq_tests(FILE *testdata) {
   char buf[4096];
-  int tests = 0, passed = 0;
+  int tests = 0, passed = 0, invalid = 0;
+  jq_state *jq = NULL;
+
+  jq = jq_init();
+  assert(jq);
 
   while (1) {
     if (!fgets(buf, sizeof(buf), testdata)) break;
     if (skipline(buf)) continue;
+    if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0;
     printf("Testing %s\n", buf);
     int pass = 1;
-    struct bytecode* bc = jq_compile(buf);
-    assert(bc);
+    tests++;
+    int compiled = jq_compile(jq, buf);
+    if (!compiled) {invalid++; continue;}
     printf("Disassembly:\n");
-    dump_disassembly(2, bc);
+    jq_dump_disassembly(jq, 2);
     printf("\n");
-    fgets(buf, sizeof(buf), testdata);
+    if (!fgets(buf, sizeof(buf), testdata)) { invalid++; break; }
     jv input = jv_parse(buf);
-    assert(jv_is_valid(input));
-    jq_init(bc, input);
+    if (!jv_is_valid(input)){ invalid++; continue; }
+    jq_start(jq, input, JQ_DEBUG_TRACE);
 
     while (fgets(buf, sizeof(buf), testdata)) {
       if (skipline(buf)) break;
       jv expected = jv_parse(buf);
-      assert(jv_is_valid(expected));
-      jv actual = jq_next();
+      if (!jv_is_valid(expected)){ invalid++; continue; }
+      jv actual = jq_next(jq);
       if (!jv_is_valid(actual)) {
         jv_free(actual);
-        printf("Insufficient results\n");
+        printf("*** Insufficient results\n");
         pass = 0;
         break;
       } else if (!jv_equal(jv_copy(expected), jv_copy(actual))) {
-        printf("Expected ");
+        printf("*** Expected ");
         jv_dump(jv_copy(expected), 0);
         printf(", but got ");
         jv_dump(jv_copy(actual), 0);
         printf("\n");
         pass = 0;
       }
-      jv as_string = jv_dump_string(jv_copy(expected), rand());
-      jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length(jv_copy(as_string)));
+      jv as_string = jv_dump_string(jv_copy(expected), rand() & ~JV_PRINT_COLOUR);
+      jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length_bytes(jv_copy(as_string)));
       assert(jv_equal(jv_copy(expected), jv_copy(reparsed)));
       jv_free(as_string);
       jv_free(reparsed);
@@ -70,9 +82,9 @@ static void run_jq_tests() {
       jv_free(actual);
     }
     if (pass) {
-      jv extra = jq_next();
+      jv extra = jq_next(jq);
       if (jv_is_valid(extra)) {
-        printf("Superfluous result: ");
+        printf("*** Superfluous result: ");
         jv_dump(extra, 0);
         printf("\n");
         pass = 0;
@@ -80,13 +92,10 @@ static void run_jq_tests() {
         jv_free(extra);
       }
     }
-    jq_teardown();
-    bytecode_free(bc);
-    tests++;
     passed+=pass;
   }
-  fclose(testdata);
-  printf("%d of %d tests passed\n", passed,tests);
+  jq_teardown(&jq);
+  printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
   if (passed != tests) exit(1);
 }
 
@@ -116,9 +125,9 @@ static void jv_test() {
     jv_free(a2);
 
 
-    assert(a.val.complex.ptr->count == 1);
+    assert(jv_get_refcnt(a) == 1);
     a = jv_array_append(a, jv_copy(a));
-    assert(a.val.complex.ptr->count == 1);
+    assert(jv_get_refcnt(a) == 1);
 
     assert(jv_array_length(jv_copy(a)) == 2);
     assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
@@ -151,9 +160,9 @@ static void jv_test() {
 
     jv_free(subarray);
 
-    void* before = sub2.val.complex.ptr;
+    void* before = sub2.val.nontrivial.ptr;
     sub2 = jv_array_append(sub2, jv_number(200));
-    void* after = sub2.val.complex.ptr;
+    void* after = sub2.val.nontrivial.ptr;
     assert(before == after);
     jv_free(sub2);
 
@@ -165,6 +174,18 @@ static void jv_test() {
     jv_free(a3);
 
 
+    jv a4 = jv_array();
+    a4 = jv_array_append(a4, jv_number(1));
+    a4 = jv_array_append(a4, jv_number(2));
+    jv a5 = jv_copy(a4);
+    a4 = jv_array_append(a4, jv_number(3));
+    a4 = jv_array_slice(a4, 0, 1);
+    assert(jv_array_length(jv_copy(a4)) == 1);
+    a4 = jv_array_append(a4, jv_number(4));
+    assert(jv_array_length(a4) == 2);
+    assert(jv_array_length(a5) == 2);
+
+
     assert(jv_array_length(jv_copy(a)) == 2);
     assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
     assert(jv_array_length(jv_array_get(jv_copy(a), 1)) == 1);
@@ -180,8 +201,8 @@ static void jv_test() {
     assert(jv_equal(jv_string("foo"), jv_string_sized("foo", 3)));
     char nasty[] = "foo\0";
     jv shortstr = jv_string(nasty), longstr = jv_string_sized(nasty, sizeof(nasty));
-    assert(jv_string_length(shortstr) == (int)strlen(nasty));
-    assert(jv_string_length(longstr) == (int)sizeof(nasty));
+    assert(jv_string_length_bytes(shortstr) == (int)strlen(nasty));
+    assert(jv_string_length_bytes(longstr) == (int)sizeof(nasty));
 
   
     char a1s[] = "hello", a2s[] = "hello", bs[] = "goodbye";
@@ -202,7 +223,7 @@ static void jv_test() {
     for (int i=0; i<(int)sizeof(big); i++) big[i] = 'a';
     big[sizeof(big)-1] = 0;
     jv str = jv_string_fmt("%s", big);
-    assert(jv_string_length(jv_copy(str)) == sizeof(big) - 1);
+    assert(jv_string_length_bytes(jv_copy(str)) == sizeof(big) - 1);
     assert(!strcmp(big, jv_string_value(str)));
     jv_free(str);
   }

jv.c

@@ -6,26 +6,33 @@
 #include <string.h>
 #include <stdarg.h>
 
+#include "jv_alloc.h"
 #include "jv.h"
+#include "jv_unicode.h"
 
 /*
  * Internal refcounting helpers
  */
 
-static void jvp_refcnt_init(jv_complex* c) {
+typedef struct jv_refcnt {
+  size_t count;
+} jv_refcnt;
+
+
+static void jvp_refcnt_init(jv_nontrivial* c) {
   c->ptr->count = 1;
 }
 
-static void jvp_refcnt_inc(jv_complex* c) {
+static void jvp_refcnt_inc(jv_nontrivial* c) {
   c->ptr->count++;
 }
 
-static int jvp_refcnt_dec(jv_complex* c) {
+static int jvp_refcnt_dec(jv_nontrivial* c) {
   c->ptr->count--;
   return c->ptr->count == 0;
 }
 
-static int jvp_refcnt_unshared(jv_complex* c) {
+static int jvp_refcnt_unshared(jv_nontrivial* c) {
   assert(c->ptr->count > 0);
   return c->ptr->count == 1;
 }
@@ -85,9 +92,9 @@ typedef struct {
 jv jv_invalid_with_msg(jv err) {
   jv x;
   x.kind = JV_KIND_INVALID;
-  x.val.complex.i[0] = x.val.complex.i[1] = 0;
-  jvp_invalid* i = malloc(sizeof(jvp_invalid));
-  x.val.complex.ptr = &i->refcnt;
+  x.val.nontrivial.i[0] = x.val.nontrivial.i[1] = 0;
+  jvp_invalid* i = jv_mem_alloc(sizeof(jvp_invalid));
+  x.val.nontrivial.ptr = &i->refcnt;
   i->refcnt.count = 1;
   i->errmsg = err;
   return x;
@@ -98,7 +105,7 @@ jv jv_invalid() {
 }
 
 jv jv_invalid_get_msg(jv inv) {
-  jv x = jv_copy(((jvp_invalid*)inv.val.complex.ptr)->errmsg);
+  jv x = jv_copy(((jvp_invalid*)inv.val.nontrivial.ptr)->errmsg);
   jv_free(inv);
   return x;
 }
@@ -110,10 +117,10 @@ int jv_invalid_has_msg(jv inv) {
   return r;
 }
 
-static void jvp_invalid_free(jv_complex* x) {
+static void jvp_invalid_free(jv_nontrivial* x) {
   if (jvp_refcnt_dec(x)) {
     jv_free(((jvp_invalid*)x->ptr)->errmsg);
-    free(x->ptr);
+    jv_mem_free(x->ptr);
   }
 }
 
@@ -152,38 +159,38 @@ typedef struct {
   jv elements[];
 } jvp_array;
 
-static jvp_array* jvp_array_ptr(jv_complex* a) {
+static jvp_array* jvp_array_ptr(jv_nontrivial* a) {
   return (jvp_array*)a->ptr;
 }
 
 static jvp_array* jvp_array_alloc(unsigned size) {
-  jvp_array* a = malloc(sizeof(jvp_array) + sizeof(jv) * size);
+  jvp_array* a = jv_mem_alloc(sizeof(jvp_array) + sizeof(jv) * size);
   a->refcnt.count = 1;
   a->length = 0;
   a->alloc_length = size;
   return a;
 }
 
-static jv_complex jvp_array_new(unsigned size) {
-  jv_complex r = {&jvp_array_alloc(size)->refcnt, {0, 0}};
+static jv_nontrivial jvp_array_new(unsigned size) {
+  jv_nontrivial r = {&jvp_array_alloc(size)->refcnt, {0, 0}};
   return r;
 }
 
-static void jvp_array_free(jv_complex* a) {
+static void jvp_array_free(jv_nontrivial* a) {
   if (jvp_refcnt_dec(a)) {
     jvp_array* array = jvp_array_ptr(a);
     for (int i=0; i<array->length; i++) {
       jv_free(array->elements[i]);
     }
-    free(array);
+    jv_mem_free(array);
   }
 }
 
-static int jvp_array_length(jv_complex* a) {
+static int jvp_array_length(jv_nontrivial* a) {
   return a->i[1] - a->i[0];
 }
 
-static jv* jvp_array_read(jv_complex* a, int i) {
+static jv* jvp_array_read(jv_nontrivial* a, int i) {
   if (i >= 0 && i < jvp_array_length(a)) {
     jvp_array* array = jvp_array_ptr(a);
     assert(i + a->i[0] < array->length);
@@ -193,7 +200,7 @@ static jv* jvp_array_read(jv_complex* a, int i) {
   }
 }
 
-static jv* jvp_array_write(jv_complex* a, int i) {
+static jv* jvp_array_write(jv_nontrivial* a, int i) {
   assert(i >= 0);
   jvp_array* array = jvp_array_ptr(a);
 
@@ -213,7 +220,7 @@ static jv* jvp_array_write(jv_complex* a, int i) {
         array->elements[j] = JV_NULL;
       }
       array->length = imax(pos + 1, array->length);
-      a->i[1] = imax(pos + 1, array->length);
+      a->i[1] = imax(pos + 1, a->i[1]);
       return &array->elements[pos];
     }
   }
@@ -236,7 +243,7 @@ static jv* jvp_array_write(jv_complex* a, int i) {
   return &new_array->elements[i];
 }
 
-static int jvp_array_equal(jv_complex* a, jv_complex* b) {
+static int jvp_array_equal(jv_nontrivial* a, jv_nontrivial* b) {
   if (jvp_array_length(a) != jvp_array_length(b)) 
     return 0;
   if (jvp_array_ptr(a) == jvp_array_ptr(b) &&
@@ -250,12 +257,11 @@ static int jvp_array_equal(jv_complex* a, jv_complex* b) {
   return 1;
 }
 
-static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
+static jv_nontrivial jvp_array_slice(jv_nontrivial* a, int start, int end) {
   // FIXME: maybe slice should reallocate if the slice is small enough
   assert(start <= end);
-  jvp_array* array = jvp_array_ptr(a);
-  assert(a->i[1] + end < array->length);
-  jv_complex slice = *a;
+  assert(a->i[0] + end <= a->i[1]);
+  jv_nontrivial slice = *a;
   slice.i[0] += start;
   slice.i[1] = slice.i[0] + (end - start);
   return slice;
@@ -268,7 +274,7 @@ static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
 jv jv_array_sized(int n) {
   jv j;
   j.kind = JV_KIND_ARRAY;
-  j.val.complex = jvp_array_new(n);
+  j.val.nontrivial = jvp_array_new(n);
   return j;
 }
 
@@ -278,14 +284,14 @@ jv jv_array() {
 
 int jv_array_length(jv j) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
-  int len = jvp_array_length(&j.val.complex);
+  int len = jvp_array_length(&j.val.nontrivial);
   jv_free(j);
   return len;
 }
 
 jv jv_array_get(jv j, int idx) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
-  jv* slot = jvp_array_read(&j.val.complex, idx);
+  jv* slot = jvp_array_read(&j.val.nontrivial, idx);
   jv val;
   if (slot) {
     val = jv_copy(*slot);
@@ -299,7 +305,7 @@ jv jv_array_get(jv j, int idx) {
 jv jv_array_set(jv j, int idx, jv val) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
   // copy/free of val,j coalesced
-  jv* slot = jvp_array_write(&j.val.complex, idx);
+  jv* slot = jvp_array_write(&j.val.nontrivial, idx);
   jv_free(*slot);
   *slot = val;
   return j;
@@ -315,8 +321,8 @@ jv jv_array_concat(jv a, jv b) {
   assert(jv_get_kind(b) == JV_KIND_ARRAY);
 
   // FIXME: could be much faster
-  for (int i=0; i<jv_array_length(jv_copy(b)); i++) {
-    a = jv_array_append(a, jv_array_get(jv_copy(b), i));
+  jv_array_foreach(b, i, elem) {
+    a = jv_array_append(a, elem);
   }
   jv_free(b);
   return a;
@@ -325,10 +331,31 @@ jv jv_array_concat(jv a, jv b) {
 jv jv_array_slice(jv a, int start, int end) {
   assert(jv_get_kind(a) == JV_KIND_ARRAY);
   // copy/free of a coalesced
-  a.val.complex = jvp_array_slice(&a.val.complex, start, end);
+  a.val.nontrivial = jvp_array_slice(&a.val.nontrivial, start, end);
   return a;
 }
 
+int jv_array_contains(jv a, jv b) {
+  int r = 1;
+  jv_array_foreach(b, bi, belem) {
+    int ri = 0;
+    jv_array_foreach(a, ai, aelem) {
+      if (jv_contains(aelem, jv_copy(belem))) {
+        ri = 1;
+        break;
+      }
+    }
+    jv_free(belem);
+    if (!ri) {
+      r = 0;
+      break;
+    }
+  }
+  jv_free(a);
+  jv_free(b);
+  return r;
+}
+
 
 /*
  * Strings (internal helpers)
@@ -344,40 +371,40 @@ typedef struct {
   char data[];
 } jvp_string;
 
-static jvp_string* jvp_string_ptr(jv_complex* a) {
+static jvp_string* jvp_string_ptr(jv_nontrivial* a) {
   return (jvp_string*)a->ptr;
 }
 
 static jvp_string* jvp_string_alloc(uint32_t size) {
-  jvp_string* s = malloc(sizeof(jvp_string) + size + 1);
+  jvp_string* s = jv_mem_alloc(sizeof(jvp_string) + size + 1);
   s->refcnt.count = 1;
   s->alloc_length = size;
   return s;
 }
 
-static jv_complex jvp_string_new(const char* data, uint32_t length) {
+static jv_nontrivial jvp_string_new(const char* data, uint32_t length) {
   jvp_string* s = jvp_string_alloc(length);
   s->length_hashed = length << 1;
   memcpy(s->data, data, length);
   s->data[length] = 0;
-  jv_complex r = {&s->refcnt, {0,0}};
+  jv_nontrivial r = {&s->refcnt, {0,0}};
   return r;
 }
 
-static void jvp_string_free(jv_complex* s) {
+static void jvp_string_free(jv_nontrivial* s) {
   if (jvp_refcnt_dec(s)) {
     jvp_string* str = jvp_string_ptr(s);
-    free(str);
+    jv_mem_free(str);
   }
 }
 
 static void jvp_string_free_p(jvp_string* s) {
-  jv_complex p = {&s->refcnt,{0,0}};
+  jv_nontrivial p = {&s->refcnt,{0,0}};
   jvp_string_free(&p);
 }
 
 static jvp_string* jvp_string_copy_p(jvp_string* s) {
-  jv_complex p = {&s->refcnt,{0,0}};
+  jv_nontrivial p = {&s->refcnt,{0,0}};
   jvp_refcnt_inc(&p);
   return s;
 }
@@ -392,7 +419,7 @@ static uint32_t jvp_string_remaining_space(jvp_string* s) {
   return r;
 }
 
-static void jvp_string_append(jv_complex* string, const char* data, uint32_t len) {
+static void jvp_string_append(jv_nontrivial* string, const char* data, uint32_t len) {
   jvp_string* s = jvp_string_ptr(string);
   uint32_t currlen = jvp_string_length(s);
     
@@ -412,7 +439,7 @@ static void jvp_string_append(jv_complex* string, const char* data, uint32_t len
     memcpy(news->data + currlen, data, len);
     news->data[currlen + len] = 0;
     jvp_string_free(string);
-    jv_complex r = {&news->refcnt, {0,0}};
+    jv_nontrivial r = {&news->refcnt, {0,0}};
     *string = r;
   }
 }
@@ -487,7 +514,7 @@ static int jvp_string_equal_hashed(jvp_string* a, jvp_string* b) {
   return memcmp(a->data, b->data, jvp_string_length(a)) == 0;
 }
 
-static int jvp_string_equal(jv_complex* a, jv_complex* b) {
+static int jvp_string_equal(jv_nontrivial* a, jv_nontrivial* b) {
   jvp_string* stra = jvp_string_ptr(a);
   jvp_string* strb = jvp_string_ptr(b);
   if (jvp_string_length(stra) != jvp_string_length(strb)) return 0;
@@ -501,7 +528,7 @@ static int jvp_string_equal(jv_complex* a, jv_complex* b) {
 jv jv_string_sized(const char* str, int len) {
   jv j;
   j.kind = JV_KIND_STRING;
-  j.val.complex = jvp_string_new(str, len);
+  j.val.nontrivial = jvp_string_new(str, len);
   return j;
 }
 
@@ -509,34 +536,44 @@ jv jv_string(const char* str) {
   return jv_string_sized(str, strlen(str));
 }
 
-int jv_string_length(jv j) {
+int jv_string_length_bytes(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  int r = jvp_string_length(jvp_string_ptr(&j.val.complex));
+  int r = jvp_string_length(jvp_string_ptr(&j.val.nontrivial));
   jv_free(j);
   return r;
 }
 
-uint32_t jv_string_hash(jv j) {
+int jv_string_length_codepoints(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.complex));
+  const char* i = jv_string_value(j);
+  const char* end = i + jv_string_length_bytes(jv_copy(j));
+  int c = 0, len = 0;
+  while ((i = jvp_utf8_next(i, end, &c))) len++;
+  jv_free(j);
+  return len;
+}
+
+unsigned long jv_string_hash(jv j) {
+  assert(jv_get_kind(j) == JV_KIND_STRING);
+  uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.nontrivial));
   jv_free(j);
   return hash;
 }
 
 const char* jv_string_value(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  return jvp_string_ptr(&j.val.complex)->data;
+  return jvp_string_ptr(&j.val.nontrivial)->data;
 }
 
 jv jv_string_concat(jv a, jv b) {
-  jvp_string* sb = jvp_string_ptr(&b.val.complex);
-  jvp_string_append(&a.val.complex, sb->data, jvp_string_length(sb));
+  jvp_string* sb = jvp_string_ptr(&b.val.nontrivial);
+  jvp_string_append(&a.val.nontrivial, sb->data, jvp_string_length(sb));
   jv_free(b);
   return a;
 }
 
 jv jv_string_append_buf(jv a, const char* buf, int len) {
-  jvp_string_append(&a.val.complex, buf, len);
+  jvp_string_append(&a.val.nontrivial, buf, len);
   return a;
 }
 
@@ -547,17 +584,17 @@ jv jv_string_append_str(jv a, const char* str) {
 jv jv_string_fmt(const char* fmt, ...) {
   int size = 1024;
   while (1) {
-    char* buf = malloc(size);
+    char* buf = jv_mem_alloc(size);
     va_list args;
     va_start(args, fmt);
     int n = vsnprintf(buf, size, fmt, args);
     va_end(args);
     if (n < size) {
       jv ret = jv_string_sized(buf, n);
-      free(buf);
+      jv_mem_free(buf);
       return ret;
     } else {
-      free(buf);
+      jv_mem_free(buf);
       size = n * 2;
     }
   }
@@ -581,15 +618,15 @@ typedef struct {
 } jvp_object;
 
 
-/* warning: complex justification of alignment */
-static jv_complex jvp_object_new(int size) {
+/* warning: nontrivial justification of alignment */
+static jv_nontrivial jvp_object_new(int size) {
   // Allocates an object of (size) slots and (size*2) hash buckets.
 
   // size must be a power of two
   assert(size > 0 && (size & (size - 1)) == 0);
-  jvp_object* obj = malloc(sizeof(jvp_object) + 
-                           sizeof(struct object_slot) * size +
-                           sizeof(int) * (size * 2));
+  jvp_object* obj = jv_mem_alloc(sizeof(jvp_object) + 
+                                 sizeof(struct object_slot) * size +
+                                 sizeof(int) * (size * 2));
   obj->refcnt.count = 1;
   for (int i=0; i<size; i++) {
     obj->elements[i].next = i - 1;
@@ -599,7 +636,7 @@ static jv_complex jvp_object_new(int size) {
   }
   obj->first_free = size - 1;
   int* hashbuckets = (int*)(&obj->elements[size]);
-  jv_complex r = {&obj->refcnt, 
+  jv_nontrivial r = {&obj->refcnt, 
                   {size*2 - 1, (char*)hashbuckets - (char*)obj}};
   for (int i=0; i<size*2; i++) {
     hashbuckets[i] = -1;
@@ -607,39 +644,39 @@ static jv_complex jvp_object_new(int size) {
   return r;
 }
 
-static jvp_object* jvp_object_ptr(jv_complex* o) {
+static jvp_object* jvp_object_ptr(jv_nontrivial* o) {
   return (jvp_object*)o->ptr;
 }
 
-static uint32_t jvp_object_mask(jv_complex* o) {
+static uint32_t jvp_object_mask(jv_nontrivial* o) {
   return o->i[0];
 }
 
-static int jvp_object_size(jv_complex* o) {
+static int jvp_object_size(jv_nontrivial* o) {
   return (o->i[0] + 1) >> 1;
 }
 
-static int* jvp_object_buckets(jv_complex* o) {
+static int* jvp_object_buckets(jv_nontrivial* o) {
   int* buckets = (int*)((char*)o->ptr + o->i[1]);
   assert(buckets == (int*)&jvp_object_ptr(o)->elements[jvp_object_size(o)]);
   return buckets;
 }
 
-static int* jvp_object_find_bucket(jv_complex* object, jvp_string* key) {
+static int* jvp_object_find_bucket(jv_nontrivial* object, jvp_string* key) {
   return jvp_object_buckets(object) + (jvp_object_mask(object) & jvp_string_hash(key));
 }
 
-static struct object_slot* jvp_object_get_slot(jv_complex* object, int slot) {
+static struct object_slot* jvp_object_get_slot(jv_nontrivial* object, int slot) {
   assert(slot == -1 || (slot >= 0 && slot < jvp_object_size(object)));
   if (slot == -1) return 0;
   else return &jvp_object_ptr(object)->elements[slot];
 }
 
-static struct object_slot* jvp_object_next_slot(jv_complex* object, struct object_slot* slot) {
+static struct object_slot* jvp_object_next_slot(jv_nontrivial* object, struct object_slot* slot) {
   return jvp_object_get_slot(object, slot->next);
 }
 
-static struct object_slot* jvp_object_find_slot(jv_complex* object, jvp_string* keystr, int* bucket) {
+static struct object_slot* jvp_object_find_slot(jv_nontrivial* object, jvp_string* keystr, int* bucket) {
   for (struct object_slot* curr = jvp_object_get_slot(object, *bucket); 
        curr; 
        curr = jvp_object_next_slot(object, curr)) {
@@ -650,7 +687,7 @@ static struct object_slot* jvp_object_find_slot(jv_complex* object, jvp_string*
   return 0;
 }
 
-static struct object_slot* jvp_object_add_slot(jv_complex* object, jvp_string* key, int* bucket) {
+static struct object_slot* jvp_object_add_slot(jv_nontrivial* object, jvp_string* key, int* bucket) {
   jvp_object* o = jvp_object_ptr(object);
   int newslot_idx = o->first_free;
   struct object_slot* newslot = jvp_object_get_slot(object, newslot_idx);
@@ -663,7 +700,7 @@ static struct object_slot* jvp_object_add_slot(jv_complex* object, jvp_string* k
   return newslot;
 }
 
-static void jvp_object_free_slot(jv_complex* object, struct object_slot* slot) {
+static void jvp_object_free_slot(jv_nontrivial* object, struct object_slot* slot) {
   jvp_object* o = jvp_object_ptr(object);
   slot->next = o->first_free;
   assert(slot->string);
@@ -673,14 +710,14 @@ static void jvp_object_free_slot(jv_complex* object, struct object_slot* slot) {
   o->first_free = slot - jvp_object_get_slot(object, 0);
 }
 
-static jv* jvp_object_read(jv_complex* object, jvp_string* key) {
+static jv* jvp_object_read(jv_nontrivial* object, jvp_string* key) {
   int* bucket = jvp_object_find_bucket(object, key);
   struct object_slot* slot = jvp_object_find_slot(object, key, bucket);
   if (slot == 0) return 0;
   else return &slot->value;
 }
 
-static void jvp_object_free(jv_complex* o) {
+static void jvp_object_free(jv_nontrivial* o) {
   if (jvp_refcnt_dec(o)) {
     for (int i=0; i<jvp_object_size(o); i++) {
       struct object_slot* slot = jvp_object_get_slot(o, i);
@@ -689,14 +726,14 @@ static void jvp_object_free(jv_complex* o) {
         jv_free(slot->value);
       }
     }
-    free(jvp_object_ptr(o));
+    jv_mem_free(jvp_object_ptr(o));
   }
 }
 
-static void jvp_object_rehash(jv_complex* object) {
+static void jvp_object_rehash(jv_nontrivial* object) {
   assert(jvp_refcnt_unshared(object));
   int size = jvp_object_size(object);
-  jv_complex new_object = jvp_object_new(size * 2);
+  jv_nontrivial new_object = jvp_object_new(size * 2);
   for (int i=0; i<size; i++) {
     struct object_slot* slot = jvp_object_get_slot(object, i);
     if (!slot->string) continue;
@@ -708,15 +745,15 @@ static void jvp_object_rehash(jv_complex* object) {
     new_slot->value = slot->value;
   }
   // references are transported, just drop the old table
-  free(jvp_object_ptr(object));
+  jv_mem_free(jvp_object_ptr(object));
   *object = new_object;
 }
 
-static void jvp_object_unshare(jv_complex* object) {
+static void jvp_object_unshare(jv_nontrivial* object) {
   if (jvp_refcnt_unshared(object))
     return;
 
-  jv_complex new_object = jvp_object_new(jvp_object_size(object));
+  jv_nontrivial new_object = jvp_object_new(jvp_object_size(object));
   jvp_object_ptr(&new_object)->first_free = jvp_object_ptr(object)->first_free;
   for (int i=0; i<jvp_object_size(&new_object); i++) {
     struct object_slot* old_slot = jvp_object_get_slot(object, i);
@@ -737,7 +774,7 @@ static void jvp_object_unshare(jv_complex* object) {
   assert(jvp_refcnt_unshared(object));
 }
 
-static jv* jvp_object_write(jv_complex* object, jvp_string* key) {
+static jv* jvp_object_write(jv_nontrivial* object, jvp_string* key) {
   jvp_object_unshare(object);
   int* bucket = jvp_object_find_bucket(object, key);
   struct object_slot* slot = jvp_object_find_slot(object, key, bucket);
@@ -760,7 +797,7 @@ static jv* jvp_object_write(jv_complex* object, jvp_string* key) {
   return &slot->value;
 }
 
-static int jvp_object_delete(jv_complex* object, jvp_string* key) {
+static int jvp_object_delete(jv_nontrivial* object, jvp_string* key) {
   jvp_object_unshare(object);
   int* bucket = jvp_object_find_bucket(object, key);
   int* prev_ptr = bucket;
@@ -777,7 +814,7 @@ static int jvp_object_delete(jv_complex* object, jvp_string* key) {
   return 0;
 }
 
-static int jvp_object_length(jv_complex* object) {
+static int jvp_object_length(jv_nontrivial* object) {
   int n = 0;
   for (int i=0; i<jvp_object_size(object); i++) {
     struct object_slot* slot = jvp_object_get_slot(object, i);
@@ -786,7 +823,7 @@ static int jvp_object_length(jv_complex* object) {
   return n;
 }
 
-static int jvp_object_equal(jv_complex* o1, jv_complex* o2) {
+static int jvp_object_equal(jv_nontrivial* o1, jv_nontrivial* o2) {
   int len2 = jvp_object_length(o2);
   int len1 = 0;
   for (int i=0; i<jvp_object_size(o1); i++) {
@@ -808,14 +845,14 @@ static int jvp_object_equal(jv_complex* o1, jv_complex* o2) {
 jv jv_object() {
   jv j;
   j.kind = JV_KIND_OBJECT;
-  j.val.complex = jvp_object_new(8);
+  j.val.nontrivial = jvp_object_new(8);
   return j;
 }
 
 jv jv_object_get(jv object, jv key) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
-  jv* slot = jvp_object_read(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jv* slot = jvp_object_read(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv val;
   if (slot) {
     val = jv_copy(*slot);
@@ -831,7 +868,7 @@ jv jv_object_set(jv object, jv key, jv value) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
   // copy/free of object, key, value coalesced
-  jv* slot = jvp_object_write(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jv* slot = jvp_object_write(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv_free(*slot);
   *slot = value;
   return object;
@@ -840,29 +877,46 @@ jv jv_object_set(jv object, jv key, jv value) {
 jv jv_object_delete(jv object, jv key) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
-  jvp_object_delete(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jvp_object_delete(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv_free(key);
   return object;
 }
 
 int jv_object_length(jv object) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
-  int n = jvp_object_length(&object.val.complex);
+  int n = jvp_object_length(&object.val.nontrivial);
   jv_free(object);
   return n;
 }
 
 jv jv_object_merge(jv a, jv b) {
   assert(jv_get_kind(a) == JV_KIND_OBJECT);
-  jv_object_foreach(i, b) {
-    a = jv_object_set(a, 
-                      jv_object_iter_key(b, i),
-                      jv_object_iter_value(b, i));
+  jv_object_foreach(b, k, v) {
+    a = jv_object_set(a, k, v);
   }
   jv_free(b);
   return a;
 }
 
+int jv_object_contains(jv a, jv b) {
+  assert(jv_get_kind(a) == JV_KIND_OBJECT);
+  assert(jv_get_kind(b) == JV_KIND_OBJECT);
+  int r = 1;
+
+  jv_object_foreach(b, key, b_val) {
+    jv a_val = jv_object_get(jv_copy(a), jv_copy(key));
+
+    r = jv_contains(a_val, b_val);
+    jv_free(key);
+
+    if (!r) break;
+  }
+
+  jv_free(a);
+  jv_free(b);
+  return r;
+}
+
 /*
  * Object iteration (internal helpers)
  */
@@ -881,7 +935,7 @@ int jv_object_iter(jv object) {
 int jv_object_iter_next(jv object, int iter) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(iter != ITER_FINISHED);
-  jv_complex* o = &object.val.complex;
+  jv_nontrivial* o = &object.val.nontrivial;
   struct object_slot* slot;
   do {
     iter++;
@@ -893,18 +947,18 @@ int jv_object_iter_next(jv object, int iter) {
 }
 
 jv jv_object_iter_key(jv object, int iter) {
-  jvp_string* s = jvp_object_get_slot(&object.val.complex, iter)->string;
+  jvp_string* s = jvp_object_get_slot(&object.val.nontrivial, iter)->string;
   assert(s);
   jv j;
   j.kind = JV_KIND_STRING;
-  j.val.complex.ptr = &s->refcnt;
-  j.val.complex.i[0] = 0;
-  j.val.complex.i[1] = 0;
+  j.val.nontrivial.ptr = &s->refcnt;
+  j.val.nontrivial.i[0] = 0;
+  j.val.nontrivial.i[1] = 0;
   return jv_copy(j);
 }
 
 jv jv_object_iter_value(jv object, int iter) {
-  return jv_copy(jvp_object_get_slot(&object.val.complex, iter)->value);
+  return jv_copy(jvp_object_get_slot(&object.val.nontrivial, iter)->value);
 }
 
 /*
@@ -915,20 +969,20 @@ jv jv_copy(jv j) {
       jv_get_kind(j) == JV_KIND_STRING || 
       jv_get_kind(j) == JV_KIND_OBJECT ||
       jv_get_kind(j) == JV_KIND_INVALID) {
-    jvp_refcnt_inc(&j.val.complex);
+    jvp_refcnt_inc(&j.val.nontrivial);
   }
   return j;
 }
 
 void jv_free(jv j) {
   if (jv_get_kind(j) == JV_KIND_ARRAY) {
-    jvp_array_free(&j.val.complex);
+    jvp_array_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_STRING) {
-    jvp_string_free(&j.val.complex);
+    jvp_string_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_OBJECT) {
-    jvp_object_free(&j.val.complex);
+    jvp_object_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_INVALID) {
-    jvp_invalid_free(&j.val.complex);
+    jvp_invalid_free(&j.val.nontrivial);
   }
 }
 
@@ -937,7 +991,7 @@ int jv_get_refcnt(jv j) {
   case JV_KIND_ARRAY:
   case JV_KIND_STRING:
   case JV_KIND_OBJECT:
-    return j.val.complex.ptr->count;
+    return j.val.nontrivial.ptr->count;
   default:
     return 1;
   }
@@ -953,20 +1007,20 @@ int jv_equal(jv a, jv b) {
     r = 0;
   } else if (jv_get_kind(a) == JV_KIND_NUMBER) {
     r = jv_number_value(a) == jv_number_value(b);
-  } else if (a.val.complex.ptr == b.val.complex.ptr &&
-             a.val.complex.i[0] == b.val.complex.i[0] &&
-             a.val.complex.i[1] == b.val.complex.i[1]) {
+  } else if (a.val.nontrivial.ptr == b.val.nontrivial.ptr &&
+             a.val.nontrivial.i[0] == b.val.nontrivial.i[0] &&
+             a.val.nontrivial.i[1] == b.val.nontrivial.i[1]) {
     r = 1;
   } else {
     switch (jv_get_kind(a)) {
     case JV_KIND_ARRAY:
-      r = jvp_array_equal(&a.val.complex, &b.val.complex);
+      r = jvp_array_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     case JV_KIND_STRING:
-      r = jvp_string_equal(&a.val.complex, &b.val.complex);
+      r = jvp_string_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     case JV_KIND_OBJECT:
-      r = jvp_object_equal(&a.val.complex, &b.val.complex);
+      r = jvp_object_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     default:
       r = 1;
@@ -977,3 +1031,21 @@ int jv_equal(jv a, jv b) {
   jv_free(b);
   return r;
 }
+
+int jv_contains(jv a, jv b) {
+  int r = 1;
+  if (jv_get_kind(a) != jv_get_kind(b)) {
+    r = 0;
+  } else if (jv_get_kind(a) == JV_KIND_OBJECT) {
+    r = jv_object_contains(jv_copy(a), jv_copy(b));
+  } else if (jv_get_kind(a) == JV_KIND_ARRAY) {
+    r = jv_array_contains(jv_copy(a), jv_copy(b));
+  } else if (jv_get_kind(a) == JV_KIND_STRING) {
+    r = strstr(jv_string_value(a), jv_string_value(b)) != 0;
+  } else {
+    r = jv_equal(jv_copy(a), jv_copy(b));
+  }
+  jv_free(a);
+  jv_free(b);
+  return r;
+}

jv.h

@@ -1,12 +1,6 @@
 #ifndef JV_H
 #define JV_H
 
-#include <stdint.h>
-#include <assert.h>
-#include <stddef.h>
-
-
-
 typedef enum {
   JV_KIND_INVALID,
   JV_KIND_NULL,
@@ -18,20 +12,17 @@ typedef enum {
   JV_KIND_OBJECT
 } jv_kind;
 
-typedef struct {
-  size_t count;
-} jv_refcnt;
-
+struct jv_refcnt;
 typedef struct{
-  jv_refcnt* ptr;
+  struct jv_refcnt* ptr;
   int i[2];
-} jv_complex;
+} jv_nontrivial;
 
 typedef struct {
   jv_kind kind;
   union {
     double number;
-    jv_complex complex;
+    jv_nontrivial nontrivial;
   } val;
 } jv;
 
@@ -47,7 +38,10 @@ static int jv_is_valid(jv x) { return jv_get_kind(x) != JV_KIND_INVALID; }
 jv jv_copy(jv);
 void jv_free(jv);
 
+int jv_get_refcnt(jv);
+
 int jv_equal(jv, jv);
+int jv_contains(jv, jv);
 
 jv jv_invalid();
 jv jv_invalid_with_msg(jv);
@@ -71,12 +65,19 @@ jv jv_array_set(jv, int, jv);
 jv jv_array_append(jv, jv);
 jv jv_array_concat(jv, jv);
 jv jv_array_slice(jv, int, int);
-
+#define jv_array_foreach(a, i, x) \
+  for (int jv_len__ = jv_array_length(jv_copy(a)), i=0, jv_j__ = 1;     \
+       jv_j__; jv_j__ = 0)                                              \
+    for (jv x;                                                          \
+         i < jv_len__ ?                                                 \
+           (x = jv_array_get(jv_copy(a), i), 1) : 0;                    \
+         i++)
 
 jv jv_string(const char*);
 jv jv_string_sized(const char*, int);
-int jv_string_length(jv);
-uint32_t jv_string_hash(jv);
+int jv_string_length_bytes(jv);
+int jv_string_length_codepoints(jv);
+unsigned long jv_string_hash(jv);
 const char* jv_string_value(jv);
 jv jv_string_concat(jv, jv);
 jv jv_string_fmt(const char*, ...);
@@ -95,90 +96,48 @@ int jv_object_iter_next(jv, int);
 int jv_object_iter_valid(jv, int);
 jv jv_object_iter_key(jv, int);
 jv jv_object_iter_value(jv, int);
-#define jv_object_foreach(i,t) \
-  for (int i = jv_object_iter(t);               \
-       jv_object_iter_valid(t, i);              \
-       i = jv_object_iter_next(t, i))           \
+#define jv_object_foreach(t, k, v)                                      \
+  for (int jv_i__ = jv_object_iter(t), jv_j__ = 1; jv_j__; jv_j__ = 0)  \
+    for (jv k, v;                                                       \
+         jv_object_iter_valid((t), jv_i__) ?                            \
+           (k = jv_object_iter_key(t, jv_i__),                          \
+            v = jv_object_iter_value(t, jv_i__),                        \
+            1)                                                          \
+           : 0;                                                         \
+         jv_i__ = jv_object_iter_next(t, jv_i__))                       \
  
 
 
 int jv_get_refcnt(jv);
 
-enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2 };
+enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4, JV_PRINT_SORTED = 8 };
 void jv_dump(jv, int flags);
 jv jv_dump_string(jv, int flags);
 
 jv jv_parse(const char* string);
 jv jv_parse_sized(const char* string, int length);
 
+typedef void (*jv_nomem_handler_f)(void *);
+void jv_nomem_handler(jv_nomem_handler_f, void *);
 
-static jv jv_lookup(jv t, jv k) {
-  jv v;
-  if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) {
-    v = jv_object_get(t, k);
-    if (!jv_is_valid(v)) {
-      jv_free(v);
-      v = jv_null();
-    }
-  } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_NUMBER) {
-    // FIXME: don't do lookup for noninteger index
-    v = jv_array_get(t, (int)jv_number_value(k));
-    if (!jv_is_valid(v)) {
-      jv_free(v);
-      v = jv_null();
-    }
-  } else if (jv_get_kind(t) == JV_KIND_NULL && 
-             (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER)) {
-    jv_free(t);
-    jv_free(k);
-    v = jv_null();
-  } else {
-    v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with %s",
-                                          jv_kind_name(jv_get_kind(t)),
-                                          jv_kind_name(jv_get_kind(k))));
-    jv_free(t);
-    jv_free(k);
-  }
-  return v;
-}
-
-static jv jv_modify(jv t, jv k, jv v) {
-  int isnull = jv_get_kind(t) == JV_KIND_NULL;
-  if (jv_get_kind(k) == JV_KIND_STRING && 
-      (jv_get_kind(t) == JV_KIND_OBJECT || isnull)) {
-    if (isnull) t = jv_object();
-    t = jv_object_set(t, k, v);
-  } else if (jv_get_kind(k) == JV_KIND_NUMBER &&
-             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
-    if (isnull) t = jv_array();
-    t = jv_array_set(t, (int)jv_number_value(k), v);
-  } else {
-    jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s",
-                                               jv_kind_name(jv_get_kind(t)),
-                                               jv_kind_name(jv_get_kind(v))));
-    jv_free(t);
-    jv_free(k);
-    jv_free(v);
-    t = err;
-  }
-  return t;
-}
-
-static jv jv_insert(jv root, jv value, jv* path, int pathlen) {
-  if (pathlen == 0) {
-    jv_free(root);
-    return value;
-  }
-  return jv_modify(root, jv_copy(*path), 
-                   jv_insert(jv_lookup(jv_copy(root), jv_copy(*path)), value, path+1, pathlen-1));
-}
-
-
-
-
-
+jv jv_load_file(const char *, int);
 
+struct jv_parser;
+struct jv_parser* jv_parser_new();
+void jv_parser_set_buf(struct jv_parser*, const char*, int, int);
+jv jv_parser_next(struct jv_parser*);
+void jv_parser_free(struct jv_parser*);
 
+jv jv_get(jv, jv);
+jv jv_set(jv, jv, jv);
+jv jv_has(jv, jv);
+jv jv_setpath(jv, jv, jv);
+jv jv_getpath(jv, jv);
+jv jv_delpaths(jv, jv);
+jv jv_keys(jv /*object or array*/);
+int jv_cmp(jv, jv);
+jv jv_group(jv, jv);
+jv jv_sort(jv, jv);
 
 
 #endif

jv_alloc.c

@@ -0,0 +1,119 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include "jv_alloc.h"
+
+struct nomem_handler {
+    jv_nomem_handler_f handler;
+    void *data;
+};
+
+#ifndef USE_PTHREAD_KEY
+#ifdef _MSC_VER
+static __declspec(thread) struct nomem_handler nomem_handler;
+#else
+static __thread struct nomem_handler nomem_handler;
+#endif
+
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  nomem_handler.handler = handler;
+}
+
+static void memory_exhausted() {
+  if (nomem_handler.handler)
+    nomem_handler.handler(nomem_handler.data); // Maybe handler() will longjmp() to safety
+  // Or not
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+#else
+#ifdef HAVE_PTHREAD_KEY_CREATE
+#include <pthread.h>
+
+pthread_key_t nomem_handler_key;
+pthread_once_t mem_once = PTHREAD_ONCE_INIT;
+
+static void tsd_init(void) {
+  if (pthread_key_create(&nomem_handler_key, NULL) != 0) {
+    fprintf(stderr, "error: cannot create thread specific key");
+    abort();
+  }
+}
+
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  pthread_once(&mem_once, tsd_init); // cannot fail
+  struct nomem_handler *nomem_handler = calloc(1, sizeof (nomem_handler));
+  if (nomem_handler == NULL) {
+    handler(data);
+    fprintf(stderr, "error: cannot allocate memory\n");
+    abort();
+  }
+  nomem_handler.handler = handler;
+  nomem_handler.data = data;
+  if (pthread_setspecific(nomem_handler_key, nomem_handler) != 0) {
+    handler(data);
+    fprintf(stderr, "error: cannot set thread specific data");
+    abort();
+  }
+}
+
+static void memory_exhausted() {
+  jv_nomem_handler_f handler;
+
+  pthread_once(&mem_once, tsd_init);
+  handler = pthread_getspecific(nomem_handler_key);
+  if (handler)
+    handler(); // Maybe handler() will longjmp() to safety
+  // Or not
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+
+#else
+
+static struct nomem_handler nomem_handler;
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  nomem_handler.handler = handler;
+  nomem_handler.data = data;
+}
+
+static void memory_exhausted() {
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+
+#endif /* HAVE_PTHREAD_KEY_CREATE */
+#endif /* !USE_PTHREAD_KEY */
+
+
+void* jv_mem_alloc(size_t sz) {
+  void* p = malloc(sz);
+  if (!p) {
+    memory_exhausted();
+  }
+  return p;
+}
+
+void* jv_mem_alloc_unguarded(size_t sz) {
+  return malloc(sz);
+}
+
+void jv_mem_free(void* p) {
+  free(p);
+}
+
+void* jv_mem_realloc(void* p, size_t sz) {
+  p = realloc(p, sz);
+  if (!p) {
+    memory_exhausted();
+  }
+  return p;
+}
+
+#ifndef NDEBUG
+volatile char jv_mem_uninitialised;
+__attribute__((constructor)) void jv_mem_uninit_setup(){
+  char* p = malloc(1);
+  jv_mem_uninitialised = *p;
+  free(p);
+}
+#endif

jv_alloc.h

@@ -0,0 +1,23 @@
+#ifndef JV_ALLOC_H
+#define JV_ALLOC_H
+
+#include <stddef.h>
+#include "jv.h"
+
+#ifndef NDEBUG
+extern volatile char jv_mem_uninitialised;
+#endif
+
+static void jv_mem_invalidate(void* mem, size_t n) {
+#ifndef NDEBUG
+  char* m = mem;
+  while (n--) *m++ ^= jv_mem_uninitialised ^ jv_mem_uninitialised;
+#endif
+}
+
+void* jv_mem_alloc(size_t);
+void* jv_mem_alloc_unguarded(size_t);
+void jv_mem_free(void*);
+__attribute__((warn_unused_result)) void* jv_mem_realloc(void*, size_t);
+
+#endif

jv_aux.c

@@ -0,0 +1,565 @@
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "jv_alloc.h"
+
+static int parse_slice(jv array, jv slice, int* pstart, int* pend) {
+  // Array slices
+  int len = jv_array_length(jv_copy(array));
+  jv start_jv = jv_object_get(jv_copy(slice), jv_string("start"));
+  jv end_jv = jv_object_get(slice, jv_string("end"));
+  if (jv_get_kind(start_jv) == JV_KIND_NULL) {
+    jv_free(start_jv);
+    start_jv = jv_number(0);
+  }
+  if (jv_get_kind(end_jv) == JV_KIND_NULL) {
+    jv_free(end_jv);
+    end_jv = jv_number(len);
+  }
+  if (jv_get_kind(start_jv) != JV_KIND_NUMBER ||
+      jv_get_kind(end_jv) != JV_KIND_NUMBER) {
+    jv_free(start_jv);
+    jv_free(end_jv);
+    return 0;
+  } else {
+    int start = (int)jv_number_value(start_jv);
+    int end = (int)jv_number_value(end_jv);
+    if (start < 0) start = len + start;
+    if (end < 0) end = len + end;
+
+    if (start < 0) start = 0;
+    if (start > len) start = len;
+    if (end > len) end = len;
+    if (end < start) end = start;
+    assert(0 <= start && start <= end && end <= len);
+    *pstart = start;
+    *pend = end;
+    return 1;
+  }
+}
+
+jv jv_get(jv t, jv k) {
+  jv v;
+  if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) {
+    v = jv_object_get(t, k);
+    if (!jv_is_valid(v)) {
+      jv_free(v);
+      v = jv_null();
+    }
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_NUMBER) {
+    // FIXME: don't do lookup for noninteger index
+    v = jv_array_get(t, (int)jv_number_value(k));
+    if (!jv_is_valid(v)) {
+      jv_free(v);
+      v = jv_null();
+    }
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_OBJECT) {
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      v = jv_array_slice(t, start, end);
+    } else {
+      v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+      jv_free(t);
+    }
+  } else if (jv_get_kind(t) == JV_KIND_NULL && 
+             (jv_get_kind(k) == JV_KIND_STRING || 
+              jv_get_kind(k) == JV_KIND_NUMBER || 
+              jv_get_kind(k) == JV_KIND_OBJECT)) {
+    jv_free(t);
+    jv_free(k);
+    v = jv_null();
+  } else {
+    v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with %s",
+                                          jv_kind_name(jv_get_kind(t)),
+                                          jv_kind_name(jv_get_kind(k))));
+    jv_free(t);
+    jv_free(k);
+  }
+  return v;
+}
+
+jv jv_set(jv t, jv k, jv v) {
+  if (!jv_is_valid(v)) {
+    jv_free(t);
+    jv_free(k);
+    return v;
+  }
+  int isnull = jv_get_kind(t) == JV_KIND_NULL;
+  if (jv_get_kind(k) == JV_KIND_STRING && 
+      (jv_get_kind(t) == JV_KIND_OBJECT || isnull)) {
+    if (isnull) t = jv_object();
+    t = jv_object_set(t, k, v);
+  } else if (jv_get_kind(k) == JV_KIND_NUMBER &&
+             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
+    if (isnull) t = jv_array();
+    t = jv_array_set(t, (int)jv_number_value(k), v);
+  } else if (jv_get_kind(k) == JV_KIND_OBJECT &&
+             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
+    if (isnull) t = jv_array();
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      if (jv_get_kind(v) == JV_KIND_ARRAY) {
+        int array_len = jv_array_length(jv_copy(t));
+        assert(0 <= start && start <= end && end <= array_len);
+        int slice_len = end - start;
+        int insert_len = jv_array_length(jv_copy(v));
+        if (slice_len < insert_len) {
+          // array is growing
+          int shift = insert_len - slice_len;
+          for (int i = array_len - 1; i >= end; i--) {
+            t = jv_array_set(t, i + shift, jv_array_get(jv_copy(t), i));
+          }
+        } else if (slice_len > insert_len) {
+          // array is shrinking
+          int shift = slice_len - insert_len;
+          for (int i = end; i < array_len; i++) {
+            t = jv_array_set(t, i - shift, jv_array_get(jv_copy(t), i));
+          }
+          t = jv_array_slice(t, 0, array_len - shift);
+        }
+        for (int i=0; i < insert_len; i++) {
+          t = jv_array_set(t, start + i, jv_array_get(jv_copy(v), i));
+        }
+        jv_free(v);
+      } else {
+        jv_free(t);
+        jv_free(v);
+        t = jv_invalid_with_msg(jv_string_fmt("A slice of an array can only be assigned another array"));
+      }
+    } else {
+      jv_free(t);
+      jv_free(k);
+      jv_free(v);
+      t = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+    }
+  } else {
+    jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s",
+                                               jv_kind_name(jv_get_kind(k)),
+                                               jv_kind_name(jv_get_kind(t))));
+    jv_free(t);
+    jv_free(k);
+    jv_free(v);
+    t = err;
+  }
+  return t;
+}
+
+jv jv_has(jv t, jv k) {
+  assert(jv_is_valid(t));
+  assert(jv_is_valid(k));
+  jv ret;
+  if (jv_get_kind(t) == JV_KIND_NULL) {
+    jv_free(t);
+    jv_free(k);
+    ret = jv_false();
+  } else if (jv_get_kind(t) == JV_KIND_OBJECT &&
+             jv_get_kind(k) == JV_KIND_STRING) {
+    jv elem = jv_object_get(t, k);
+    ret = jv_bool(jv_is_valid(elem));
+    jv_free(elem);
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY &&
+             jv_get_kind(k) == JV_KIND_NUMBER) {
+    jv elem = jv_array_get(t, (int)jv_number_value(k));
+    ret = jv_bool(jv_is_valid(elem));
+    jv_free(elem);
+  } else {
+    ret = jv_invalid_with_msg(jv_string_fmt("Cannot check whether %s has a %s key",
+                                            jv_kind_name(jv_get_kind(t)),
+                                            jv_kind_name(jv_get_kind(k))));
+    jv_free(t);
+    jv_free(k);
+  }
+  return ret;
+}
+
+// assumes keys is a sorted array
+jv jv_dels(jv t, jv keys) {
+  assert(jv_get_kind(keys) == JV_KIND_ARRAY);
+  assert(jv_is_valid(t));
+  
+  if (jv_get_kind(t) == JV_KIND_NULL || jv_array_length(jv_copy(keys)) == 0) {
+    // no change
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY) {
+    // extract slices, they must be handled differently
+    jv orig_keys = keys;
+    keys = jv_array();
+    jv new_array = jv_array();
+    jv starts = jv_array(), ends = jv_array();
+    jv_array_foreach(orig_keys, i, key) {
+      if (jv_get_kind(key) == JV_KIND_NUMBER) {
+        keys = jv_array_append(keys, key);
+      } else if (jv_get_kind(key) == JV_KIND_OBJECT) {
+        int start, end;
+        if (parse_slice(t, key, &start, &end)) {
+          starts = jv_array_append(starts, jv_number(start));
+          ends = jv_array_append(ends, jv_number(end));
+        } else {
+          jv_free(new_array);
+          jv_free(key);
+          new_array = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+          goto arr_out;
+        }
+      } else {
+        jv_free(new_array);
+        new_array = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array",
+                                                      jv_kind_name(jv_get_kind(key))));
+        jv_free(key);
+        goto arr_out;
+      }
+    }
+
+    int kidx = 0;
+    jv_array_foreach(t, i, elem) {
+      int del = 0;
+      while (kidx < jv_array_length(jv_copy(keys))) {
+        int delidx = (int)jv_number_value(jv_array_get(jv_copy(keys), kidx));
+        if (i == delidx) {
+          del = 1;
+        }
+        if (i < delidx) {
+          break;
+        }
+        kidx++;
+      }
+      for (int sidx=0; !del && sidx<jv_array_length(jv_copy(starts)); sidx++) {
+        if ((int)jv_number_value(jv_array_get(jv_copy(starts), sidx)) <= i &&
+            i < (int)jv_number_value(jv_array_get(jv_copy(ends), sidx))) {
+          del = 1;
+        }
+      }
+      if (!del)
+        new_array = jv_array_append(new_array, elem);
+      else
+        jv_free(elem);
+    }
+  arr_out:
+    jv_free(starts);
+    jv_free(ends);
+    jv_free(orig_keys);
+    jv_free(t);
+    t = new_array;
+  } else if (jv_get_kind(t) == JV_KIND_OBJECT) {
+    jv_array_foreach(keys, i, k) {
+      if (jv_get_kind(k) != JV_KIND_STRING) {
+        jv_free(t);
+        t = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s field of object",
+                                              jv_kind_name(jv_get_kind(k))));
+        jv_free(k);
+        break;
+      }
+      t = jv_object_delete(t, k);      
+    }
+  } else {
+    jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete fields from %s",
+                                               jv_kind_name(jv_get_kind(t))));
+    jv_free(t);
+    t = err;
+  }
+  jv_free(keys);
+  return t;
+}
+
+jv jv_setpath(jv root, jv path, jv value) {
+  if (jv_get_kind(path) != JV_KIND_ARRAY) {
+    jv_free(value);
+    jv_free(root);
+    jv_free(path);
+    return jv_invalid_with_msg(jv_string("Path must be specified as an array"));
+  }
+  if (!jv_is_valid(root)){
+    jv_free(value);
+    jv_free(path);
+    return root;
+  }
+  if (jv_array_length(jv_copy(path)) == 0) {
+    jv_free(path);
+    jv_free(root);
+    return value;
+  }
+  jv pathcurr = jv_array_get(jv_copy(path), 0);
+  jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path)));
+  return jv_set(root, pathcurr, 
+                jv_setpath(jv_get(jv_copy(root), jv_copy(pathcurr)), pathrest, value));
+}
+
+jv jv_getpath(jv root, jv path) {
+  if (jv_get_kind(path) != JV_KIND_ARRAY) {
+    jv_free(root);
+    jv_free(path);
+    return jv_invalid_with_msg(jv_string("Path must be specified as an array"));
+  }
+  if (!jv_is_valid(root)) {
+    jv_free(path);
+    return root;
+  }
+  if (jv_array_length(jv_copy(path)) == 0) {
+    jv_free(path);
+    return root;
+  }
+  jv pathcurr = jv_array_get(jv_copy(path), 0);
+  jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path)));
+  return jv_getpath(jv_get(root, pathcurr), pathrest);
+}
+
+// assumes paths is a sorted array of arrays
+static jv delpaths_sorted(jv object, jv paths, int start) {
+  jv delkeys = jv_array();
+  for (int i=0; i<jv_array_length(jv_copy(paths));) {
+    int j = i;
+    assert(jv_array_length(jv_array_get(jv_copy(paths), i)) > start);
+    int delkey = jv_array_length(jv_array_get(jv_copy(paths), i)) == start + 1;
+    jv key = jv_array_get(jv_array_get(jv_copy(paths), i), start);
+    while (j < jv_array_length(jv_copy(paths)) &&
+           jv_equal(jv_copy(key), jv_array_get(jv_array_get(jv_copy(paths), j), start)))
+      j++;
+    // if i <= entry < j, then entry starts with key
+    if (delkey) {
+      // deleting this entire key, we don't care about any more specific deletions
+      delkeys = jv_array_append(delkeys, key);
+    } else {
+      // deleting certain sub-parts of this key
+      jv subobject = jv_get(jv_copy(object), jv_copy(key));
+      if (!jv_is_valid(subobject)) {
+        jv_free(key);
+        jv_free(object);
+        object = subobject;
+        break;
+      } else if (jv_get_kind(subobject) == JV_KIND_NULL) {
+        jv_free(key);
+        jv_free(subobject);
+      } else {
+        jv newsubobject = delpaths_sorted(subobject, jv_array_slice(jv_copy(paths), i, j), start+1);
+        if (!jv_is_valid(newsubobject)) {
+          jv_free(key);
+          jv_free(object);
+          object = newsubobject;
+          break;
+        }
+        object = jv_set(object, key, newsubobject);
+      }
+      if (!jv_is_valid(object)) break;
+    }
+    i = j;
+  }
+  jv_free(paths);
+  if (jv_is_valid(object))
+    object = jv_dels(object, delkeys);
+  else 
+    jv_free(delkeys);
+  return object;
+}
+
+jv jv_delpaths(jv object, jv paths) {
+  paths = jv_sort(paths, jv_copy(paths));
+  jv_array_foreach(paths, i, elem) {
+    if (jv_get_kind(elem) != JV_KIND_ARRAY) {
+      jv_free(object);
+      jv_free(paths);
+      jv err = jv_invalid_with_msg(jv_string_fmt("Path must be specified as array, not %s",
+                                                 jv_kind_name(jv_get_kind(elem))));
+      jv_free(elem);
+      return err;
+    }
+    jv_free(elem);
+  }
+  if (jv_array_length(jv_copy(paths)) == 0) {
+    // nothing is being deleted
+    jv_free(paths);
+    return object;
+  }
+  if (jv_array_length(jv_array_get(jv_copy(paths), 0)) == 0) {
+    // everything is being deleted
+    jv_free(paths);
+    jv_free(object);
+    return jv_null();
+  }
+  return delpaths_sorted(object, paths, 0);
+}
+
+
+static int string_cmp(const void* pa, const void* pb){
+  const jv* a = pa;
+  const jv* b = pb;
+  int lena = jv_string_length_bytes(jv_copy(*a));
+  int lenb = jv_string_length_bytes(jv_copy(*b));
+  int minlen = lena < lenb ? lena : lenb;
+  int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen);
+  if (r == 0) r = lena - lenb;
+  return r;
+}
+
+jv jv_keys(jv x) {
+  if (jv_get_kind(x) == JV_KIND_OBJECT) {
+    int nkeys = jv_object_length(jv_copy(x));
+    jv* keys = jv_mem_alloc(sizeof(jv) * nkeys);
+    int kidx = 0;
+    jv_object_foreach(x, key, value) {
+      keys[kidx++] = key;
+      jv_free(value);
+    }
+    qsort(keys, nkeys, sizeof(jv), string_cmp);
+    jv answer = jv_array_sized(nkeys);
+    for (int i = 0; i<nkeys; i++) {
+      answer = jv_array_append(answer, keys[i]);
+    }
+    jv_mem_free(keys);
+    jv_free(x);
+    return answer;
+  } else if (jv_get_kind(x) == JV_KIND_ARRAY) {
+    int n = jv_array_length(x);
+    jv answer = jv_array();
+    for (int i=0; i<n; i++){
+      answer = jv_array_set(answer, i, jv_number(i));
+    }
+    return answer;
+  } else {
+    assert(0 && "jv_keys passed something neither object nor array");
+    return jv_invalid();
+  }
+}
+
+int jv_cmp(jv a, jv b) {
+  if (jv_get_kind(a) != jv_get_kind(b)) {
+    int r = (int)jv_get_kind(a) - (int)jv_get_kind(b);
+    jv_free(a);
+    jv_free(b);
+    return r;
+  }
+  int r = 0;
+  switch (jv_get_kind(a)) {
+  default:
+    assert(0 && "invalid kind passed to jv_cmp");
+  case JV_KIND_NULL:
+  case JV_KIND_FALSE:
+  case JV_KIND_TRUE:
+    // there's only one of each of these values
+    r = 0;
+    break;
+
+  case JV_KIND_NUMBER: {
+    double da = jv_number_value(a), db = jv_number_value(b);
+    
+    // handle NaN as though it were null
+    if (da != da) r = jv_cmp(jv_null(), jv_number(db));
+    else if (db != db) r = jv_cmp(jv_number(da), jv_null());
+    else if (da < db) r = -1;
+    else if (da == db) r = 0;
+    else r = 1;
+    break;
+  }
+
+  case JV_KIND_STRING: {
+    r = string_cmp(&a, &b);
+    break;
+  }
+
+  case JV_KIND_ARRAY: {
+    // Lexical ordering of arrays
+    int i = 0;
+    while (r == 0) {
+      int a_done = i >= jv_array_length(jv_copy(a));
+      int b_done = i >= jv_array_length(jv_copy(b));
+      if (a_done || b_done) {
+        r = b_done - a_done; //suddenly, logic
+        break;
+      }
+      jv xa = jv_array_get(jv_copy(a), i);
+      jv xb = jv_array_get(jv_copy(b), i);
+      r = jv_cmp(xa, xb);
+      i++;
+    }
+    break;
+  }
+
+  case JV_KIND_OBJECT: {
+    jv keys_a = jv_keys(jv_copy(a));
+    jv keys_b = jv_keys(jv_copy(b));
+    r = jv_cmp(jv_copy(keys_a), keys_b);
+    if (r == 0) {
+      jv_array_foreach(keys_a, i, key) {
+        jv xa = jv_object_get(jv_copy(a), jv_copy(key));
+        jv xb = jv_object_get(jv_copy(b), key);
+        r = jv_cmp(xa, xb);
+        if (r) break;
+      }
+    }
+    jv_free(keys_a);
+    break;
+  }
+  }
+
+  jv_free(a);
+  jv_free(b);
+  return r;
+}
+
+
+struct sort_entry {
+  jv object;
+  jv key;
+};
+
+static int sort_cmp(const void* pa, const void* pb) {
+  const struct sort_entry* a = pa;
+  const struct sort_entry* b = pb;
+  int r = jv_cmp(jv_copy(a->key), jv_copy(b->key));
+  // comparing by address if r == 0 makes the sort stable
+  return r ? r : (int)(a - b);
+}
+
+static struct sort_entry* sort_items(jv objects, jv keys) {
+  assert(jv_get_kind(objects) == JV_KIND_ARRAY);
+  assert(jv_get_kind(keys) == JV_KIND_ARRAY);
+  assert(jv_array_length(jv_copy(objects)) == jv_array_length(jv_copy(keys)));
+  int n = jv_array_length(jv_copy(objects));
+  struct sort_entry* entries = jv_mem_alloc(sizeof(struct sort_entry) * n);
+  for (int i=0; i<n; i++) {
+    entries[i].object = jv_array_get(jv_copy(objects), i);
+    entries[i].key = jv_array_get(jv_copy(keys), i);
+  }
+  jv_free(objects);
+  jv_free(keys);
+  qsort(entries, n, sizeof(struct sort_entry), sort_cmp);
+  return entries;
+}
+
+jv jv_sort(jv objects, jv keys) {
+  assert(jv_get_kind(objects) == JV_KIND_ARRAY);
+  assert(jv_get_kind(keys) == JV_KIND_ARRAY);
+  assert(jv_array_length(jv_copy(objects)) == jv_array_length(jv_copy(keys)));
+  int n = jv_array_length(jv_copy(objects));
+  struct sort_entry* entries = sort_items(objects, keys);
+  jv ret = jv_array();
+  for (int i=0; i<n; i++) {
+    jv_free(entries[i].key);
+    ret = jv_array_set(ret, i, entries[i].object);
+  }
+  jv_mem_free(entries);
+  return ret;
+}
+
+jv jv_group(jv objects, jv keys) {
+  assert(jv_get_kind(objects) == JV_KIND_ARRAY);
+  assert(jv_get_kind(keys) == JV_KIND_ARRAY);
+  assert(jv_array_length(jv_copy(objects)) == jv_array_length(jv_copy(keys)));
+  int n = jv_array_length(jv_copy(objects));
+  struct sort_entry* entries = sort_items(objects, keys);
+  jv ret = jv_array();
+  jv curr_key = entries[0].key;
+  jv group = jv_array_append(jv_array(), entries[0].object);
+  for (int i = 1; i < n; i++) {
+    if (jv_equal(jv_copy(curr_key), jv_copy(entries[i].key))) {
+      jv_free(entries[i].key);
+    } else {
+      jv_free(curr_key);
+      curr_key = entries[i].key;
+      ret = jv_array_append(ret, group);
+      group = jv_array();
+    }
+    group = jv_array_append(group, entries[i].object);
+  }
+  jv_free(curr_key);
+  ret = jv_array_append(ret, group);
+  jv_mem_free(entries);
+  return ret;
+}

jv_dtoa.c

@@ -195,6 +195,11 @@
 
 #include "jv_dtoa.h"
 
+#include "jv_alloc.h"
+#define MALLOC jv_mem_alloc
+#define FREE jv_mem_free
+
+
 
 #ifndef Long
 #define Long long
@@ -572,7 +577,7 @@ void jvp_dtoa_context_free(struct dtoa_context* C) {
     while (C->freelist[k]) {
       Bigint* v = C->freelist[k];
       C->freelist[k] = v->next;
-      free(v);
+      FREE(v);
     }
   }
 }

jv_file.c

@@ -0,0 +1,46 @@
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "jv.h"
+
+jv jv_load_file(const char* filename, int raw) {
+  FILE* file = fopen(filename, "r");
+  struct jv_parser* parser;
+  jv data;
+  if (!file) {
+    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
+                                             filename,
+                                             strerror(errno)));
+  }
+  if (raw) {
+    data = jv_string("");
+  } else {
+    data = jv_array();
+    parser = jv_parser_new();
+  }
+  while (!feof(file) && !ferror(file)) {
+    char buf[4096];
+    size_t n = fread(buf, 1, sizeof(buf), file);
+    if (raw) {
+      data = jv_string_concat(data, jv_string_sized(buf, (int)n));
+    } else {
+      jv_parser_set_buf(parser, buf, n, !feof(file));
+      jv value;
+      while (jv_is_valid((value = jv_parser_next(parser))))
+        data = jv_array_append(data, value);
+      jv_free(value);
+    }
+  }
+  if (!raw)
+      jv_parser_free(parser);
+  int badread = ferror(file);
+  fclose(file);
+  if (badread) {
+    jv_free(data);
+    return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
+                                             filename));
+  }
+  return data;
+}

jv_parse.c

@@ -1,10 +1,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 #include "jv.h"
 #include "jv_dtoa.h"
-#include "jv_parse.h"
 #include "jv_unicode.h"
+#include "jv_alloc.h"
+#include "jv_dtoa.h"
 
 typedef const char* presult;
 
@@ -15,7 +17,35 @@ typedef const char* presult;
 #define pfunc presult
 #endif
 
-void jv_parser_init(struct jv_parser* p) {
+struct jv_parser {
+  const char* curr_buf;
+  int curr_buf_length;
+  int curr_buf_pos;
+  int curr_buf_is_partial;
+  unsigned bom_strip_position;
+
+  jv* stack;
+  int stackpos;
+  int stacklen;
+  jv next;
+  
+  char* tokenbuf;
+  int tokenpos;
+  int tokenlen;
+
+  int line, column;
+  
+  struct dtoa_context dtoa;
+
+  enum {
+    JV_PARSER_NORMAL,
+    JV_PARSER_STRING,
+    JV_PARSER_STRING_ESCAPE
+  } st;
+};
+
+
+static void parser_init(struct jv_parser* p) {
   p->stack = 0;
   p->stacklen = p->stackpos = 0;
   p->next = jv_invalid();
@@ -24,15 +54,18 @@ void jv_parser_init(struct jv_parser* p) {
   p->st = JV_PARSER_NORMAL;
   p->curr_buf = 0;
   p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0;
+  p->bom_strip_position = 0;
+  p->line = 1;
+  p->column = 0;
   jvp_dtoa_context_init(&p->dtoa);
 }
 
-void jv_parser_free(struct jv_parser* p) {
+static void parser_free(struct jv_parser* p) {
   jv_free(p->next);
   for (int i=0; i<p->stackpos; i++) 
     jv_free(p->stack[i]);
-  free(p->stack);
-  free(p->tokenbuf);
+  jv_mem_free(p->stack);
+  jv_mem_free(p->tokenbuf);
   jvp_dtoa_context_free(&p->dtoa);
 }
 
@@ -47,7 +80,7 @@ static void push(struct jv_parser* p, jv v) {
   assert(p->stackpos <= p->stacklen);
   if (p->stackpos == p->stacklen) {
     p->stacklen = p->stacklen * 2 + 10;
-    p->stack = realloc(p->stack, p->stacklen * sizeof(jv));
+    p->stack = jv_mem_realloc(p->stack, p->stacklen * sizeof(jv));
   }
   assert(p->stackpos < p->stacklen);
   p->stack[p->stackpos++] = v;
@@ -141,7 +174,7 @@ static void tokenadd(struct jv_parser* p, char c) {
   assert(p->tokenpos <= p->tokenlen);
   if (p->tokenpos == p->tokenlen) {
     p->tokenlen = p->tokenlen*2 + 256;
-    p->tokenbuf = realloc(p->tokenbuf, p->tokenlen);
+    p->tokenbuf = jv_mem_realloc(p->tokenbuf, p->tokenlen);
   }
   assert(p->tokenpos < p->tokenlen);
   p->tokenbuf[p->tokenpos++] = c;
@@ -279,7 +312,7 @@ static chclass classify(char c) {
 }
 
 
-static presult OK = "output produced";
+static const presult OK = "output produced";
 
 static int check_done(struct jv_parser* p, jv* out) {
   if (p->stackpos == 0 && jv_is_valid(p->next)) {
@@ -292,6 +325,11 @@ static int check_done(struct jv_parser* p, jv* out) {
 }
 
 static pfunc scan(struct jv_parser* p, char ch, jv* out) {
+  p->column++;
+  if (ch == '\n') {
+    p->line++;
+    p->column = 0;
+  }
   presult answer = 0;
   if (p->st == JV_PARSER_NORMAL) {
     chclass cls = classify(ch);
@@ -332,9 +370,38 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
   return answer;
 }
 
+struct jv_parser* jv_parser_new() {
+  struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser));
+  parser_init(p);
+  return p;
+}
+
+void jv_parser_free(struct jv_parser* p) {
+  parser_free(p);
+  jv_mem_free(p);
+}
+
+static const unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};
+
 void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) {
   assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length)
          && "previous buffer not exhausted");
+  while (p->bom_strip_position < sizeof(UTF8_BOM)) {
+    if ((unsigned char)*buf == UTF8_BOM[p->bom_strip_position]) {
+      // matched a BOM character
+      buf++;
+      length--;
+      p->bom_strip_position++;
+    } else {
+      if (p->bom_strip_position == 0) {
+        // no BOM in this document
+        p->bom_strip_position = sizeof(UTF8_BOM);
+      } else {
+        // malformed BOM (prefix present, rest missing)
+        p->bom_strip_position = 0xff;
+      }
+    }
+  }
   p->curr_buf = buf;
   p->curr_buf_length = length;
   p->curr_buf_pos = 0;
@@ -343,6 +410,7 @@ void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_
 
 jv jv_parser_next(struct jv_parser* p) {
   assert(p->curr_buf && "a buffer must be provided");
+  if (p->bom_strip_position == 0xff) return jv_invalid_with_msg(jv_string("Malformed BOM"));
   jv value;
   presult msg = 0;
   while (!msg && p->curr_buf_pos < p->curr_buf_length) {
@@ -352,7 +420,7 @@ jv jv_parser_next(struct jv_parser* p) {
   if (msg == OK) {
     return value;
   } else if (msg) {
-    return jv_invalid_with_msg(jv_string(msg));
+    return jv_invalid_with_msg(jv_string_fmt("%s at line %d, column %d", msg, p->line, p->column));
   } else if (p->curr_buf_is_partial) {
     assert(p->curr_buf_pos == p->curr_buf_length);
     // need another buffer
@@ -376,7 +444,7 @@ jv jv_parser_next(struct jv_parser* p) {
 
 jv jv_parse_sized(const char* string, int length) {
   struct jv_parser parser;
-  jv_parser_init(&parser);
+  parser_init(&parser);
   jv_parser_set_buf(&parser, string, length, 0);
   jv value = jv_parser_next(&parser);
   if (jv_is_valid(value)) {
@@ -401,7 +469,7 @@ jv jv_parse_sized(const char* string, int length) {
     jv_free(value);
     value = jv_invalid_with_msg(jv_string("Expected JSON value"));
   }
-  jv_parser_free(&parser);
+  parser_free(&parser);
 
   if (!jv_is_valid(value) && jv_invalid_has_msg(jv_copy(value))) {
     jv msg = jv_invalid_get_msg(value);

jv_parse.h

@@ -1,34 +0,0 @@
-#ifndef JV_PARSE_H
-#define JV_PARSE_H
-#include "jv_dtoa.h"
-struct jv_parser {
-  const char* curr_buf;
-  int curr_buf_length;
-  int curr_buf_pos;
-  int curr_buf_is_partial;
-
-  jv* stack;
-  int stackpos;
-  int stacklen;
-  jv next;
-  
-  char* tokenbuf;
-  int tokenpos;
-  int tokenlen;
-  
-  struct dtoa_context dtoa;
-
-  enum {
-    JV_PARSER_NORMAL,
-    JV_PARSER_STRING,
-    JV_PARSER_STRING_ESCAPE
-  } st;
-};
-
-void jv_parser_init(struct jv_parser* p);
-void jv_parser_free(struct jv_parser* p);
-
-void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial);
-
-jv jv_parser_next(struct jv_parser* p);
-#endif

jv_print.c

@@ -2,10 +2,25 @@
 #include <stdio.h>
 #include <float.h>
 #include <string.h>
+#include <assert.h>
 
 #include "jv_dtoa.h"
 #include "jv_unicode.h"
 
+#define ESC "\033"
+#define COL(c) (ESC "[" c "m")
+#define COLRESET (ESC "[0m")
+
+// Colour table. See http://en.wikipedia.org/wiki/ANSI_escape_code#Colors
+// for how to choose these.
+static const jv_kind colour_kinds[] = 
+  {JV_KIND_NULL,   JV_KIND_FALSE, JV_KIND_TRUE, JV_KIND_NUMBER,
+   JV_KIND_STRING, JV_KIND_ARRAY, JV_KIND_OBJECT};
+static const char* const colours[] =
+  {COL("30;1"),    COL("0"),      COL("0"),     COL("0"),
+   COL("32"),      COL("37"),     COL("37")};
+#define FIELD_COLOUR COL("34;1")
+
 static void put_buf(const char* s, int len, FILE* fout, jv* strout) {
   if (strout) {
     *strout = jv_string_append_buf(*strout, s, len);
@@ -31,10 +46,11 @@ static void put_space(int n, FILE* fout, jv* strout) {
 static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) {
   assert(jv_get_kind(str) == JV_KIND_STRING);
   const char* i = jv_string_value(str);
-  const char* end = i + jv_string_length(jv_copy(str));
+  const char* end = i + jv_string_length_bytes(jv_copy(str));
   const char* cstart;
   int c = 0;
   char buf[32];
+  put_char('"', F, S);
   while ((i = jvp_utf8_next((cstart = i), end, &c))) {
     assert(c != -1);
     int unicode_escape = 0;
@@ -91,13 +107,25 @@ static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) {
     }
   }
   assert(c != -1);
+  put_char('"', F, S);
 }
 
 enum { INDENT = 2 };
 
 static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FILE* F, jv* S) {
   char buf[JVP_DTOA_FMT_MAX_LEN];
+  const char* colour = 0;
+  if (flags & JV_PRINT_COLOUR) {
+    for (unsigned i=0; i<sizeof(colour_kinds)/sizeof(colour_kinds[0]); i++) {
+      if (jv_get_kind(x) == colour_kinds[i]) {
+        colour = colours[i];
+        put_str(colour, F, S);
+        break;
+      }
+    }
+  }
   switch (jv_get_kind(x)) {
+  default:
   case JV_KIND_INVALID:
     assert(0 && "Invalid value");
     break;
@@ -124,9 +152,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
     break;
   }
   case JV_KIND_STRING:
-    put_char('"', F, S);
     jvp_dump_string(x, flags & JV_PRINT_ASCII, F, S);
-    put_char('"', F, S);
     break;
   case JV_KIND_ARRAY: {
     if (jv_array_length(jv_copy(x)) == 0) {
@@ -138,21 +164,23 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       put_char('\n', F, S);
       put_space(indent+INDENT, F, S);
     }
-    for (int i=0; i<jv_array_length(jv_copy(x)); i++) {
+    jv_array_foreach(x, i, elem) {
       if (i!=0) {
         if (flags & JV_PRINT_PRETTY) {
           put_str(",\n", F, S);
           put_space(indent+INDENT, F, S);
         } else {
-          put_str(", ", F, S);
+          put_str(",", F, S);
         }
       }
-      jv_dump_term(C, jv_array_get(jv_copy(x), i), flags, indent + INDENT, F, S);
+      jv_dump_term(C, elem, flags, indent + INDENT, F, S);
+      if (colour) put_str(colour, F, S);
     }
     if (flags & JV_PRINT_PRETTY) {
       put_char('\n', F, S);
       put_space(indent, F, S);
     }
+    if (colour) put_str(colour, F, S);
     put_char(']', F, S);
     break;
   }
@@ -167,28 +195,69 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       put_space(indent+INDENT, F, S);
     }
     int first = 1;
-    jv_object_foreach(i, x) {
+    int i = 0;
+    jv keyset = jv_null();
+    while (1) {
+      jv key, value;
+      if (flags & JV_PRINT_SORTED) {
+        if (first) {
+          keyset = jv_keys(jv_copy(x));
+          i = 0;
+        } else {
+          i++;
+        }
+        if (i >= jv_array_length(jv_copy(keyset))) {
+          jv_free(keyset);
+          break;
+        }
+        key = jv_array_get(jv_copy(keyset), i);
+        value = jv_object_get(jv_copy(x), jv_copy(key));
+      } else {
+        if (first) {
+          i = jv_object_iter(x);
+        } else {
+          i = jv_object_iter_next(x, i);
+        }
+        if (!jv_object_iter_valid(x, i)) break;
+        key = jv_object_iter_key(x, i);
+        value = jv_object_iter_value(x, i);
+      }
+
       if (!first) {
         if (flags & JV_PRINT_PRETTY){
           put_str(",\n", F, S);
           put_space(indent+INDENT, F, S);
         } else {
-          put_str(", ", F, S);
+          put_str(",", F, S);
         }
       }
+      if (colour) put_str(COLRESET, F, S);
+
       first = 0;
-      jv_dump_term(C, jv_object_iter_key(x, i), flags, indent + INDENT, F, S);
-      put_str(": ", F, S);
-      jv_dump_term(C, jv_object_iter_value(x, i), flags, indent + INDENT, F, S);
+      if (colour) put_str(FIELD_COLOUR, F, S);
+      jvp_dump_string(key, flags & JV_PRINT_ASCII, F, S);
+      jv_free(key);
+      if (colour) put_str(COLRESET, F, S);
+
+      if (colour) put_str(colour, F, S);
+      put_str((flags & JV_PRINT_PRETTY) ? ": " : ":", F, S);
+      if (colour) put_str(COLRESET, F, S);
+      
+      jv_dump_term(C, value, flags, indent + INDENT, F, S);
+      if (colour) put_str(colour, F, S);
     }
     if (flags & JV_PRINT_PRETTY) {
       put_char('\n', F, S);
       put_space(indent, F, S);
     }
+    if (colour) put_str(colour, F, S);
     put_char('}', F, S);
   }
   }
   jv_free(x);
+  if (colour) {
+    put_str(COLRESET, F, S);
+  }
 }
 
 void jv_dump(jv x, int flags) {

jv_unicode.c

@@ -1,7 +1,7 @@
 #include <stdio.h>
 #include <assert.h>
 #include "jv_unicode.h"
-#include "jv_utf8_tables.gen.h"
+#include "jv_utf8_tables.h"
 
 const char* jvp_utf8_next(const char* in, const char* end, int* codepoint) {
   if (in == end) {

lexer.gen.h

@@ -1,362 +0,0 @@
-#ifndef jq_yyHEADER_H
-#define jq_yyHEADER_H 1
-#define jq_yyIN_HEADER 1
-
-#line 6 "lexer.gen.h"
-
-#line 8 "lexer.gen.h"
-
-#define  YY_INT_ALIGNED short int
-
-/* A lexical scanner generated by flex */
-
-#define FLEX_SCANNER
-#define YY_FLEX_MAJOR_VERSION 2
-#define YY_FLEX_MINOR_VERSION 5
-#define YY_FLEX_SUBMINOR_VERSION 35
-#if YY_FLEX_SUBMINOR_VERSION > 0
-#define FLEX_BETA
-#endif
-
-/* First, we deal with  platform-specific or compiler-specific issues. */
-
-/* begin standard C headers. */
-#include <stdio.h>
-#include <string.h>
-#include <errno.h>
-#include <stdlib.h>
-
-/* end standard C headers. */
-
-/* flex integer type definitions */
-
-#ifndef FLEXINT_H
-#define FLEXINT_H
-
-/* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */
-
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
-
-/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
- * if you want the limit (max/min) macros for int types. 
- */
-#ifndef __STDC_LIMIT_MACROS
-#define __STDC_LIMIT_MACROS 1
-#endif
-
-#include <inttypes.h>
-typedef int8_t flex_int8_t;
-typedef uint8_t flex_uint8_t;
-typedef int16_t flex_int16_t;
-typedef uint16_t flex_uint16_t;
-typedef int32_t flex_int32_t;
-typedef uint32_t flex_uint32_t;
-#else
-typedef signed char flex_int8_t;
-typedef short int flex_int16_t;
-typedef int flex_int32_t;
-typedef unsigned char flex_uint8_t; 
-typedef unsigned short int flex_uint16_t;
-typedef unsigned int flex_uint32_t;
-
-/* Limits of integral types. */
-#ifndef INT8_MIN
-#define INT8_MIN               (-128)
-#endif
-#ifndef INT16_MIN
-#define INT16_MIN              (-32767-1)
-#endif
-#ifndef INT32_MIN
-#define INT32_MIN              (-2147483647-1)
-#endif
-#ifndef INT8_MAX
-#define INT8_MAX               (127)
-#endif
-#ifndef INT16_MAX
-#define INT16_MAX              (32767)
-#endif
-#ifndef INT32_MAX
-#define INT32_MAX              (2147483647)
-#endif
-#ifndef UINT8_MAX
-#define UINT8_MAX              (255U)
-#endif
-#ifndef UINT16_MAX
-#define UINT16_MAX             (65535U)
-#endif
-#ifndef UINT32_MAX
-#define UINT32_MAX             (4294967295U)
-#endif
-
-#endif /* ! C99 */
-
-#endif /* ! FLEXINT_H */
-
-#ifdef __cplusplus
-
-/* The "const" storage-class-modifier is valid. */
-#define YY_USE_CONST
-
-#else	/* ! __cplusplus */
-
-/* C99 requires __STDC__ to be defined as 1. */
-#if defined (__STDC__)
-
-#define YY_USE_CONST
-
-#endif	/* defined (__STDC__) */
-#endif	/* ! __cplusplus */
-
-#ifdef YY_USE_CONST
-#define yyconst const
-#else
-#define yyconst
-#endif
-
-/* An opaque pointer. */
-#ifndef YY_TYPEDEF_YY_SCANNER_T
-#define YY_TYPEDEF_YY_SCANNER_T
-typedef void* yyscan_t;
-#endif
-
-/* For convenience, these vars (plus the bison vars far below)
-   are macros in the reentrant scanner. */
-#define yyin yyg->yyin_r
-#define yyout yyg->yyout_r
-#define yyextra yyg->yyextra_r
-#define yyleng yyg->yyleng_r
-#define yytext yyg->yytext_r
-#define yylineno (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno)
-#define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column)
-#define yy_flex_debug yyg->yy_flex_debug_r
-
-/* Size of default input buffer. */
-#ifndef YY_BUF_SIZE
-#ifdef __ia64__
-/* On IA-64, the buffer size is 16k, not 8k.
- * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case.
- * Ditto for the __ia64__ case accordingly.
- */
-#define YY_BUF_SIZE 32768
-#else
-#define YY_BUF_SIZE 16384
-#endif /* __ia64__ */
-#endif
-
-#ifndef YY_TYPEDEF_YY_BUFFER_STATE
-#define YY_TYPEDEF_YY_BUFFER_STATE
-typedef struct yy_buffer_state *YY_BUFFER_STATE;
-#endif
-
-#ifndef YY_TYPEDEF_YY_SIZE_T
-#define YY_TYPEDEF_YY_SIZE_T
-typedef size_t yy_size_t;
-#endif
-
-#ifndef YY_STRUCT_YY_BUFFER_STATE
-#define YY_STRUCT_YY_BUFFER_STATE
-struct yy_buffer_state
-	{
-	FILE *yy_input_file;
-
-	char *yy_ch_buf;		/* input buffer */
-	char *yy_buf_pos;		/* current position in input buffer */
-
-	/* Size of input buffer in bytes, not including room for EOB
-	 * characters.
-	 */
-	yy_size_t yy_buf_size;
-
-	/* Number of characters read into yy_ch_buf, not including EOB
-	 * characters.
-	 */
-	int yy_n_chars;
-
-	/* Whether we "own" the buffer - i.e., we know we created it,
-	 * and can realloc() it to grow it, and should free() it to
-	 * delete it.
-	 */
-	int yy_is_our_buffer;
-
-	/* Whether this is an "interactive" input source; if so, and
-	 * if we're using stdio for input, then we want to use getc()
-	 * instead of fread(), to make sure we stop fetching input after
-	 * each newline.
-	 */
-	int yy_is_interactive;
-
-	/* Whether we're considered to be at the beginning of a line.
-	 * If so, '^' rules will be active on the next match, otherwise
-	 * not.
-	 */
-	int yy_at_bol;
-
-    int yy_bs_lineno; /**< The line count. */
-    int yy_bs_column; /**< The column count. */
-    
-	/* Whether to try to fill the input buffer when we reach the
-	 * end of it.
-	 */
-	int yy_fill_buffer;
-
-	int yy_buffer_status;
-
-	};
-#endif /* !YY_STRUCT_YY_BUFFER_STATE */
-
-void jq_yyrestart (FILE *input_file ,yyscan_t yyscanner );
-void jq_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
-YY_BUFFER_STATE jq_yy_create_buffer (FILE *file,int size ,yyscan_t yyscanner );
-void jq_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
-void jq_yy_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner );
-void jq_yypush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner );
-void jq_yypop_buffer_state (yyscan_t yyscanner );
-
-YY_BUFFER_STATE jq_yy_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner );
-YY_BUFFER_STATE jq_yy_scan_string (yyconst char *yy_str ,yyscan_t yyscanner );
-YY_BUFFER_STATE jq_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner );
-
-void *jq_yyalloc (yy_size_t ,yyscan_t yyscanner );
-void *jq_yyrealloc (void *,yy_size_t ,yyscan_t yyscanner );
-void jq_yyfree (void * ,yyscan_t yyscanner );
-
-/* Begin user sect3 */
-
-#define jq_yywrap(n) 1
-#define YY_SKIP_YYWRAP
-
-#define yytext_ptr yytext_r
-
-#ifdef YY_HEADER_EXPORT_START_CONDITIONS
-#define INITIAL 0
-#define IN_PAREN 1
-#define IN_BRACKET 2
-#define IN_BRACE 3
-#define IN_QQINTERP 4
-#define IN_QQSTRING 5
-
-#endif
-
-#ifndef YY_NO_UNISTD_H
-/* Special case for "unistd.h", since it is non-ANSI. We include it way
- * down here because we want the user's section 1 to have been scanned first.
- * The user has a chance to override it with an option.
- */
-#include <unistd.h>
-#endif
-
-#define YY_EXTRA_TYPE int
-
-int jq_yylex_init (yyscan_t* scanner);
-
-int jq_yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner);
-
-/* Accessor methods to globals.
-   These are made visible to non-reentrant scanners for convenience. */
-
-int jq_yylex_destroy (yyscan_t yyscanner );
-
-int jq_yyget_debug (yyscan_t yyscanner );
-
-void jq_yyset_debug (int debug_flag ,yyscan_t yyscanner );
-
-YY_EXTRA_TYPE jq_yyget_extra (yyscan_t yyscanner );
-
-void jq_yyset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner );
-
-FILE *jq_yyget_in (yyscan_t yyscanner );
-
-void jq_yyset_in  (FILE * in_str ,yyscan_t yyscanner );
-
-FILE *jq_yyget_out (yyscan_t yyscanner );
-
-void jq_yyset_out  (FILE * out_str ,yyscan_t yyscanner );
-
-int jq_yyget_leng (yyscan_t yyscanner );
-
-char *jq_yyget_text (yyscan_t yyscanner );
-
-int jq_yyget_lineno (yyscan_t yyscanner );
-
-void jq_yyset_lineno (int line_number ,yyscan_t yyscanner );
-
-YYSTYPE * jq_yyget_lval (yyscan_t yyscanner );
-
-void jq_yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner );
-
-       YYLTYPE *jq_yyget_lloc (yyscan_t yyscanner );
-    
-        void jq_yyset_lloc (YYLTYPE * yylloc_param ,yyscan_t yyscanner );
-    
-/* Macros after this point can all be overridden by user definitions in
- * section 1.
- */
-
-#ifndef YY_SKIP_YYWRAP
-#ifdef __cplusplus
-extern "C" int jq_yywrap (yyscan_t yyscanner );
-#else
-extern int jq_yywrap (yyscan_t yyscanner );
-#endif
-#endif
-
-#ifndef yytext_ptr
-static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner);
-#endif
-
-#ifdef YY_NEED_STRLEN
-static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner);
-#endif
-
-#ifndef YY_NO_INPUT
-
-#endif
-
-/* Amount of stuff to slurp up with each read. */
-#ifndef YY_READ_BUF_SIZE
-#ifdef __ia64__
-/* On IA-64, the buffer size is 16k, not 8k */
-#define YY_READ_BUF_SIZE 16384
-#else
-#define YY_READ_BUF_SIZE 8192
-#endif /* __ia64__ */
-#endif
-
-/* Number of entries by which start-condition stack grows. */
-#ifndef YY_START_STACK_INCR
-#define YY_START_STACK_INCR 25
-#endif
-
-/* Default declaration of generated scanner - a define so the user can
- * easily add parameters.
- */
-#ifndef YY_DECL
-#define YY_DECL_IS_OURS 1
-
-extern int jq_yylex \
-               (YYSTYPE * yylval_param,YYLTYPE * yylloc_param ,yyscan_t yyscanner);
-
-#define YY_DECL int jq_yylex \
-               (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner)
-#endif /* !YY_DECL */
-
-/* yy_get_previous_state - get the state just before the EOB char was reached */
-
-#undef YY_NEW_FILE
-#undef YY_FLUSH_BUFFER
-#undef yy_set_bol
-#undef yy_new_buffer
-#undef yy_set_interactive
-#undef YY_DO_BEFORE_ACTION
-
-#ifdef YY_DECL_IS_OURS
-#undef YY_DECL_IS_OURS
-#undef YY_DECL
-#endif
-
-#line 104 "lexer.l"
-
-
-#line 361 "lexer.gen.h"
-#undef jq_yyIN_HEADER
-#endif /* jq_yyHEADER_H */

lexer.l

@@ -1,6 +1,11 @@
 %{
+#include <assert.h>
+#include "jv_alloc.h"
 #include "compile.h"
-#include "parser.gen.h"  /* Generated by bison. */
+
+struct lexer_param;
+
+#include "parser.h"  /* Generated by bison. */
 
 #define YY_USER_ACTION                           \
   do {                                           \
@@ -22,6 +27,7 @@
 %}
 
 %option noyywrap nounput noinput nodefault
+%option noyyalloc noyyrealloc noyyfree
 %option reentrant
 %option extra-type="int"
 %option bison-bridge bison-locations
@@ -31,6 +37,9 @@
 
 %%
 
+"#"[^\r\n]* { /* comments */ }
+
+"!=" { return NEQ; }
 "==" { return EQ; }
 "as" { return AS; }
 "def" { return DEF; }
@@ -41,6 +50,7 @@
 "and" { return AND; }
 "or" { return OR; }
 "end" { return END; }
+"reduce" { return REDUCE; }
 "//" { return DEFINEDOR; }
 "|=" { return SETPIPE; }
 "+=" { return SETPLUS; }
@@ -60,8 +70,11 @@
   return try_exit(yytext[0], YY_START, yyscanner);
 }
 
+"@"[a-zA-Z0-9_]+ {
+  yylval->literal = jv_string_sized(yytext + 1, yyleng - 1); return FORMAT;
+}
 
--?[0-9.]+([eE][+-]?[0-9]+)? { 
+[0-9.]+([eE][+-]?[0-9]+)? { 
    yylval->literal = jv_parse_sized(yytext, yyleng); return LITERAL; 
 }
 
@@ -81,7 +94,7 @@
   (\\[^u(]|\\u[a-zA-Z0-9]{0,4})+ {
     /* pass escapes to the json parser */
     jv escapes = jv_string_fmt("\"%.*s\"", yyleng, yytext);
-    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length(jv_copy(escapes)));
+    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length_bytes(jv_copy(escapes)));
     jv_free(escapes);
     return QQSTRING_TEXT;
   }
@@ -96,6 +109,7 @@
   
 
 [a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext); return IDENT;}
+\.[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext+1); return FIELD;}
 
 [ \n\t]+  {}
 
@@ -120,6 +134,10 @@ static int try_exit(int c, int state, yyscan_t yyscanner) {
     match = ')'; 
     ret = QQSTRING_INTERP_END;
     break;
+
+  default:
+    // may not be the best error to give
+    return INVALID_CHARACTER;
   }
   assert(match);
   if (match == c) {
@@ -143,3 +161,13 @@ static int enter(int c, int currstate, yyscan_t yyscanner) {
   yy_push_state(state, yyscanner);
   return c;
 }
+
+void* yyalloc(size_t sz, void* extra) {
+  return jv_mem_alloc(sz);
+}
+void* yyrealloc(void* p, size_t sz, void* extra) {
+  return jv_mem_realloc(p, sz);
+}
+void yyfree(void* p, void* extra) {
+  jv_mem_free(p);
+}

locfile.c

@@ -0,0 +1,65 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <stdarg.h>
+#include "jv_alloc.h"
+#include "locfile.h"
+
+
+void locfile_init(struct locfile* l, const char* data, int length) {
+  l->data = data;
+  l->length = length;
+  l->nlines = 1;
+  for (int i=0; i<length; i++) {
+    if (data[i] == '\n') l->nlines++;
+  }
+  l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
+  l->linemap[0] = 0;
+  int line = 1;
+  for (int i=0; i<length; i++) {
+    if (data[i] == '\n') {
+      l->linemap[line] = i+1;   // at start of line, not of \n
+      line++;
+    }
+  }
+  l->linemap[l->nlines] = length+1;   // virtual last \n
+}
+
+void locfile_free(struct locfile* l) {
+  jv_mem_free(l->linemap);
+}
+
+static int locfile_get_line(struct locfile* l, int pos) {
+  assert(pos < l->length);
+  int line = 1;
+  while (l->linemap[line] <= pos) line++;   // == if pos at start (before, never ==, because pos never on \n)
+  assert(line-1 < l->nlines);
+  return line-1;
+}
+
+static int locfile_line_length(struct locfile* l, int line) {
+  assert(line < l->nlines);
+  return l->linemap[line+1] - l->linemap[line] -1;   // -1 to omit \n
+}
+
+void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
+  va_list fmtargs;
+  va_start(fmtargs, fmt);
+  vfprintf(stderr, fmt, fmtargs);
+  va_end(fmtargs);
+  fprintf(stderr, "\n");
+  if (loc.start == -1) {
+    fprintf(stderr, "<unknown location>\n");
+    return;
+  }
+  int startline = locfile_get_line(l, loc.start);
+  int offset = l->linemap[startline];
+  fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset);
+  fprintf(stderr, "%*s", loc.start - offset, "");
+  for (int i = loc.start; 
+       i < loc.end && i < offset + locfile_line_length(l, startline);
+       i++){
+    fprintf(stderr, "^");
+  }
+  fprintf(stderr, "\n");
+}

locfile.h

@@ -1,9 +1,6 @@
 #ifndef _LOCFILE_H
 #define _LOCFILE_H
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include <stdarg.h>
+
 typedef struct {
   int start, end;
 } location;
@@ -17,62 +14,10 @@ struct locfile {
   int nlines;
 };
 
-static void locfile_init(struct locfile* l, const char* data, int length) {
-  l->data = data;
-  l->length = length;
-  l->nlines = 1;
-  for (int i=0; i<length; i++) {
-    if (data[i] == '\n') l->nlines++;
-  }
-  l->linemap = malloc(sizeof(int) * (l->nlines + 1));
-  l->linemap[0] = 0;
-  int line = 1;
-  for (int i=0; i<length; i++) {
-    if (data[i] == '\n') {
-      l->linemap[line] = i;
-      line++;
-    }
-  }
-  l->linemap[l->nlines] = length;
-}
+void locfile_init(struct locfile* l, const char* data, int length);
 
-static void locfile_free(struct locfile* l) {
-  free(l->linemap);
-}
+void locfile_free(struct locfile* l);
 
-static int locfile_get_line(struct locfile* l, int pos) {
-  assert(pos < l->length);
-  int line = 0;
-  while (l->linemap[line+1] < pos) line++;
-  assert(line < l->nlines);
-  return line;
-}
-
-static int locfile_line_length(struct locfile* l, int line) {
-  assert(line < l->nlines);
-  return l->linemap[line+1] - l->linemap[line];
-}
-
-static void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
-  va_list fmtargs;
-  va_start(fmtargs, fmt);
-  vfprintf(stderr, fmt, fmtargs);
-  va_end(fmtargs);
-  fprintf(stderr, "\n");
-  if (loc.start == -1) {
-    fprintf(stderr, "<unknown location>\n");
-    return;
-  }
-  int startline = locfile_get_line(l, loc.start);
-  int offset = l->linemap[startline];
-  fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset);
-  fprintf(stderr, "%*s", loc.start - offset, "");
-  for (int i = loc.start; 
-       i < loc.end && i < offset + locfile_line_length(l, startline);
-       i++){
-    fprintf(stderr, "^");
-  }
-  fprintf(stderr, "\n");
-}
+void locfile_locate(struct locfile* l, location loc, const char* fmt, ...);
 
 #endif

main.c

@@ -1,23 +1,26 @@
+#include <stdlib.h>
 #include <stdio.h>
+#include <errno.h>
 #include <string.h>
 #include <ctype.h>
+#include <unistd.h>
 #include "compile.h"
-#include "builtin.h"
 #include "jv.h"
-#include "jv_parse.h"
-#include "locfile.h"
-#include "parser.h"
-#include "execute.h"
-#include "version.gen.h"
+#include "jq.h"
+#include "jv_alloc.h"
+#include "version.h"
+
+int jq_testsuite(int argc, char* argv[]);
 
 static const char* progname;
 
 static void usage() {
   fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", JQ_VERSION);
-  fprintf(stderr, "Usage: %s [options] <jq filter>\n\n", progname);
+  fprintf(stderr, "Usage: %s [options] <jq filter> [file...]\n\n", progname);
   fprintf(stderr, "For a description of the command line options and\n");
   fprintf(stderr, "how to write jq filters (and why you might want to)\n");
-  fprintf(stderr, "see the jq documentation at http://stedolan.github.com/jq\n\n");
+  fprintf(stderr, "see the jq manpage, or the online documentation at\n");
+  fprintf(stderr, "http://stedolan.github.com/jq\n\n");
   exit(1);
 }
 
@@ -49,52 +52,166 @@ enum {
   RAW_OUTPUT = 8,
   COMPACT_OUTPUT = 16,
   ASCII_OUTPUT = 32,
+  COLOUR_OUTPUT = 64,
+  NO_COLOUR_OUTPUT = 128,
+  SORTED_OUTPUT = 256,
+
+  FROM_FILE = 512,
+
+  /* debugging only */
+  DUMP_DISASM = 2048,
 };
 static int options = 0;
-static struct bytecode* bc;
 
-static void process(jv value) {
-  jq_init(bc, value);
+static void process(jq_state *jq, jv value, int flags) {
+  jq_start(jq, value, flags);
   jv result;
-  while (jv_is_valid(result = jq_next())) {
+  while (jv_is_valid(result = jq_next(jq))) {
     if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
-      fwrite(jv_string_value(result), 1, jv_string_length(jv_copy(result)), stdout);
+      fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
+      jv_free(result);
     } else {
-      int dumpopts = 0;
+      int dumpopts;
+      /* Disable colour by default on Windows builds as Windows
+         terminals tend not to display it correctly */
+#ifdef WIN32
+      dumpopts = 0;
+#else
+      dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0;
+#endif
+      if (options & SORTED_OUTPUT) dumpopts |= JV_PRINT_SORTED;
       if (!(options & COMPACT_OUTPUT)) dumpopts |= JV_PRINT_PRETTY;
       if (options & ASCII_OUTPUT) dumpopts |= JV_PRINT_ASCII;
+      if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR;
+      if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR;
       jv_dump(result, dumpopts);
     }
     printf("\n");
   }
   jv_free(result);
-  jq_teardown();
+}
+
+FILE* current_input;
+const char** input_filenames;
+int ninput_files;
+int next_input_idx;
+static int read_more(char* buf, size_t size) {
+  while (!current_input || feof(current_input)) {
+    if (current_input) {
+      fclose(current_input);
+      current_input = 0;
+    }
+    if (next_input_idx == ninput_files) {
+      return 0;
+    }
+    if (!strcmp(input_filenames[next_input_idx], "-")) {
+      current_input = stdin;
+    } else {
+      current_input = fopen(input_filenames[next_input_idx], "r");
+    }
+    if (!current_input) {
+      fprintf(stderr, "%s: %s: %s\n", progname, input_filenames[next_input_idx], strerror(errno));
+    }
+    next_input_idx++;
+  }
+
+  if (!fgets(buf, size, current_input)) buf[0] = 0;
+  return 1;
 }
 
 int main(int argc, char* argv[]) {
+  jq_state *jq;
+  int ret = 0;
+  int compiled = 0;
+
   if (argc) progname = argv[0];
 
+  if (argc > 1 && !strcmp(argv[1], "--run-tests")) {
+    return jq_testsuite(argc, argv);
+  }
+
+  jq = jq_init();
+  if (jq == NULL) {
+    perror("malloc");
+    return 1;
+  }
+
   const char* program = 0;
+  input_filenames = jv_mem_alloc(sizeof(const char*) * argc);
+  ninput_files = 0;
+  int further_args_are_files = 0;
+  int jq_flags = 0;
+  jv program_arguments = jv_array();
   for (int i=1; i<argc; i++) {
-    if (!isoptish(argv[i])) {
-      if (program) usage();
-      program = argv[i];
+    if (further_args_are_files) {
+      input_filenames[ninput_files++] = argv[i];
+    } else if (!strcmp(argv[i], "--")) {
+      if (!program) usage();
+      further_args_are_files = 1;
+    } else if (!isoptish(argv[i])) {
+      if (program) {
+        input_filenames[ninput_files++] = argv[i];
+      } else {
+        program = argv[i];
+      }
     } else if (isoption(argv[i], 's', "slurp")) {
       options |= SLURP;
     } else if (isoption(argv[i], 'r', "raw-output")) {
       options |= RAW_OUTPUT;
     } else if (isoption(argv[i], 'c', "compact-output")) {
       options |= COMPACT_OUTPUT;
+    } else if (isoption(argv[i], 'C', "color-output")) {
+      options |= COLOUR_OUTPUT;
+    } else if (isoption(argv[i], 'M', "monochrome-output")) {
+      options |= NO_COLOUR_OUTPUT;
     } else if (isoption(argv[i], 'a', "ascii-output")) {
       options |= ASCII_OUTPUT;
+    } else if (isoption(argv[i], 'S', "sort-keys")) {
+      options |= SORTED_OUTPUT;
     } else if (isoption(argv[i], 'R', "raw-input")) {
       options |= RAW_INPUT;
     } else if (isoption(argv[i], 'n', "null-input")) {
       options |= PROVIDE_NULL;
+    } else if (isoption(argv[i], 'f', "from-file")) {
+      options |= FROM_FILE;
+    } else if (isoption(argv[i], 0, "arg")) {
+      if (i >= argc - 2) {
+        fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname);
+        die();
+      }
+      jv arg = jv_object();
+      arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
+      arg = jv_object_set(arg, jv_string("value"), jv_string(argv[i+2]));
+      program_arguments = jv_array_append(program_arguments, arg);
+      i += 2; // skip the next two arguments
+    } else if (isoption(argv[i], 0, "argfile")) {
+      if (i >= argc - 2) {
+        fprintf(stderr, "%s: --argfile takes two parameters (e.g. -a varname filename)\n", progname);
+        die();
+      }
+      jv arg = jv_object();
+      arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
+      jv data = jv_load_file(argv[i+2], 0);
+      if (!jv_is_valid(data)) {
+        data = jv_invalid_get_msg(data);
+        fprintf(stderr, "%s: Bad JSON in --argfile %s %s: %s\n", progname,
+                argv[i+1], argv[i+2], jv_string_value(data));
+        jv_free(data);
+        return 1;
+      }
+      if (jv_get_kind(data) == JV_KIND_ARRAY && jv_array_length(jv_copy(data)) == 1)
+          data = jv_array_get(data, 0);
+      arg = jv_object_set(arg, jv_string("value"), data);
+      program_arguments = jv_array_append(program_arguments, arg);
+      i += 2; // skip the next two arguments
+    } else if (isoption(argv[i],  0,  "debug-dump-disasm")) {
+      options |= DUMP_DISASM;
+    } else if (isoption(argv[i],  0,  "debug-trace")) {
+      jq_flags |= JQ_DEBUG_TRACE;
     } else if (isoption(argv[i], 'h', "help")) {
       usage();
     } else if (isoption(argv[i], 'V', "version")) {
-      fprintf(stderr, "jq version %s\n", JQ_VERSION);
+      printf("jq-%s\n", JQ_VERSION);
       return 0;
     } else {
       fprintf(stderr, "%s: Unknown option %s\n", progname, argv[i]);
@@ -102,74 +219,87 @@ int main(int argc, char* argv[]) {
     }
   }
   if (!program) usage();
+  if (ninput_files == 0) current_input = stdin;
 
   if ((options & PROVIDE_NULL) && (options & (RAW_INPUT | SLURP))) {
-    fprintf(stderr, "%s: --null-input cannot be used with --raw-input or --slurp\n", program);
+    fprintf(stderr, "%s: --null-input cannot be used with --raw-input or --slurp\n", progname);
     die();
   }
+  
+  if (options & FROM_FILE) {
+    jv data = jv_load_file(program, 1);
+    if (!jv_is_valid(data)) {
+      data = jv_invalid_get_msg(data);
+      fprintf(stderr, "%s: %s\n", progname, jv_string_value(data));
+      jv_free(data);
+      return 1;
+    }
+    compiled = jq_compile_args(jq, jv_string_value(data), program_arguments);
+    jv_free(data);
+  } else {
+    compiled = jq_compile_args(jq, program, program_arguments);
+  }
+  if (!compiled) return 1;
 
-  bc = jq_compile(program);
-  if (!bc) return 1;
-
-#if JQ_DEBUG
-  dump_disassembly(0, bc);
-  printf("\n");
-#endif
+  if (options & DUMP_DISASM) {
+    jq_dump_disassembly(jq, 0);
+    printf("\n");
+  }
 
   if (options & PROVIDE_NULL) {
-    process(jv_null());
+    process(jq, jv_null(), jq_flags);
   } else {
     jv slurped;
-    if (options & SLURP) slurped = jv_invalid();
-    int first = 1;
-    struct jv_parser parser;
-    jv_parser_init(&parser);
-    while (!feof(stdin)) {
-      char buf[4096];
-      if (!fgets(buf, sizeof(buf), stdin)) buf[0] = 0;
+    if (options & SLURP) {
+      if (options & RAW_INPUT) {
+        slurped = jv_string("");
+      } else {
+        slurped = jv_array();
+      }
+    }
+    struct jv_parser* parser = jv_parser_new();
+    char buf[4096];
+    while (read_more(buf, sizeof(buf))) {
       if (options & RAW_INPUT) {
         int len = strlen(buf);
         if (len > 0) {
           if (options & SLURP) {
-            if (first) slurped = jv_string(buf);
-            else slurped = jv_string_concat(slurped, jv_string(buf));
+            slurped = jv_string_concat(slurped, jv_string(buf));
           } else {
             if (buf[len-1] == '\n') buf[len-1] = 0;
-            process(jv_string(buf));
+            process(jq, jv_string(buf), jq_flags);
           }
         }
       } else {
-        jv_parser_set_buf(&parser, buf, strlen(buf), !feof(stdin));
+        jv_parser_set_buf(parser, buf, strlen(buf), !feof(stdin));
         jv value;
-        while (jv_is_valid((value = jv_parser_next(&parser)))) {
+        while (jv_is_valid((value = jv_parser_next(parser)))) {
           if (options & SLURP) {
-            if (first) slurped = jv_array();
             slurped = jv_array_append(slurped, value);
           } else {
-            process(value);
+            process(jq, value, jq_flags);
           }
         }
         if (jv_invalid_has_msg(jv_copy(value))) {
           jv msg = jv_invalid_get_msg(value);
           fprintf(stderr, "parse error: %s\n", jv_string_value(msg));
           jv_free(msg);
+          ret = 1;
           break;
         } else {
           jv_free(value);
         }
       }
-      first = 0;
     }
-    jv_parser_free(&parser);
+    jv_parser_free(parser);
+    if (ret != 0)
+      goto out;
     if (options & SLURP) {
-      if (jv_is_valid(slurped)) {
-        process(slurped);
-      } else {
-        jv_free(slurped);
-      }
+      process(jq, slurped, jq_flags);
     }
   }
-
-  bytecode_free(bc);
-  return 0;
+out:
+  jv_mem_free(input_filenames);
+  jq_teardown(&jq);
+  return ret;
 }

opcode.c

@@ -1,33 +0,0 @@
-#include "opcode.h"
-
-// flags, length
-#define NONE 0, 1
-#define CONSTANT OP_HAS_CONSTANT, 2
-#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
-#define BRANCH OP_HAS_BRANCH, 2
-#define CFUNC (OP_HAS_SYMBOL | OP_HAS_CFUNC | OP_HAS_VARIABLE_LENGTH_ARGLIST), 2
-#define UFUNC (OP_HAS_UFUNC | OP_HAS_VARIABLE_LENGTH_ARGLIST), 2
-#define CLOSURE_PARAM_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
-#define CLOSURE_CREATE_IMM (OP_HAS_BLOCK | OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
-#define CLOSURE_CREATE_C_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
-#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
-
-#define OP(name, imm, in, out) \
-  {name, #name, imm, in, out},
-
-static const struct opcode_description opcode_descriptions[] = {
-#include "opcode_list.h"
-};
-
-static const struct opcode_description invalid_opcode_description = {
-  -1, "#INVALID", 0, 0, 0, 0
-};
-
-
-const struct opcode_description* opcode_describe(opcode op) {
-  if ((int)op >= 0 && (int)op < NUM_OPCODES) {
-    return &opcode_descriptions[op];
-  } else {
-    return &invalid_opcode_description;
-  }
-}

opcode.h

@@ -1,48 +0,0 @@
-#ifndef OPCODE_H
-#define OPCODE_H
-#include <assert.h>
-
-typedef enum {
-#define OP(name, imm, in, out) name,
-#include "opcode_list.h"
-#undef OP
-} opcode;
-
-enum {
-  NUM_OPCODES = 
-#define OP(name, imm, in, out) +1
-#include "opcode_list.h"
-#undef OP
-};
-
-enum {
-  OP_HAS_CONSTANT = 2,
-  OP_HAS_VARIABLE = 4,
-  OP_HAS_BRANCH = 8,
-  OP_HAS_SYMBOL = 16,
-  OP_HAS_CFUNC = 32,
-  OP_HAS_UFUNC = 64,
-  OP_IS_CALL_PSEUDO = 128,
-  OP_HAS_VARIABLE_LENGTH_ARGLIST = 256,
-  OP_HAS_BLOCK = 512,
-  OP_HAS_BINDING = 1024,
-};
-struct opcode_description {
-  opcode op;
-  const char* name;
-
-  int flags;
-
-  // length in 16-bit units
-  int length;
-
-  int stack_in, stack_out;
-};
-
-const struct opcode_description* opcode_describe(opcode op);
-
-static inline int opcode_length(opcode op) {
-  return opcode_describe(op)->length;
-}
-
-#endif

opcode_list.h

@@ -1,30 +1,32 @@
 OP(LOADK, CONSTANT, 1, 1)
 OP(DUP,   NONE,     1, 2)
-OP(SWAP,  NONE,     2, 2)
+OP(DUP2,  NONE,     2, 3)
 OP(POP,   NONE,     1, 0)
 OP(LOADV, VARIABLE, 1, 1)
+OP(LOADVN, VARIABLE, 1, 1)
 OP(STOREV, VARIABLE, 1, 0)
 OP(INDEX, NONE,     2, 1)
-//OP(DISPLAY, NONE,   1, 0)
-OP(YIELD, NONE, 1, 0)
 OP(EACH,  NONE,     1, 1)
 OP(FORK,  BRANCH,   0, 0)
 OP(JUMP,  BRANCH,   0, 0)
 OP(JUMP_F,BRANCH,   1, 0)
 OP(BACKTRACK, NONE, 0, 0)
-OP(APPEND, NONE,    2, 1)
+OP(APPEND, VARIABLE,1, 0)
 OP(INSERT, NONE,    4, 2)
+OP(RANGE, VARIABLE, 1, 1)
 
-OP(ASSIGN, VARIABLE, 3, 0)
+OP(SUBEXP_BEGIN,  NONE,     1, 2)
+OP(SUBEXP_END,    NONE,     2, 2)
 
-OP(CALL_BUILTIN_1_1, CFUNC, 1, 1)
-OP(CALL_BUILTIN_2_1, CFUNC, 2, 1)
-OP(CALL_BUILTIN_3_1, CFUNC, 3, 1)
+OP(PATH_BEGIN, NONE, 1, 2)
+OP(PATH_END,   NONE, 2, 1)
 
-OP(CALL_1_1, UFUNC, 1, 1)
+OP(CALL_BUILTIN, CFUNC, -1, 1)
+
+OP(CALL_JQ, UFUNC, 1, 1)
 OP(RET, NONE, 1, 1)
 
-OP(CLOSURE_PARAM, CLOSURE_PARAM_IMM, 0, 0)
+OP(CLOSURE_PARAM, DEFINITION, 0, 0)
 OP(CLOSURE_REF, CLOSURE_REF_IMM, 0, 0)
-OP(CLOSURE_CREATE, CLOSURE_CREATE_IMM, 0, 0)
-OP(CLOSURE_CREATE_C, CLOSURE_CREATE_C_IMM, 0, 0)
+OP(CLOSURE_CREATE, DEFINITION, 0, 0)
+OP(CLOSURE_CREATE_C, DEFINITION, 0, 0)

parser.gen.h

@@ -1,130 +0,0 @@
-/* A Bison parser, made by GNU Bison 2.5.  */
-
-/* Bison interface for Yacc-like parsers in C
-   
-      Copyright (C) 1984, 1989-1990, 2000-2011 Free Software Foundation, Inc.
-   
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-   
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-   
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
-
-/* As a special exception, you may create a larger work that contains
-   part or all of the Bison parser skeleton and distribute that work
-   under terms of your choice, so long as that work isn't itself a
-   parser generator using the skeleton or a modified version thereof
-   as a parser skeleton.  Alternatively, if you modify or redistribute
-   the parser skeleton itself, you may (at your option) remove this
-   special exception, which will cause the skeleton and the resulting
-   Bison output files to be licensed under the GNU General Public
-   License without this special exception.
-   
-   This special exception was added by the Free Software Foundation in
-   version 2.2 of Bison.  */
-
-/* "%code requires" blocks.  */
-
-/* Line 2068 of yacc.c  */
-#line 6 "parser.y"
-
-#include "locfile.h"
-#define YYLTYPE location
-#define YYLLOC_DEFAULT(Loc, Rhs, N)             \
-  do {                                          \
-    if (N) {                                    \
-      (Loc).start = YYRHSLOC(Rhs, 1).start;     \
-      (Loc).end = YYRHSLOC(Rhs, N).end;         \
-    } else {                                    \
-      (Loc).start = YYRHSLOC(Rhs, 0).end;       \
-      (Loc).end = YYRHSLOC(Rhs, 0).end;         \
-    }                                           \
-  } while (0)
- 
-
-
-/* Line 2068 of yacc.c  */
-#line 55 "parser.gen.h"
-
-/* Tokens.  */
-#ifndef YYTOKENTYPE
-# define YYTOKENTYPE
-   /* Put the tokens into the symbol table, so that GDB and other debuggers
-      know about them.  */
-   enum yytokentype {
-     INVALID_CHARACTER = 258,
-     IDENT = 259,
-     LITERAL = 260,
-     EQ = 261,
-     DEFINEDOR = 262,
-     AS = 263,
-     DEF = 264,
-     IF = 265,
-     THEN = 266,
-     ELSE = 267,
-     ELSE_IF = 268,
-     END = 269,
-     AND = 270,
-     OR = 271,
-     SETPIPE = 272,
-     SETPLUS = 273,
-     SETMINUS = 274,
-     SETMULT = 275,
-     SETDIV = 276,
-     SETDEFINEDOR = 277,
-     LESSEQ = 278,
-     GREATEREQ = 279,
-     QQSTRING_START = 280,
-     QQSTRING_TEXT = 281,
-     QQSTRING_INTERP_START = 282,
-     QQSTRING_INTERP_END = 283,
-     QQSTRING_END = 284
-   };
-#endif
-
-
-
-#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
-typedef union YYSTYPE
-{
-
-/* Line 2068 of yacc.c  */
-#line 24 "parser.y"
-
-  jv literal;
-  block blk;
-
-
-
-/* Line 2068 of yacc.c  */
-#line 108 "parser.gen.h"
-} YYSTYPE;
-# define YYSTYPE_IS_TRIVIAL 1
-# define yystype YYSTYPE /* obsolescent; will be withdrawn */
-# define YYSTYPE_IS_DECLARED 1
-#endif
-
-
-
-#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED
-typedef struct YYLTYPE
-{
-  int first_line;
-  int first_column;
-  int last_line;
-  int last_column;
-} YYLTYPE;
-# define yyltype YYLTYPE /* obsolescent; will be withdrawn */
-# define YYLTYPE_IS_DECLARED 1
-# define YYLTYPE_IS_TRIVIAL 1
-#endif
-
-
-

parser.y

@@ -1,10 +1,16 @@
 %{
 #include <stdio.h>
 #include <string.h>
+#include <assert.h>
 #include "compile.h"
+#include "jv_alloc.h"
+#define YYMALLOC jv_mem_alloc
+#define YYFREE jv_mem_free
 %}
 %code requires {
 #include "locfile.h"
+struct lexer_param;
+
 #define YYLTYPE location
 #define YYLLOC_DEFAULT(Loc, Rhs, N)             \
   do {                                          \
@@ -32,17 +38,20 @@
 %parse-param {block* answer}
 %parse-param {int* errors}
 %parse-param {struct locfile* locations}
-%parse-param {yyscan_t lexer}
+%parse-param {struct lexer_param* lexer_param_ptr}
 %lex-param {block* answer}
 %lex-param {int* errors}
 %lex-param {struct locfile* locations}
-%lex-param {yyscan_t lexer}
+%lex-param {struct lexer_param* lexer_param_ptr}
 
 
 %token INVALID_CHARACTER
 %token <literal> IDENT
+%token <literal> FIELD
 %token <literal> LITERAL
+%token <literal> FORMAT
 %token EQ "=="
+%token NEQ "!="
 %token DEFINEDOR "//"
 %token AS "as"
 %token DEF "def"
@@ -50,6 +59,7 @@
 %token THEN "then"
 %token ELSE "else"
 %token ELSE_IF "elif"
+%token REDUCE "reduce"
 %token END "end"
 %token AND "and"
 %token OR "or"
@@ -70,65 +80,71 @@
 
  /* revolting hack */
 %left ';'
-%left '|'
+%right '|'
 %left ','
 %right "//"
 %nonassoc '=' SETPIPE SETPLUS SETMINUS SETMULT SETDIV SETDEFINEDOR
 %left OR
 %left AND
-%nonassoc EQ '<' '>' LESSEQ GREATEREQ
+%nonassoc NEQ EQ '<' '>' LESSEQ GREATEREQ
 %left '+' '-'
 %left '*' '/'
 
 
 %type <blk> Exp Term MkDict MkDictPair ExpD ElseBody QQString FuncDef FuncDefs String
 %{
-#include "lexer.gen.h"
-#define FAIL(loc, msg)                                   \
-  do {                                                   \
-    location l = loc;                                    \
-    yyerror(&l, answer, errors, locations, lexer, msg);  \
-    /*YYERROR*/;                                         \
+#include "lexer.h"
+struct lexer_param {
+  yyscan_t lexer;
+};
+#define FAIL(loc, msg)                                             \
+  do {                                                             \
+    location l = loc;                                              \
+    yyerror(&l, answer, errors, locations, lexer_param_ptr, msg);  \
+    /*YYERROR*/;                                                   \
   } while (0)
 
 void yyerror(YYLTYPE* loc, block* answer, int* errors, 
-             struct locfile* locations, yyscan_t lexer, const char *s){
+             struct locfile* locations, struct lexer_param* lexer_param_ptr, const char *s){
   (*errors)++;
   locfile_locate(locations, *loc, "error: %s", s);
 }
 
 int yylex(YYSTYPE* yylval, YYLTYPE* yylloc, block* answer, int* errors, 
-          struct locfile* locations, yyscan_t lexer) {
-  while (1) {
-    int tok = jq_yylex(yylval, yylloc, lexer);
-    if (tok == INVALID_CHARACTER) {
-      FAIL(*yylloc, "Invalid character");
+          struct locfile* locations, struct lexer_param* lexer_param_ptr) {
+  yyscan_t lexer = lexer_param_ptr->lexer;
+  int tok = jq_yylex(yylval, yylloc, lexer);
+  if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
+    jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
+    if (jv_get_kind(msg) == JV_KIND_STRING) {
+      FAIL(*yylloc, jv_string_value(msg));
     } else {
-      if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
-        jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
-        if (jv_get_kind(msg) == JV_KIND_STRING) {
-          FAIL(*yylloc, jv_string_value(msg));
-        } else {
-          FAIL(*yylloc, "Invalid literal");
-        }
-        jv_free(msg);
-        jv_free(yylval->literal);
-        yylval->literal = jv_null();
-      }
-      return tok;
+      FAIL(*yylloc, "Invalid literal");
     }
+    jv_free(msg);
+    jv_free(yylval->literal);
+    yylval->literal = jv_null();
   }
+  return tok;
 }
 
 static block gen_dictpair(block k, block v) {
-  block b = gen_subexp(k);
-  block_append(&b, gen_subexp(v));
-  block_append(&b, gen_op_simple(INSERT));
-  return b;
+  return BLOCK(gen_subexp(k), gen_subexp(v), gen_op_simple(INSERT));
 }
 
 static block gen_index(block obj, block key) {
-  return block_join(obj, block_join(gen_subexp(key), gen_op_simple(INDEX)));
+  return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX));
+}
+
+static block gen_slice_index(block obj, block start, block end) {
+  block key = BLOCK(gen_subexp(gen_const(jv_object())),
+                    gen_subexp(gen_const(jv_string("start"))),
+                    gen_subexp(start),
+                    gen_op_simple(INSERT),
+                    gen_subexp(gen_const(jv_string("end"))),
+                    gen_subexp(end),
+                    gen_op_simple(INSERT));
+  return BLOCK(key, obj, gen_op_simple(INDEX));
 }
 
 static block gen_binop(block a, block b, int op) {
@@ -139,6 +155,7 @@ static block gen_binop(block a, block b, int op) {
   case '*': funcname = "_multiply"; break;
   case '/': funcname = "_divide"; break;
   case EQ: funcname = "_equal"; break;
+  case NEQ: funcname = "_notequal"; break;
   case '<': funcname = "_less"; break;
   case '>': funcname = "_greater"; break;
   case LESSEQ: funcname = "_lesseq"; break;
@@ -146,25 +163,31 @@ static block gen_binop(block a, block b, int op) {
   }
   assert(funcname);
 
-  block c = gen_noop();
-  block_append(&c, gen_subexp(a));
-  block_append(&c, gen_subexp(b));
-  block_append(&c, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, funcname)));
-  return c;
+  return gen_call(funcname, BLOCK(gen_lambda(a), gen_lambda(b)));
 }
 
-static block gen_format(block a) {
-  return block_join(a, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, "tostring")));
+static block gen_format(block a, jv fmt) {
+  return BLOCK(a, gen_call("format", BLOCK(gen_lambda(gen_const(fmt)))));
+}
+
+static block gen_definedor_assign(block object, block val) {
+  block tmp = gen_op_var_fresh(STOREV, "tmp");
+  return BLOCK(gen_op_simple(DUP),
+               val, tmp,
+               gen_call("_modify", BLOCK(gen_lambda(object),
+                                         gen_lambda(gen_definedor(gen_noop(), 
+                                                                  gen_op_bound(LOADV, tmp))))));
 }
  
-static block gen_update(block a, block op, int optype) {
-  block assign = a;
-  block_append(&assign, gen_op_simple(DUP));
-  if (optype) {
-    op = gen_binop(gen_noop(), op, optype);
-  }
-  block_append(&assign, op);
-  return gen_assign(assign);
+static block gen_update(block object, block val, int optype) {
+  block tmp = gen_op_var_fresh(STOREV, "tmp");
+  return BLOCK(gen_op_simple(DUP),
+               val,
+               tmp,
+               gen_call("_modify", BLOCK(gen_lambda(object), 
+                                         gen_lambda(gen_binop(gen_noop(),
+                                                              gen_op_bound(LOADV, tmp),
+                                                              optype)))));
 }
 
 %}
@@ -192,27 +215,25 @@ FuncDef Exp %prec ';' {
 } |
 
 Term "as" '$' IDENT '|' Exp {
-  $$ = gen_op_simple(DUP);
-  block_append(&$$, $1);
-  block_append(&$$, block_bind(gen_op_var_unbound(STOREV, jv_string_value($4)), $6, OP_HAS_VARIABLE));
+  $$ = gen_var_binding($1, jv_string_value($4), $6);
   jv_free($4);
 } |
 
+"reduce" Term "as" '$' IDENT '(' Exp ';' Exp ')' {
+  $$ = gen_reduce(jv_string_value($5), $2, $7, $9);
+  jv_free($5);
+} |
+
 "if" Exp "then" Exp ElseBody {
   $$ = gen_cond($2, $4, $5);
 } |
-"if" Exp error {
+"if" Exp "then" error {
   FAIL(@$, "Possibly unterminated 'if' statment");
   $$ = $2;
 } |
 
 Exp '=' Exp {
-  block assign = gen_op_simple(DUP);
-  block_append(&assign, $3);
-  block_append(&assign, gen_op_simple(SWAP));
-  block_append(&assign, $1);
-  block_append(&assign, gen_op_simple(SWAP));
-  $$ = gen_assign(assign);
+  $$ = gen_call("_assign", BLOCK(gen_lambda($1), gen_lambda($3)));
 } |
 
 Exp "or" Exp {
@@ -228,11 +249,11 @@ Exp "//" Exp {
 } |
 
 Exp "//=" Exp {
-  $$ = gen_update($1, gen_definedor(gen_noop(), $3), 0);
+  $$ = gen_definedor_assign($1, $3);
 } |
 
 Exp "|=" Exp {
-  $$ = gen_update($1, $3, 0);
+  $$ = gen_call("_modify", BLOCK(gen_lambda($1), gen_lambda($3)));
 } |
 
 Exp '|' Exp { 
@@ -251,6 +272,10 @@ Exp "+=" Exp {
   $$ = gen_update($1, $3, '+');
 } |
 
+'-' Exp {
+  $$ = BLOCK($2, gen_call("_negate", gen_noop()));
+} |
+
 Exp '-' Exp {
   $$ = gen_binop($1, $3, '-');
 } |
@@ -279,6 +304,10 @@ Exp "==" Exp {
   $$ = gen_binop($1, $3, EQ);
 } |
 
+Exp "!=" Exp {
+  $$ = gen_binop($1, $3, NEQ);
+} |
+
 Exp '<' Exp {
   $$ = gen_binop($1, $3, '<');
 } |
@@ -295,42 +324,56 @@ Exp ">=" Exp {
   $$ = gen_binop($1, $3, GREATEREQ);
 } |
 
-String {
-  $$ = $1;
-} |
-
 Term { 
   $$ = $1; 
 }
 
-String:
-QQSTRING_START QQString QQSTRING_END {
-  $$ = $2;
-}
-
 FuncDef:
 "def" IDENT ':' Exp ';' {
-  block body = block_join($4, gen_op_simple(RET));
-  $$ = gen_op_block_defn_rec(CLOSURE_CREATE, jv_string_value($2), body);
+  $$ = gen_function(jv_string_value($2), gen_noop(), $4);
   jv_free($2);
 } |
 
 "def" IDENT '(' IDENT ')' ':' Exp ';' {
-  block body = block_bind(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), block_join($7, gen_op_simple(RET)), OP_IS_CALL_PSEUDO);
-  $$ = gen_op_block_defn_rec(CLOSURE_CREATE, jv_string_value($2), body);
+  $$ = gen_function(jv_string_value($2), 
+                    gen_param(jv_string_value($4)), 
+                    $7);
   jv_free($2);
   jv_free($4);
+} |
+
+"def" IDENT '(' IDENT ';' IDENT ')' ':' Exp ';' {
+  $$ = gen_function(jv_string_value($2), 
+                    BLOCK(gen_param(jv_string_value($4)), 
+                          gen_param(jv_string_value($6))),
+                    $9);
+  jv_free($2);
+  jv_free($4);
+  jv_free($6);
 }
 
+
+
+String:
+QQSTRING_START { $<literal>$ = jv_string("text"); } QQString QQSTRING_END {
+  $$ = $3;
+  jv_free($<literal>2);
+} |
+FORMAT QQSTRING_START { $<literal>$ = $1; } QQString QQSTRING_END {
+  $$ = $4;
+  jv_free($<literal>3);
+}
+
+
 QQString:
 /* empty */ {
-  $$ = gen_op_const(LOADK, jv_string(""));
+  $$ = gen_const(jv_string(""));
 } |
 QQString QQSTRING_TEXT {
-  $$ = gen_binop($1, gen_op_const(LOADK, $2), '+');
+  $$ = gen_binop($1, gen_const($2), '+');
 } |
 QQString QQSTRING_INTERP_START Exp QQSTRING_INTERP_END {
-  $$ = gen_binop($1, gen_format($3), '+');
+  $$ = gen_binop($1, gen_format($3, jv_copy($<literal>0)), '+');
 }
 
 
@@ -356,12 +399,27 @@ Term:
 '.' {
   $$ = gen_noop(); 
 } |
-Term '.' IDENT {
-  $$ = gen_index($1, gen_op_const(LOADK, $3)); 
+Term FIELD {
+  $$ = gen_index($1, gen_const($2));
 } |
-'.' IDENT { 
-  $$ = gen_index(gen_noop(), gen_op_const(LOADK, $2)); 
+FIELD { 
+  $$ = gen_index(gen_noop(), gen_const($1)); 
 } |
+Term '.' String {
+  $$ = gen_index($1, $3);
+} |
+'.' String {
+  $$ = gen_index(gen_noop(), $2);
+} |
+'.' error {
+  FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
+  $$ = gen_noop();
+} |
+'.' IDENT error {
+  jv_free($2);
+  FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
+  $$ = gen_noop();
+} | 
 /* FIXME: string literals */
 Term '[' Exp ']' {
   $$ = gen_index($1, $3); 
@@ -369,8 +427,23 @@ Term '[' Exp ']' {
 Term '[' ']' {
   $$ = block_join($1, gen_op_simple(EACH)); 
 } |
+Term '[' Exp ':' Exp ']' {
+  $$ = gen_slice_index($1, $3, $5);
+} |
+Term '[' Exp ':' ']' {
+  $$ = gen_slice_index($1, $3, gen_const(jv_null()));
+} |
+Term '[' ':' Exp ']' {
+  $$ = gen_slice_index($1, gen_const(jv_null()), $4);
+} |
 LITERAL {
-  $$ = gen_op_const(LOADK, $1); 
+  $$ = gen_const($1); 
+} |
+String {
+  $$ = $1;
+} |
+FORMAT {
+  $$ = gen_format(gen_noop(), $1);
 } |
 '(' Exp ')' { 
   $$ = $2; 
@@ -379,28 +452,26 @@ LITERAL {
   $$ = gen_collect($2); 
 } |
 '[' ']' { 
-  $$ = gen_op_const(LOADK, jv_array()); 
+  $$ = gen_const(jv_array()); 
 } |
 '{' MkDict '}' { 
-  $$ = gen_subexp(gen_op_const(LOADK, jv_object()));
-  block_append(&$$, $2);
-  block_append(&$$, gen_op_simple(POP));
+  $$ = BLOCK(gen_subexp(gen_const(jv_object())), $2, gen_op_simple(POP));
 } |
 '$' IDENT {
-  $$ = gen_location(@$, gen_op_var_unbound(LOADV, jv_string_value($2)));
+  $$ = gen_location(@$, gen_op_unbound(LOADV, jv_string_value($2)));
   jv_free($2);
 } | 
 IDENT {
-  $$ = gen_location(@$, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, jv_string_value($1))));
+  $$ = gen_location(@$, gen_call(jv_string_value($1), gen_noop()));
   jv_free($1);
 } |
 IDENT '(' Exp ')' {
-  $$ = gen_op_call(CALL_1_1, 
-                   block_join(gen_op_block_unbound(CLOSURE_REF, jv_string_value($1)),
-                              block_bind(gen_op_block_defn(CLOSURE_CREATE,
-                                                "lambda",
-                                                           block_join($3, gen_op_simple(RET))),
-                                         gen_noop(), OP_IS_CALL_PSEUDO)));
+  $$ = gen_call(jv_string_value($1), gen_lambda($3));
+  $$ = gen_location(@1, $$);
+  jv_free($1);
+} |
+IDENT '(' Exp ';' Exp ')' {
+  $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5)));
   $$ = gen_location(@1, $$);
   jv_free($1);
 } |
@@ -419,14 +490,18 @@ MkDict:
 
 MkDictPair
 : IDENT ':' ExpD { 
-  $$ = gen_dictpair(gen_op_const(LOADK, $1), $3);
+  $$ = gen_dictpair(gen_const($1), $3);
  }
 | String ':' ExpD {
   $$ = gen_dictpair($1, $3);
   }
+| String {
+  $$ = gen_dictpair($1, BLOCK(gen_op_simple(POP), gen_op_simple(DUP2),
+                              gen_op_simple(DUP2), gen_op_simple(INDEX)));
+  }
 | IDENT {
-  $$ = gen_dictpair(gen_op_const(LOADK, jv_copy($1)),
-                    gen_index(gen_noop(), gen_op_const(LOADK, $1)));
+  $$ = gen_dictpair(gen_const(jv_copy($1)),
+                    gen_index(gen_noop(), gen_const($1)));
   }
 | '(' Exp ')' ':' ExpD {
   $$ = gen_dictpair($2, $5);
@@ -435,15 +510,15 @@ MkDictPair
 %%
 
 int jq_parse(struct locfile* locations, block* answer) {
-  yyscan_t scanner;
+  struct lexer_param scanner;
   YY_BUFFER_STATE buf;
-  jq_yylex_init_extra(0, &scanner);
-  buf = jq_yy_scan_bytes(locations->data, locations->length, scanner);
+  jq_yylex_init_extra(0, &scanner.lexer);
+  buf = jq_yy_scan_bytes(locations->data, locations->length, scanner.lexer);
   int errors = 0;
   *answer = gen_noop();
-  yyparse(answer, &errors, locations, scanner);
-  jq_yy_delete_buffer(buf, scanner);
-  jq_yylex_destroy(scanner);
+  yyparse(answer, &errors, locations, &scanner);
+  jq_yy_delete_buffer(buf, scanner.lexer);
+  jq_yylex_destroy(scanner.lexer);
   if (errors > 0) {
     block_free(*answer);
     *answer = gen_noop();

scripts/crosscompile

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# This script is used to cross-compile binaries for 
+# platforms other than the current one
+
+# Usage: $0 <platformname> <configure options>
+# <platformname> is arbitrary, it is the name
+# of the directory which will be created to contain
+# the output binaries.
+
+# e.g. $0 win32 --host=i686-w64-mingw32
+
+set -e
+cd `dirname "$0"`/../build
+
+plat="$1"
+[ -z "$plat" ] && exit 1
+shift
+
+[ -d "$plat" ] || mkdir "$plat"
+rm -rf "$plat/tmp"
+mkdir "$plat/tmp"
+cd "$plat/tmp"
+mkdir install_other
+../../../configure \
+    --prefix="`pwd`/install_other" \
+    --bindir="`pwd`/.." \
+    "$@"
+make install-binaries
+cd ..
+rm -rf tmp

scripts/gen_utf8_tables.py

@@ -1,16 +1,18 @@
 #!/usr/bin/python
 
+# This program was used to generate jv_utf8_tables.gen.h
+
 mask = lambda n: (1 << n) - 1
 
 
 def print_table(type, name, t):
     assert len(t) == 256
-    print "static const",type, name+"[]", "="
+    print("static const %s %s[] =" % (type, name))
     first = True
     for i in range(0,len(t),16):
-        print (" {" if i == 0 else "  ") +\
-            ", ".join("0x%02x"%n for n in t[i:i+16]) + \
-            ("," if i + 16 < 256 else "};")
+        print ((" {" if i == 0 else "  ") +
+               ", ".join("0x%02x"%n for n in t[i:i+16]) + 
+               ("," if i + 16 < 256 else "};"))
 
 
 def utf8info(c):
@@ -24,7 +26,7 @@ def utf8info(c):
         
 table = lambda i: [utf8info(c)[i] for c in range(256)]
 
-print "#define UTF8_CONTINUATION_BYTE ((unsigned char)255)"
+print("#define UTF8_CONTINUATION_BYTE ((unsigned char)255)")
 
 print_table("unsigned char", "utf8_coding_length", table(0))
 print_table("unsigned char", "utf8_coding_bits", table(1))

scripts/version

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+cd `dirname "$0"`
+git rev-parse --verify -q jq-1.0 > /dev/null 2>&1
+git describe --always --match 'jq-*' --dirty | sed 's/^jq-//'

setup.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# RUN ME after pulling the code from git!
+#
+# This script is handy after pulling the code out of git without having
+# to remember all the typing needed to build up and tear down all the
+# crap that autotools puts in your source repo.  The intent is to keep
+# all generated files out of git.
+
+if [ "clean" == "$1" ]; then
+  make distclean || echo "continuing..."
+  rm -rf tmp autom4te.cache config
+  rm -f INSTALL Makefile.in aclocal.m4 configure config.h.in ChangeLog
+  rm -f jv_utf8_tables.gen.h lexer.c lexer.h parser.c parser.h
+elif [ "superclean" == "$1" ]; then
+  # if autoconf errors during distcheck, it leaves files that need chmod'ing
+  ver=$(tr -d '\n' <VERSION)
+  if [ "x${ver}" != "x" ]; then
+    if [ -d jq-${ver} ]; then
+      chmod -R u+w jq-${ver}
+      rm -rf jq-${ver}
+    fi
+    rm -f jq-${ver}*
+  fi
+else
+  autoreconf --install
+  ./configure --prefix=/opt/junk
+  make check
+  [ -d tmp ] && mv tmp tmp-
+  mkdir tmp
+  make DESTDIR=./tmp install
+  make distcheck
+fi

tests/all.test

@@ -21,6 +21,11 @@ null
 null
 1
 
+
+-1
+null
+-1
+
 # FIXME: much more number testing needed
 
 {}
@@ -31,6 +36,13 @@ null
 null
 []
 
+# The input line starts with a 0xFEFF (byte order mark) codepoint
+# No, there is no reason to have a byte order mark in UTF8 text.
+# But apparently people do, so jq shouldn't break on it.
+.
+"byte order mark"
+"byte order mark"
+
 # We test escapes by matching them against Unicode codepoints
 # FIXME: more tests needed for weird unicode stuff (e.g. utf16 pairs)
 "Aa\r\n\t\b\f\u03bc"
@@ -45,6 +57,24 @@ null
 null
 "interpolation"
 
+@text,@json,([1,.] | @csv),@html,@uri,@sh,@base64
+"<>&'\""
+"<>&'\""
+"\"<>&'\\\"\""
+"1,\"<>&'\"\"\""
+"&lt;&gt;&amp;&apos;&quot;"
+"%3c%3e%26'%22"
+"'<>&'\\''\"'"
+"PD4mJyI="
+
+@uri
+"\u03bc"
+"%ce%bc"
+
+@html "<b>\(.)</b>"
+"<script>hax</script>"
+"<b>&lt;script&gt;hax&lt;/script&gt;</b>"
+
 #
 # Dictionary construction syntax
 #
@@ -57,6 +87,10 @@ null
 {"a":1, "b":2, "c":3, "d":"c"}
 {"a":1, "b":2, "c":1, "e":2}
 
+{"a",b,"a$\(1+1)"}
+{"a":1, "b":2, "c":3, "a$2":4}
+{"a":1, "b":2, "a$2":4}
+
 #
 # Field access, piping
 #
@@ -81,6 +115,10 @@ null
 {"foo": {"bar": 42}, "bar": "badvalue"}
 42
 
+."foo"."bar"
+{"foo": {"bar": 20}}
+20
+
 
 #
 # Multiple outputs, iteration
@@ -97,6 +135,11 @@ null
 1
 1
 
+1,.
+[]
+1
+[]
+
 [.]
 [2]
 [[2]]
@@ -127,6 +170,25 @@ null
 2
 3
 
+#
+# Slices
+#
+
+[.[3:2], .[-5:4], .[:-2], .[-2:], .[3:3][1:], .[10:]]
+[0,1,2,3,4,5,6]
+[[], [2,3], [0,1,2,3,4], [5,6], [], []]
+
+del(.[2:4],.[0],.[-2:])
+[0,1,2,3,4,5,6,7]
+[1,4,5]
+
+.[2:4] = ([], ["a","b"], ["a","b","c"])
+[0,1,2,3,4,5,6,7]
+[0,1,4,5,6,7]
+[0,1,"a","b",4,5,6,7]
+[0,1,"a","b","c",4,5,6,7]
+
+
 #
 # Variables
 #
@@ -141,6 +203,10 @@ null
 [6]
 [7]
 
+42 as $x | . | . | . + 432 | $x + 1
+34324
+43
+
 1 as $x | [$x,$x,$x as $x | $x]
 null
 [1,1,1]
@@ -159,6 +225,14 @@ null
 "wtasdf"
 2.0
 
+2-1
+null
+1
+
+2-(-1)
+null
+3
+
 1e+0+0.001e3
 "I wonder what this will be?"
 20e-1
@@ -167,6 +241,18 @@ null
 15
 19.0
 
+.+null
+{"a":42}
+{"a":42}
+
+null+.
+null
+null
+
+.a+.b
+{"a":42}
+42
+
 [1,2,3] + [.]
 null
 [1,2,3,null]
@@ -220,8 +306,8 @@ null
 [false, false, false, false, false, false, false, false, true ]
 
 [.[] | length]
-[[], {}, [1,2], {"a":42}, "asdf"]
-[0, 0, 2, 1, 4]
+[[], {}, [1,2], {"a":42}, "asdf", "\u03bc"]
+[0, 0, 2, 1, 4, 1]
 
 map(keys)
 [{}, {"abcd":1,"abc":2,"abcde":3}, {"x":1, "z": 3, "y":2}]
@@ -291,6 +377,65 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
 [1,2,3,4]
 [1,2,6,24]
 
+# test stack overflow and reallocation
+# this test is disabled for now, it takes a realllllly long time.
+# def f: if length > 1000 then . else .+[1]|f end; f | length
+# []
+# 1001
+
+reduce .[] as $x (0; . + $x)
+[1,2,4]
+7
+
+#
+# Paths
+#
+
+path(.foo[0,1])
+null
+["foo", 0]
+["foo", 1]
+
+path(.[] | select(.>3))
+[1,5,3]
+[1]
+
+path(.)
+42
+[]
+
+["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
+{"bar": 42, "foo": ["a", "b", "c", "d"]}
+"b"
+{"bar": 42, "foo": ["a", 20, "c", "d"]}
+{"bar": 42, "foo": ["a", "c", "d"]}
+
+map(getpath([2])), map(setpath([2]; 42)), map(delpaths([[2]]))
+[[0], [0,1], [0,1,2]]
+[null, null, 2]
+[[0,null,42], [0,1,42], [0,1,42]]
+[[0], [0,1], [0,1]]
+
+map(delpaths([[0,"foo"]]))
+[[{"foo":2, "x":1}], [{"bar":2}]]
+[[{"x":1}], [{"bar":2}]]
+
+["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
+{"bar":false}
+null
+{"bar":false, "foo": [null, 20]}
+{"bar":false}
+
+delpaths([[-200]])
+[1,2,3]
+[1,2,3]
+
+del(.), del(empty), del((.foo,.bar,.baz) | .[2,3,0]), del(.foo[0], .bar[0], .foo, .baz.bar[0].x)
+{"foo": [0,1,2,3,4], "bar": [0,1]}
+null
+{"foo": [0,1,2,3,4], "bar": [0,1]}
+{"foo": [1,4], "bar": [1]}
+{"bar": [1]}
 
 #
 # Assignment
@@ -314,6 +459,10 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
 [-1,1,3]
 [0.5, 1.5, 2.5]
 
+.foo += .foo
+{"foo":2}
+{"foo":4}
+
 .[0].a |= {"old":., "new":(.+1)}
 [{"a":1,"b":2}]
 [{"a":{"old":1, "new":2},"b":2}]
@@ -349,10 +498,9 @@ def inc(x): x |= .+1; inc(.[].a)
 [{"foo":[1,2], "bar": 42}, {"foo":[1], "bar": null}, {"foo":[null,false,3], "bar": 18}, {"foo":[], "bar":42}, {"foo": [null,false,null], "bar": 41}]
 [[1,2], [1], [3], [42], [41]]
 
-# FIXME: behaviour of update operators
-# .[] //= .[0]
-# ["hello",true,false,[false],null]
-# ["hello",true,"hello",[false],"hello"]
+.[] //= .[0]
+["hello",true,false,[false],null]
+["hello",true,"hello",[false],"hello"]
 
 .[] | [.[0] and .[1], .[0] or .[1]]
 [[true,[]], [false,1], [42,null], [null,false]]
@@ -372,4 +520,87 @@ def inc(x): x |= .+1; inc(.[].a)
 
 [10 >= 0, 10 >= 10, 10 >= 20, 10 <= 0, 10 <= 10, 10 <= 20]
 {}
-[true,true,false,false,true,true]
+[true,true,false,false,true,true]
+
+# And some in/equality tests
+[ 10 == 10, 10 != 10, 10 != 11, 10 == 11]
+{}
+[true,false,true,false]
+
+["hello" == "hello", "hello" != "hello", "hello" == "world", "hello" != "world" ]
+{}
+[true,false,false,true]
+
+[[1,2,3] == [1,2,3], [1,2,3] != [1,2,3], [1,2,3] == [4,5,6], [1,2,3] != [4,5,6]]
+{}
+[true,false,false,true]
+
+[{"foo":42} == {"foo":42},{"foo":42} != {"foo":42}, {"foo":42} != {"bar":42}, {"foo":42} == {"bar":42}]
+{}
+[true,false,true,false]
+
+# ugly complicated thing
+[{"foo":[1,2,{"bar":18},"world"]} == {"foo":[1,2,{"bar":18},"world"]},{"foo":[1,2,{"bar":18},"world"]} == {"foo":[1,2,{"bar":19},"world"]}]
+{}
+[true,false]
+
+# containment operator
+[("foo" | contains("foo")), ("foobar" | contains("foo")), ("foo" | contains("foobar"))]
+{}
+[true, true, false]
+
+map(.[1] as $needle | .[0] | contains($needle))
+[[[],[]], [[1,2,3], [1,2]], [[1,2,3], [3,1]], [[1,2,3], [4]], [[1,2,3], [1,4]]]
+[true, true, true, false, false]
+
+map(.[1] as $needle | .[0] | contains($needle))
+[[["foobar", "foobaz"], ["baz", "bar"]], [["foobar", "foobaz"], ["foo"]], [["foobar", "foobaz"], ["blap"]]]
+[true, true, false]
+
+[({foo: 12, bar:13} | contains({foo: 12})), ({foo: 12} | contains({})), ({foo: 12, bar:13} | contains({baz:14}))]
+{}
+[true, true, false]
+
+{foo: {baz: 12, blap: {bar: 13}}, bar: 14} | contains({bar: 14, foo: {blap: {}}})
+{}
+true
+
+{foo: {baz: 12, blap: {bar: 13}}, bar: 14} | contains({bar: 14, foo: {blap: {bar: 14}}})
+{}
+false
+
+sort
+[42,[2,5,3,11],10,{"a":42,"b":2},{"a":42},true,2,[2,6],"hello",null,[2,5,6],{"a":[],"b":1},"abc","ab",[3,10],{},false,"abcd",null]
+[null,null,false,true,2,10,42,"ab","abc","abcd","hello",[2,5,3,11],[2,5,6],[2,6],[3,10],{},{"a":42},{"a":42,"b":2},{"a":[],"b":1}]
+
+(sort_by(.b) | sort_by(.a)), sort_by(.a, .b), sort_by(.b, .c), group_by(.b), group_by(.a + .b - .c == 2)
+[{"a": 1, "b": 4, "c": 14}, {"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}, {"a": 0, "b": 2, "c": 43}]
+[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}]
+[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}]
+[{"a": 4, "b": 1, "c": 3}, {"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 3}, {"a": 1, "b": 4, "c": 14}]
+[[{"a": 4, "b": 1, "c": 3}], [{"a": 0, "b": 2, "c": 43}], [{"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}]]
+[[{"a": 1, "b": 4, "c": 14}, {"a": 0, "b": 2, "c": 43}], [{"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}]]
+
+unique
+[1,2,5,3,5,3,1,3]
+[1,2,3,5]
+
+[min, max, min_by(.[1]), max_by(.[1]), min_by(.[2]), max_by(.[2])]
+[[4,2,"a"],[3,1,"a"],[2,4,"a"],[1,3,"a"]]
+[[1,3,"a"],[4,2,"a"],[3,1,"a"],[2,4,"a"],[4,2,"a"],[1,3,"a"]]
+
+[min,max,min_by(.),max_by(.)]
+[]
+[null,null,null,null]
+
+.foo[.baz]
+{"foo":{"bar":4},"baz":"bar"}
+4
+
+.[] | .error = "no, it's OK"
+[{"error":true}]
+{"error": "no, it's OK"}
+
+[{a:1}] | .[] | .a=999
+null
+{"a": 999}

tests/run

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cat $@ | valgrind --error-exitcode=1 -q --leak-check=full ./jq --run-tests