Remove #includes from jv.h

Fix the jv_parser interface.
Use libtool's built-in symbol exporting rather than a mapfile.
2026-11-01 12:05:54 +00:00 · 2013-06-23 14:23:07 +01:00 · 2013-06-23 12:26:49 +01:00 · 2013-06-23 12:10:58 +01:00 · 2013-06-23 12:08:22 +01:00 · 2013-06-23 12:01:58 +01:00
66 changed files with 3381 additions and 1486 deletions
@@ -1,15 +1,41 @@
 *.o
+*.a
+*.lo
+*.la
+*.lai
+*.so
+*.so.*
 *~
+.*.sw[a-p]
+tags

-# Autogenerated
-*.gen.*
-
-# Test binaries
-jv_test
-jv_parse
-parsertest*~
-
-build
+jq
+jq.1

 # Something delightfully recursive happens otherwise
-docs/content/2.download/source/*
+docs/content/2.download/source/*
+
+# Autogenerated
+lexer.c
+lexer.h
+parser.c
+parser.h
+
+# Autotools junk
+.libs
+.deps
+libtool
+*.log
+stamp-h1
+config.log
+config.status
+autom4te.cache
+INSTALL
+Makefile
+jq-*.tar.gz
+configure
+aclocal.m4
+Makefile.in
+version.h
+.remake-version-h
+config.cache
@@ -0,0 +1,6 @@
+Created By:
+Stephen Dolan        <mu@netsoc.tcd.ie>
+
+Contributions by
+Anthony Shortland    <anthony.shortland@me.com>      - rpmbuild target
+Lee Thompson         <stagr.lee@gmail.com>           - autoconf stuff
@@ -1,102 +0,0 @@
-CC=gcc
-CFLAGS=-Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function $(EXTRA_CFLAGS)
-
-prefix=/usr/local
-mandir=$(prefix)/share/man
-
-
-.PHONY: all clean releasedep tarball install uninstall test releasetag
-all: jq
-
-lexer.gen.c: lexer.l
-	flex -o lexer.gen.c --header-file=lexer.gen.h lexer.l
-lexer.gen.h: lexer.gen.c
-
-parser.gen.c: parser.y lexer.gen.h
-	bison -W -d parser.y -v --report-file=parser.gen.info -o $@
-parser.gen.h: parser.gen.c
-
-jv_utf8_tables.gen.h: gen_utf8_tables.py
-	python $^ > $@
-jv_unicode.c: jv_utf8_tables.gen.h
-
-version.gen.h: VERSION
-	sed 's/.*/#define JQ_VERSION "&"/' $^ > $@
-main.c: version.gen.h
-
-JQ_SRC=parser.gen.c lexer.gen.c opcode.c bytecode.c compile.c execute.c builtin.c jv.c jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_alloc.c
-
-jq_test: CFLAGS += -DJQ_DEBUG=1
-jq_test: $(JQ_SRC) jq_test.c
-	$(CC) $(CFLAGS) $(CFLAGS_DEBUG) -o $@ $^
-
-jq: CFLAGS += -O -DJQ_DEBUG=0
-jq: $(JQ_SRC) main.c
-	$(CC) $(CFLAGS) $(CFLAGS_OPT) -o $@ $^
-
-test: jq_test
-	valgrind --error-exitcode=1 -q --leak-check=full ./jq_test >/dev/null
-
-BINARIES=jq jq_test
-PLATFORMS=linux32 linux64 osx32 osx64 win32 win64
-
-build/linux32%: CC='x86_64-linux-gnu-gcc -m32'
-build/linux64%: CC='x86_64-linux-gnu-gcc -m64'
-
-# OS X cross compilers can be gotten from
-# https://launchpad.net/~flosoft/+archive/cross-apple
-build/osx32%:   CC='i686-apple-darwin10-gcc -m32'
-build/osx64%:   CC='i686-apple-darwin10-gcc -m64'
-
-# On Debian, you can get windows compilers in the
-# gcc-mingw-w64-i686 and gcc-mingw-w64-x86-64 packages.
-build/win32%:   CC='i686-w64-mingw32-gcc -m32'   EXTRA_CFLAGS=-DJQ_DEFAULT_ENABLE_COLOR=0
-build/win64%:   CC='x86_64-w64-mingw32-gcc -m64' EXTRA_CFLAGS=-DJQ_DEFAULT_ENABLE_COLOR=0
-
-ALL_BINARIES=$(foreach platform, $(PLATFORMS), $(foreach binary, $(BINARIES), build/$(platform)/$(binary)))
-
-$(ALL_BINARIES): build/%:
-	mkdir -p $(@D)
-	make -B $(BINARIES) CC=$(CC)
-	cp $(BINARIES) $(@D)
-
-binaries: $(ALL_BINARIES)
-
-clean:
-	rm -rf build
-	rm -f $(BINARIES) *.gen.*
-
-releasedep: lexer.gen.c parser.gen.c jv_utf8_tables.gen.h
-
-releasetag:
-	git tag -s "jq-$$(cat VERSION)" -m "jq release $$(cat VERSION)"
-
-docs/content/2.download/source/jq.tgz: jq
-	mkdir -p `dirname $@`
-	tar -czvf $@ `git ls-files; ls *.gen.*`
-
-tarball: docs/content/2.download/source/jq.tgz
-
-jq.1: docs/content/3.manual/manual.yml
-	( cd docs; rake manpage; ) > $@
-
-install: jq jq.1
-	install -d -m 0755 $(prefix)/bin
-	install -m 0755 jq $(prefix)/bin
-	install -d -m 0755 $(mandir)/man1
-	install -m 0755 jq.1 $(mandir)/man1
-
-uninstall:
-	rm -vf $(prefix)/bin/jq
-	rm -vf $(mandir)/man1/jq.1
-
-
-www: docs/output www_binaries
-
-docs/output:
-	cd docs; rake build
-
-www_binaries: docs/output binaries
-	$(foreach platform, $(PLATFORMS), $(foreach binary, $(BINARIES), \
-	  mkdir -p docs/output/download/$(platform); \
-	  cp build/$(platform)/$(binary) docs/output/download/$(platform)/$(binary); ))
@@ -0,0 +1,144 @@
+### C source files to be built and distributed.
+
+LIBJQ_INCS = jq_parser.h builtin.h bytecode.h compile.h exec_stack.h	\
+   jv_alloc.h jv_dtoa.h jv_parse.h jv_unicode.h locfile.h		\
+   opcode_list.h parser.y jv_utf8_tables.h lexer.l
+
+LIBJQ_SRC = locfile.c bytecode.c compile.c execute.c builtin.c jv.c	\
+  jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_file.c	\
+  jv_alloc.c jq_test.c ${LIBJQ_INCS}
+
+
+### C build options
+
+AM_CFLAGS = -Wextra -Wall -Wno-missing-field-initializers	\
+  -Wno-unused-parameter -Wno-unused-function
+
+ACLOCAL_AMFLAGS = -I config/m4
+
+### Generating the lexer and parser
+
+# While there is some autoconf macro support for lex/flex, it doesn't support
+# header file creation so we'll use good old make
+BUILT_SOURCES = lexer.h lexer.c parser.h parser.c version.h
+lexer.c: lexer.l
+	$(AM_V_LEX) flex -o lexer.c --header-file=lexer.h $<
+lexer.h: lexer.c
+
+# Tell YACC (bison) autoconf macros that you want a header file created.
+# If the --warnings=all fails, you probably have an old version of bison
+# OSX ships an old bison, so update with homebrew or macports
+AM_YFLAGS = --warnings=all -d
+
+### libjq
+
+lib_LTLIBRARIES = libjq.la
+libjq_la_SOURCES = ${LIBJQ_SRC}
+libjq_la_LIBADD = -lm
+libjq_la_LDFLAGS = -export-symbols-regex '^j[qv]_' -version-info 1:4:0
+
+include_HEADERS = jv.h jq.h
+
+
+### Building the jq binary
+
+# Remake the version.h header file if, and only if, the git ID has changed
+.PHONY: .FORCE
+.FORCE:
+generate_ver = ver="`{ $(srcdir)/scripts/version || echo '$(VERSION)' ; } | sed 's/.*/\#define JQ_VERSION \"&\"/'`"
+.remake-version-h: .FORCE
+	@ $(generate_ver); test "x`cat version.h 2>/dev/null`" = "x$$ver" || touch .remake-version-h
+version.h: .remake-version-h
+	$(AM_V_GEN) $(generate_ver); echo "$$ver" > $@
+
+bin_PROGRAMS = jq
+jq_SOURCES = main.c
+jq_LDFLAGS = -static-libtool-libs
+jq_LDADD = libjq.la -lm
+
+### Tests (make check)
+
+TESTS = tests/all.test
+TEST_LOG_COMPILER = ${srcdir}/tests/run
+
+
+### Building the manpage
+
+# If ENABLE_DOCS is not set, just copy jq.1.default to jq.1
+# The real_docs target succeeds (possibly after building jq.1) only if ENABLE_DOCS is set
+# Thus, making "dist" depend on "real_docs" ensures that we never build a tarball with
+# a stub manpage.
+
+man_MANS = jq.1
+.PHONY: real_docs
+if ENABLE_DOCS
+jq.1: $(srcdir)/docs/content/3.manual/manual.yml
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
+jq.1.default: $(srcdir)/docs/default_manpage.md
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
+real_docs: jq.1
+	if cmp jq.1 $(srcdir)/jq.1.default > /dev/null; then\
+	  rm -f jq.1; $(MAKE) $(AM_MAKEFLAGS) jq.1;\
+	fi
+else
+jq.1: $(srcdir)/jq.1.default
+	$(AM_V_GEN) cp $^ $@
+real_docs:
+	@echo "Ruby dependencies not found, cannot build manpage." > /dev/stderr
+	@echo "Follow the instructions in docs/README.md to install them" > /dev/stderr
+	@echo "and then rerun ./configure" > /dev/stderr
+	false
+endif
+
+
+### Packaging
+
+docs/site.yml: configure.ac
+	sed 's/^jq_version: .*/jq_version: "$(VERSION)"/' $@ > $@.new
+	mv $@.new $@
+
+install-binaries: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) install-exec
+
+# Ensure "make dist" fails when we can't build the real manpage
+dist-hook: real_docs
+
+DOC_FILES = docs/content docs/public docs/templates docs/site.yml	\
+            docs/Gemfile docs/Gemfile.lock docs/Rakefile docs/README.md \
+            docs/default_manpage.md jq.1.default
+
+# setup is only used by distribution developers, not package developers.
+# Still, as a matter of allowing patching, its not a bad idea to distribute 
+# the developer setup script in the tarball.
+EXTRA_DIST = $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)		\
+	gen_utf8_tables.py jq.spec $(DOC_FILES) scripts/version	\
+	parser.h parser.c lexer.h lexer.c libjq.map
+
+# README.md is expected in Github projects, good stuff in it, so we'll 
+# distribute it and install it with the package in the doc directory.
+docdir = ${datadir}/doc/${PACKAGE}
+dist_doc_DATA = README.md COPYING AUTHORS README
+
+RELEASE ?= 1
+rpm: jq
+	@echo "Packaging jq as an RPM ..."
+	mkdir -p rpm/SOURCES rpm/BUILD rpm/BUILDROOT rpm/RPMS rpm/SPECS
+	cp jq-$$(cat VERSION).tar.gz rpm/SOURCES/
+	rpmbuild -tb --define "_topdir ${PWD}/rpm" --define "_prefix /usr" --define "myver $$(cat VERSION)" --define "myrel ${RELEASE}" rpm/SOURCES/jq-$$(cat VERSION).tar.gz
+	find rpm/RPMS/ -name "*.rpm" -exec mv {} ./ \;
+	rm -rf rpm
+
+dist-clean-local:
+	rm -f ${BUILT_SOURCES}
+
+# Not sure why this doesn't get cleaned up automatically, guess
+# automake used to man pages which are hand coded?
+# 'make clean' doesn't delete the manpage if it can't be rebuilt
+.PHONY: clean-local-docs
+clean-local-docs:
+if ENABLE_DOCS
+	rm -f jq.1
+endif 
+
+clean-local: clean-local-docs
+	rm -f version.h .remake-version-h
@@ -0,0 +1 @@
+README.md
@@ -4,11 +4,25 @@ jq
 jq is a command-line JSON processor.

 If you want to learn to use jq, read the documentation at
-[http://stedolan.github.com/jq](http://stedolan.github.com/jq). This
+[http://stedolan.github.io/jq](http://stedolan.github.io/jq). This
 documentation is generated from the docs/ folder of this repository.

 If you want to hack on jq, feel free, but be warned that its internals
-are not well-documented at the moment. Bring a hard hat and a shovel.
+are not well-documented at the moment. Bring a hard hat and a
+shovel. Also, read the wiki: http://github.com/stedolan/jq/wiki

-You can find some basic build instructions at
-[http://stedolan.github.com/jq/download](http://stedolan.github.com/jq/download)
+If you're building directly from the latest git, you'll need flex and
+bison installed. To build, run:
+
+    autoreconf -i
+    ./configure
+    make
+
+After make finishes, you'll be able to use `./jq`. You can also
+install it using:
+
+    sudo make install
+
+If you're not using the latest git version but instead building a
+released tarball (available on the website), then you won't need to
+run `autoreconf` (and shouldn't), and you won't need flex or bison.
@@ -1 +0,0 @@
-1.2
@@ -0,0 +1 @@
+*
@@ -1,28 +1,14 @@
+#include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 #include "builtin.h"
-#include "bytecode.h"
 #include "compile.h"
-#include "parser.h"
+#include "jq_parser.h"
+#include "bytecode.h"
 #include "locfile.h"
-#include "jv_aux.h"
+#include "jv_unicode.h"


-typedef jv (*func_1)(jv);
-typedef jv (*func_2)(jv,jv);
-typedef jv (*func_3)(jv,jv,jv);
-typedef jv (*func_4)(jv,jv,jv,jv);
-typedef jv (*func_5)(jv,jv,jv,jv,jv);
-jv cfunction_invoke(struct cfunction* function, jv input[]) {
-  switch (function->nargs) {
-  case 1: return ((func_1)function->fptr)(input[0]);
-  case 2: return ((func_2)function->fptr)(input[0], input[1]);
-  case 3: return ((func_3)function->fptr)(input[0], input[1], input[2]);
-  case 4: return ((func_4)function->fptr)(input[0], input[1], input[2], input[3]);
-  case 5: return ((func_5)function->fptr)(input[0], input[1], input[2], input[3], input[4]);
-  default: return jv_invalid_with_msg(jv_string("Function takes too many arguments"));
-  }
-}
-
 static jv type_error(jv bad, const char* msg) {
  jv err = jv_invalid_with_msg(jv_string_fmt("%s %s",
                                             jv_kind_name(jv_get_kind(bad)),
@@ -43,7 +29,13 @@ static jv type_error2(jv bad1, jv bad2, const char* msg) {

 static jv f_plus(jv input, jv a, jv b) {
  jv_free(input);
-  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
+  if (jv_get_kind(a) == JV_KIND_NULL) {
+    jv_free(a);
+    return b;
+  } else if (jv_get_kind(b) == JV_KIND_NULL) {
+    jv_free(b);
+    return a;
+  } else if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
    return jv_number(jv_number_value(a) + 
                     jv_number_value(b));
  } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
@@ -57,17 +49,25 @@ static jv f_plus(jv input, jv a, jv b) {
  }
 }

+static jv f_negate(jv input) {
+  if (jv_get_kind(input) != JV_KIND_NUMBER) {
+    return type_error(input, "cannot be negated");
+  }
+  jv ret = jv_number(-jv_number_value(input));
+  jv_free(input);
+  return ret;
+}
+
 static jv f_minus(jv input, jv a, jv b) {
  jv_free(input);
  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
    return jv_number(jv_number_value(a) - jv_number_value(b));
  } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
    jv out = jv_array();
-    for (int i=0; i<jv_array_length(jv_copy(a)); i++) {
-      jv x = jv_array_get(jv_copy(a), i);
+    jv_array_foreach(a, i, x) {
      int include = 1;
-      for (int j=0; j<jv_array_length(jv_copy(b)); j++) {
-        if (jv_equal(jv_copy(x), jv_array_get(jv_copy(b), j))) {
+      jv_array_foreach(b, j, y) {
+        if (jv_equal(jv_copy(x), y)) {
          include = 0;
          break;
        }
@@ -102,22 +102,6 @@ static jv f_divide(jv input, jv a, jv b) {
  }  
 }

-static jv f_add(jv array) {
-  if (jv_get_kind(array) != JV_KIND_ARRAY) {
-    return type_error(array, "cannot have its elements added");
-  } else if (jv_array_length(jv_copy(array)) == 0) {
-    jv_free(array);
-    return jv_null();
-  } else {
-    jv sum = jv_array_get(jv_copy(array), 0);
-    for (int i = 1; i < jv_array_length(jv_copy(array)); i++) {
-      sum = f_plus(jv_null(), sum, jv_array_get(jv_copy(array), i));
-    }
-    jv_free(array);
-    return sum;
-  }
-}
-
 static jv f_equal(jv input, jv a, jv b) {
  jv_free(input);
  return jv_bool(jv_equal(a, b));
@@ -161,9 +145,7 @@ static jv f_greatereq(jv input, jv a, jv b) {
 }

 static jv f_contains(jv a, jv b) {
-  jv_kind akind = jv_get_kind(a);
-
-  if (akind == jv_get_kind(b)) {
+  if (jv_get_kind(a) == jv_get_kind(b)) {
    return jv_bool(jv_contains(a, b));
  } else {
    return type_error2(a, b, "cannot have their containment checked");
@@ -190,7 +172,7 @@ static jv f_length(jv input) {
  } else if (jv_get_kind(input) == JV_KIND_OBJECT) {
    return jv_number(jv_object_length(input));
  } else if (jv_get_kind(input) == JV_KIND_STRING) {
-    return jv_number(jv_string_length(input));
+    return jv_number(jv_string_length_codepoints(input));
  } else if (jv_get_kind(input) == JV_KIND_NULL) {
    jv_free(input);
    return jv_number(0);
@@ -207,6 +189,172 @@ static jv f_tostring(jv input) {
  }
 }

+#define CHARS_ALPHANUM "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+
+static jv escape_string(jv input, const char* escapings) {
+
+  assert(jv_get_kind(input) == JV_KIND_STRING);
+  const char* lookup[128] = {0};
+  const char* p = escapings;
+  while (*p) {
+    lookup[(int)*p] = p+1;
+    p++;
+    p += strlen(p);
+    p++;
+  }
+
+  jv ret = jv_string("");
+  const char* i = jv_string_value(input);
+  const char* end = i + jv_string_length_bytes(jv_copy(input));
+  const char* cstart;
+  int c = 0;
+  while ((i = jvp_utf8_next((cstart = i), end, &c))) {
+    assert(c != -1);
+    if (c < 128 && lookup[c]) {
+      ret = jv_string_append_str(ret, lookup[c]);
+    } else {
+      ret = jv_string_append_buf(ret, cstart, i - cstart);
+    }
+  }
+  jv_free(input);
+  return ret;
+
+}
+
+static jv f_format(jv input, jv fmt) {
+  if (jv_get_kind(fmt) != JV_KIND_STRING) {
+    jv_free(input);
+    return type_error(fmt, "is not a valid format");
+  }
+  const char* fmt_s = jv_string_value(fmt);
+  if (!strcmp(fmt_s, "json")) {
+    jv_free(fmt);
+    return jv_dump_string(input, 0);
+  } else if (!strcmp(fmt_s, "text")) {
+    jv_free(fmt);
+    return f_tostring(input);
+  } else if (!strcmp(fmt_s, "csv")) {
+    jv_free(fmt);
+    if (jv_get_kind(input) != JV_KIND_ARRAY)
+      return type_error(input, "cannot be csv-formatted, only array");
+    jv line = jv_string("");
+    jv_array_foreach(input, i, x) {
+      if (i) line = jv_string_append_str(line, ",");
+      switch (jv_get_kind(x)) {
+      case JV_KIND_NULL:
+        /* null rendered as empty string */
+        jv_free(x);
+        break;
+      case JV_KIND_TRUE:
+      case JV_KIND_FALSE:
+        line = jv_string_concat(line, jv_dump_string(x, 0));
+        break;
+      case JV_KIND_NUMBER:
+        if (jv_number_value(x) != jv_number_value(x)) {
+          /* NaN, render as empty string */
+          jv_free(x);
+        } else {
+          line = jv_string_concat(line, jv_dump_string(x, 0));
+        }
+        break;
+      case JV_KIND_STRING: {
+        line = jv_string_append_str(line, "\"");
+        line = jv_string_concat(line, escape_string(x, "\"\"\"\0"));
+        line = jv_string_append_str(line, "\"");
+        break;
+      }
+      default:
+        jv_free(input);
+        jv_free(line);
+        return type_error(x, "is not valid in a csv row");
+      }
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "html")) {
+    jv_free(fmt);
+    return escape_string(f_tostring(input), "&&amp;\0<&lt;\0>&gt;\0'&apos;\0\"&quot;\0");
+  } else if (!strcmp(fmt_s, "uri")) {
+    jv_free(fmt);
+    input = f_tostring(input);
+
+    int unreserved[128] = {0};
+    const char* p = CHARS_ALPHANUM "-_.!~*'()";
+    while (*p) unreserved[(int)*p++] = 1;
+
+    jv line = jv_string("");
+    const char* s = jv_string_value(input);
+    for (int i=0; i<jv_string_length_bytes(jv_copy(input)); i++) {
+      unsigned ch = (unsigned)(unsigned char)*s;
+      if (ch < 128 && unreserved[ch]) {
+        line = jv_string_append_buf(line, s, 1);
+      } else {
+        line = jv_string_concat(line, jv_string_fmt("%%%02x", ch));
+      }
+      s++;
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "sh")) {
+    jv_free(fmt);
+    if (jv_get_kind(input) != JV_KIND_ARRAY)
+      input = jv_array_set(jv_array(), 0, input);
+    jv line = jv_string("");
+    jv_array_foreach(input, i, x) {
+      if (i) line = jv_string_append_str(line, " ");
+      switch (jv_get_kind(x)) {
+      case JV_KIND_NULL:
+      case JV_KIND_TRUE:
+      case JV_KIND_FALSE:
+      case JV_KIND_NUMBER:
+        line = jv_string_concat(line, jv_dump_string(x, 0));
+        break;
+
+      case JV_KIND_STRING: {
+        line = jv_string_append_str(line, "'");
+        line = jv_string_concat(line, escape_string(x, "''\\''\0"));
+        line = jv_string_append_str(line, "'");
+        break;
+      }
+
+      default:
+        jv_free(input);
+        jv_free(line);
+        return type_error(x, "can not be escaped for shell");
+      }
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "base64")) {
+    jv_free(fmt);
+    input = f_tostring(input);
+    jv line = jv_string("");
+    const char b64[64 + 1] = CHARS_ALPHANUM "+/";
+    const char* data = jv_string_value(input);
+    int len = jv_string_length_bytes(jv_copy(input));
+    for (int i=0; i<len; i+=3) {
+      uint32_t code = 0;
+      int n = len - i >= 3 ? 3 : len-i;
+      for (int j=0; j<3; j++) {
+        code <<= 8;
+        code |= j < n ? (unsigned)data[i+j] : 0;
+      }
+      char buf[4];
+      for (int j=0; j<4; j++) {
+        buf[j] = b64[(code >> (18 - j*6)) & 0x3f];
+      }
+      if (n < 3) buf[3] = '=';
+      if (n < 2) buf[2] = '=';
+      line = jv_string_append_buf(line, buf, sizeof(buf));
+    }
+    jv_free(input);
+    return line;
+  } else {
+    jv_free(input);
+    return jv_invalid_with_msg(jv_string_concat(fmt, jv_string(" is not a valid format")));
+  }
+}
+
 static jv f_keys(jv input) {
  if (jv_get_kind(input) == JV_KIND_OBJECT || jv_get_kind(input) == JV_KIND_ARRAY) {
    return jv_keys(input);
@@ -305,15 +453,19 @@ static jv f_error(jv input, jv msg) {
  return jv_invalid_with_msg(msg);
 }

-
-static struct cfunction function_list[] = {
+static const struct cfunction function_list[] = {
  {(cfunction_ptr)f_plus, "_plus", 3},
+  {(cfunction_ptr)f_negate, "_negate", 1},
  {(cfunction_ptr)f_minus, "_minus", 3},
  {(cfunction_ptr)f_multiply, "_multiply", 3},
  {(cfunction_ptr)f_divide, "_divide", 3},
  {(cfunction_ptr)f_tonumber, "tonumber", 1},
  {(cfunction_ptr)f_tostring, "tostring", 1},
  {(cfunction_ptr)f_keys, "keys", 1},
+  {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking
+  {(cfunction_ptr)jv_getpath, "getpath", 2},
+  {(cfunction_ptr)jv_delpaths, "delpaths", 2},
+  {(cfunction_ptr)jv_has, "has", 2},
  {(cfunction_ptr)f_equal, "_equal", 3},
  {(cfunction_ptr)f_notequal, "_notequal", 3},
  {(cfunction_ptr)f_less, "_less", 3},
@@ -323,7 +475,6 @@ static struct cfunction function_list[] = {
  {(cfunction_ptr)f_contains, "contains", 2},
  {(cfunction_ptr)f_length, "length", 1},
  {(cfunction_ptr)f_type, "type", 1},
-  {(cfunction_ptr)f_add, "add", 1},
  {(cfunction_ptr)f_sort, "sort", 1},
  {(cfunction_ptr)f_sort_by_impl, "_sort_by_impl", 2},
  {(cfunction_ptr)f_group_by_impl, "_group_by_impl", 2},
@@ -332,31 +483,53 @@ static struct cfunction function_list[] = {
  {(cfunction_ptr)f_min_by_impl, "_min_by_impl", 2},
  {(cfunction_ptr)f_max_by_impl, "_max_by_impl", 2},
  {(cfunction_ptr)f_error, "error", 2},
+  {(cfunction_ptr)f_format, "format", 2},
 };

-static struct symbol_table cbuiltins = 
-  {function_list, sizeof(function_list)/sizeof(function_list[0])};
-
-typedef block (*bytecoded_builtin)();
 struct bytecoded_builtin { const char* name; block code; };
 static block bind_bytecoded_builtins(block b) {
-  struct bytecoded_builtin builtin_defs[] = {
-    {"empty", gen_op_simple(BACKTRACK)},
-    {"false", gen_const(jv_false())},
-    {"true", gen_const(jv_true())},
-    {"null", gen_const(jv_null())},
-    {"not", gen_condbranch(gen_const(jv_false()),
-                           gen_const(jv_true()))}
-  };
  block builtins = gen_noop();
-  for (unsigned i=0; i<sizeof(builtin_defs)/sizeof(builtin_defs[0]); i++) {
-    builtins = BLOCK(builtins, gen_function(builtin_defs[i].name, gen_noop(),
-                                            builtin_defs[i].code));
+  {
+    struct bytecoded_builtin builtin_defs[] = {
+      {"empty", gen_op_simple(BACKTRACK)},
+      {"false", gen_const(jv_false())},
+      {"true", gen_const(jv_true())},
+      {"null", gen_const(jv_null())},
+      {"not", gen_condbranch(gen_const(jv_false()),
+                             gen_const(jv_true()))}
+    };
+    for (unsigned i=0; i<sizeof(builtin_defs)/sizeof(builtin_defs[0]); i++) {
+      builtins = BLOCK(builtins, gen_function(builtin_defs[i].name, gen_noop(),
+                                              builtin_defs[i].code));
+    }
  }
-  return block_bind(builtins, b, OP_IS_CALL_PSEUDO);
+  {
+    struct bytecoded_builtin builtin_def_1arg[] = {
+      {"path", BLOCK(gen_op_simple(PATH_BEGIN), 
+                     gen_call("arg", gen_noop()),
+                     gen_op_simple(PATH_END))},
+    };
+    for (unsigned i=0; i<sizeof(builtin_def_1arg)/sizeof(builtin_def_1arg[0]); i++) {
+      builtins = BLOCK(builtins, gen_function(builtin_def_1arg[i].name,
+                                              gen_param("arg"),
+                                              builtin_def_1arg[i].code));
+    }
+  }
+  {
+    block rangevar = gen_op_var_fresh(STOREV, "rangevar");
+    block init = BLOCK(gen_op_simple(DUP), gen_call("start", gen_noop()), rangevar);
+    block range = BLOCK(init, 
+                        gen_call("end", gen_noop()),
+                        gen_op_bound(RANGE, rangevar));
+    builtins = BLOCK(builtins, gen_function("range",
+                                            BLOCK(gen_param("start"), gen_param("end")),
+                                            range));
+  }
+  
+  return block_bind_referenced(builtins, b, OP_IS_CALL_PSEUDO);
 }

-static const char* jq_builtins[] = {
+static const char* const jq_builtins[] = {
  "def map(f): [.[] | f];",
  "def select(f): if f then . else empty end;",
  "def sort_by(f): _sort_by_impl(map([f]));",
@@ -364,19 +537,56 @@ static const char* jq_builtins[] = {
  "def unique: group_by(.) | map(.[0]);",
  "def max_by(f): _max_by_impl(map([f]));",
  "def min_by(f): _min_by_impl(map([f]));",
+  "def add: reduce .[] as $x (null; . + $x);",
+  "def del(f): delpaths([path(f)]);",
+  "def _assign(paths; value): value as $v | reduce path(paths) as $p (.; setpath($p; $v));",
+  "def _modify(paths; update): reduce path(paths) as $p (.; setpath($p; getpath($p) | update));",
+  "def recurse(f): ., (f | select(. != null) | recurse(f));",
+  "def to_entries: [keys[] as $k | {key: $k, value: .[$k]}];",
+  "def from_entries: map({(.key): .value}) | add;",
+  "def with_entries(f): to_entries | map(f) | from_entries;",
+  "def reverse: [.[length - 1 - range(0;length)]];",
 };


-block builtins_bind(block b) {
-  for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) {
-    struct locfile src;
-    locfile_init(&src, jq_builtins[i], strlen(jq_builtins[i]));
-    block funcs;
-    int nerrors = jq_parse_library(&src, &funcs);
-    assert(!nerrors);
-    b = block_bind(funcs, b, OP_IS_CALL_PSEUDO);
-    locfile_free(&src);
+int builtins_bind_one(block* bb, const char* code) {
+  struct locfile src;
+  locfile_init(&src, code, strlen(code));
+  block funcs;
+  int nerrors = jq_parse_library(&src, &funcs);
+  if (nerrors == 0) {
+    *bb = block_bind_referenced(funcs, *bb, OP_IS_CALL_PSEUDO);
  }
-  b = bind_bytecoded_builtins(b);
-  return gen_cbinding(&cbuiltins, b);
+  locfile_free(&src);
+  return nerrors;
+}
+
+int slurp_lib(block* bb) {
+  int nerrors = 0;
+  char* home = getenv("HOME");
+  if (home) {    // silently ignore no $HOME
+    jv filename = jv_string_append_str(jv_string(home), "/.jq");
+    jv data = jv_load_file(jv_string_value(filename), 1);
+    if (jv_is_valid(data)) {
+      nerrors = builtins_bind_one(bb, jv_string_value(data) );
+    }
+    jv_free(filename);
+    jv_free(data);
+  }
+  return nerrors;
+}
+
+int builtins_bind(block* bb) {
+  int nerrors = slurp_lib(bb);
+  if (nerrors) {
+    block_free(*bb);
+    return nerrors;
+  }
+  for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) {
+    nerrors = builtins_bind_one(bb, jq_builtins[i]);
+    assert(!nerrors);
+  }
+  *bb = bind_bytecoded_builtins(*bb);
+  *bb = gen_cbinding(function_list, sizeof(function_list)/sizeof(function_list[0]), *bb);
+  return nerrors;
 }
@@ -1,21 +1,9 @@
 #ifndef BUILTIN_H
 #define BUILTIN_H

+#include "bytecode.h"
 #include "compile.h"

-block builtins_bind(block);
-
-
-typedef void (*cfunction_ptr)(void);
-
-struct cfunction {
-  cfunction_ptr fptr;
-  const char* name;
-  int nargs;
-};
-
-
-jv cfunction_invoke(struct cfunction* function, jv input[]);
-
+int builtins_bind(block*);

 #endif
@@ -3,9 +3,39 @@
 #include <stdlib.h>

 #include "bytecode.h"
-#include "opcode.h"
 #include "jv_alloc.h"

+// flags, length
+#define NONE 0, 1
+#define CONSTANT OP_HAS_CONSTANT, 2
+#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
+#define BRANCH OP_HAS_BRANCH, 2
+#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
+#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
+#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
+#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
+
+#define OP(name, imm, in, out) \
+  {name, #name, imm, in, out},
+
+static const struct opcode_description opcode_descriptions[] = {
+#include "opcode_list.h"
+};
+
+static const struct opcode_description invalid_opcode_description = {
+  -1, "#INVALID", 0, 0, 0, 0
+};
+
+
+const struct opcode_description* opcode_describe(opcode op) {
+  if ((int)op >= 0 && (int)op < NUM_OPCODES) {
+    return &opcode_descriptions[op];
+  } else {
+    return &invalid_opcode_description;
+  }
+}
+
+
 static int bytecode_operation_length(uint16_t* codeptr) {
  int length = opcode_describe(*codeptr)->length;
  if (*codeptr == CALL_JQ) {
@@ -14,18 +44,7 @@ static int bytecode_operation_length(uint16_t* codeptr) {
  return length;
 }

-void dump_disassembly(int indent, struct bytecode* bc) {
-  if (bc->nclosures > 0) {
-    printf("%*s[params: %d]\n", indent, "", bc->nclosures);
-  }
-  dump_code(indent, bc);
-  for (int i=0; i<bc->nsubfunctions; i++) {
-    printf("%*ssubfn[%d]:\n", indent, "", i);
-    dump_disassembly(indent+2, bc->subfunctions[i]);
-  }
-}
-
-void dump_code(int indent, struct bytecode* bc) {
+static void dump_code(int indent, struct bytecode* bc) {
  int pc = 0;
  while (pc < bc->codelen) {
    printf("%*s", indent, "");
@@ -35,6 +54,43 @@ void dump_code(int indent, struct bytecode* bc) {
  }
 }

+static void symbol_table_free(struct symbol_table* syms) {
+  jv_mem_free(syms->cfunctions);
+  jv_free(syms->cfunc_names);
+  jv_mem_free(syms);
+}
+
+void dump_disassembly(int indent, struct bytecode* bc) {
+  if (bc->nclosures > 0) {
+    printf("%*s[params: ", indent, "");
+    jv params = jv_object_get(jv_copy(bc->debuginfo), jv_string("params"));
+    for (int i=0; i<bc->nclosures; i++) {
+      if (i) printf(", ");
+      jv name = jv_array_get(jv_copy(params), i);
+      printf("%s", jv_string_value(name));
+      jv_free(name);
+    }
+    jv_free(params);
+    printf("]\n");
+  }
+  dump_code(indent, bc);
+  for (int i=0; i<bc->nsubfunctions; i++) {
+    struct bytecode* subfn = bc->subfunctions[i];
+    jv name = jv_object_get(jv_copy(subfn->debuginfo), jv_string("name"));
+    printf("%*s%s:%d:\n", indent, "", jv_string_value(name), i);
+    jv_free(name);
+    dump_disassembly(indent+2, subfn);
+  }
+}
+
+static struct bytecode* getlevel(struct bytecode* bc, int level) {
+  while (level > 0) {
+    bc = bc->parent;
+    level--;
+  }
+  return bc;
+}
+
 void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
  int pc = codeptr - bc->code;
  printf("%04d ", pc);
@@ -46,15 +102,28 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
      for (int i=0; i<imm+1; i++) {
        uint16_t level = bc->code[pc++];
        uint16_t idx = bc->code[pc++];
+        jv name;
        if (idx & ARG_NEWCLOSURE) {
-          printf(" subfn[%d]", idx & ~ARG_NEWCLOSURE);
+          idx &= ~ARG_NEWCLOSURE;
+          name = jv_object_get(jv_copy(getlevel(bc,level)->subfunctions[idx]->debuginfo), 
+                               jv_string("name"));
        } else {
-          printf(" param[%d]", idx);
+          name = jv_array_get(jv_object_get(jv_copy(getlevel(bc,level)->debuginfo), 
+                                            jv_string("params")), idx);
        }
+        printf(" %s:%d",
+               jv_string_value(name),
+               idx);
+        jv_free(name);
        if (level) {
          printf("^%d", level);
        }
      }
+    } else if (op->op == CALL_BUILTIN) {
+      int func = bc->code[pc++];
+      jv name = jv_array_get(jv_copy(bc->globals->cfunc_names), func);
+      printf(" %s", jv_string_value(name));
+      jv_free(name);
    } else if (op->flags & OP_HAS_BRANCH) {
      printf(" %04d", pc + imm);
    } else if (op->flags & OP_HAS_CONSTANT) {
@@ -62,7 +131,11 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
      jv_dump(jv_array_get(jv_copy(bc->constants), imm), 0);
    } else if (op->flags & OP_HAS_VARIABLE) {
      uint16_t v = bc->code[pc++];
-      printf(" v%d", v);
+      jv name = jv_array_get(jv_object_get(jv_copy(getlevel(bc,imm)->debuginfo), jv_string("locals")), v);
+      printf(" $%s:%d", 
+             jv_string_value(name), 
+             v);
+      jv_free(name);
      if (imm) {
        printf("^%d", imm);
      }
@@ -72,12 +145,9 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
  }  
 }

-void symbol_table_free(struct symbol_table* syms) {
-  jv_mem_free(syms->cfunctions);
-  jv_mem_free(syms);
-}
-
 void bytecode_free(struct bytecode* bc) {
+  if (!bc)
+    return;
  jv_mem_free(bc->code);
  jv_free(bc->constants);
  for (int i=0; i<bc->nsubfunctions; i++)
@@ -85,5 +155,6 @@ void bytecode_free(struct bytecode* bc) {
  if (!bc->parent)
    symbol_table_free(bc->globals);
  jv_mem_free(bc->subfunctions);
+  jv_free(bc->debuginfo);
  jv_mem_free(bc);
 }
@@ -3,13 +3,56 @@
 #include <stdint.h>

 #include "jv.h"
-#include "opcode.h"
-#include "builtin.h"
+
+typedef enum {
+#define OP(name, imm, in, out) name,
+#include "opcode_list.h"
+#undef OP
+} opcode;
+
+enum {
+  NUM_OPCODES = 
+#define OP(name, imm, in, out) +1
+#include "opcode_list.h"
+#undef OP
+};
+
+enum {
+  OP_HAS_CONSTANT = 2,
+  OP_HAS_VARIABLE = 4,
+  OP_HAS_BRANCH = 8,
+  OP_HAS_CFUNC = 32,
+  OP_HAS_UFUNC = 64,
+  OP_IS_CALL_PSEUDO = 128,
+  OP_HAS_BINDING = 1024,
+};
+struct opcode_description {
+  opcode op;
+  const char* name;
+
+  int flags;
+
+  // length in 16-bit units
+  int length;
+
+  int stack_in, stack_out;
+};
+
+const struct opcode_description* opcode_describe(opcode op);
+

 #define MAX_CFUNCTION_ARGS 10
+typedef void (*cfunction_ptr)();
+struct cfunction {
+  cfunction_ptr fptr;
+  const char* name;
+  int nargs;
+};
+
 struct symbol_table {
  struct cfunction* cfunctions;
  int ncfunctions;
+  jv cfunc_names;
 };

 // The bytecode format matters in:
@@ -33,13 +76,13 @@ struct bytecode {
  int nsubfunctions;

  struct bytecode* parent;
+
+  jv debuginfo;
 };

 void dump_disassembly(int, struct bytecode* code);
-void dump_code(int, struct bytecode* code);
 void dump_operation(struct bytecode* bc, uint16_t* op);

-void symbol_table_free(struct symbol_table* syms);
 void bytecode_free(struct bytecode* bc);

 #endif
@@ -2,7 +2,6 @@
 #include <assert.h>
 #include <string.h>
 #include <stdlib.h>
-#include "opcode.h"
 #include "compile.h"
 #include "bytecode.h"
 #include "locfile.h"
@@ -30,7 +29,7 @@ struct inst {
    uint16_t intval;
    struct inst* target;
    jv constant;
-    struct cfunction* cfunc;
+    const struct cfunction* cfunc;
  } imm;

  location source;
@@ -153,33 +152,26 @@ void inst_set_target(block b, block target) {
  b.first->imm.target = target.last;
 }

-block gen_op_var_unbound(opcode op, const char* name) {
+block gen_op_unbound(opcode op, const char* name) {
+  assert(opcode_describe(op)->flags & OP_HAS_BINDING);
+  inst* i = inst_new(op);
+  i->symbol = strdup(name);
+  return inst_block(i);
+}
+
+block gen_op_var_fresh(opcode op, const char* name) {
  assert(opcode_describe(op)->flags & OP_HAS_VARIABLE);
-  inst* i = inst_new(op);
-  i->symbol = strdup(name);
-  return inst_block(i);
+  return block_bind(gen_op_unbound(op, name),
+                    gen_noop(), OP_HAS_VARIABLE);
 }

-block gen_op_var_bound(opcode op, block binder) {
+block gen_op_bound(opcode op, block binder) {
  assert(block_is_single(binder));
-  block b = gen_op_var_unbound(op, binder.first->symbol);
+  block b = gen_op_unbound(op, binder.first->symbol);
  b.first->bound_by = binder.first;
  return b;
 }

-block gen_op_block_unbound(opcode op, const char* name) {
-  assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO);
-  inst* i = inst_new(op);
-  i->symbol = strdup(name);
-  return inst_block(i);
-}
-
-block gen_op_block_bound(opcode op, block binder) {
-  assert(block_is_single(binder));
-  block b = gen_op_block_unbound(op, binder.first->symbol);
-  b.first->bound_by = binder.first;
-  return b;
-}

 static void inst_join(inst* a, inst* b) {
  assert(a && b);
@@ -216,13 +208,14 @@ int block_has_only_binders(block binders, int bindflags) {
  return 1;
 }

-static void block_bind_subblock(block binder, block body, int bindflags) {
+static int block_bind_subblock(block binder, block body, int bindflags) {
  assert(block_is_single(binder));
  assert((opcode_describe(binder.first->op)->flags & bindflags) == bindflags);
  assert(binder.first->symbol);
  assert(binder.first->bound_by == 0 || binder.first->bound_by == binder.first);

  binder.first->bound_by = binder.first;
+  int nrefs = 0;
  for (inst* i = body.first; i; i = i->next) {
    int flags = opcode_describe(i->op)->flags;
    if ((flags & bindflags) == bindflags &&
@@ -230,12 +223,14 @@ static void block_bind_subblock(block binder, block body, int bindflags) {
        !strcmp(i->symbol, binder.first->symbol)) {
      // bind this instruction
      i->bound_by = binder.first;
+      nrefs++;
    }
    // binding recurses into closures
-    block_bind_subblock(binder, i->subfn, bindflags);
+    nrefs += block_bind_subblock(binder, i->subfn, bindflags);
    // binding recurses into argument list
-    block_bind_subblock(binder, i->arglist, bindflags);
+    nrefs += block_bind_subblock(binder, i->arglist, bindflags);
  }
+  return nrefs;
 }

 static void block_bind_each(block binder, block body, int bindflags) {
@@ -251,6 +246,21 @@ block block_bind(block binder, block body, int bindflags) {
  return block_join(binder, body);
 }

+block block_bind_referenced(block binder, block body, int bindflags) {
+  assert(block_has_only_binders(binder, bindflags));
+  bindflags |= OP_HAS_BINDING;
+  block refd = gen_noop();
+  for (inst* curr; (curr = block_take(&binder));) {
+    block b = inst_block(curr);
+    if (block_bind_subblock(b, body, bindflags)) {
+      refd = BLOCK(refd, b);
+    } else {
+      block_free(b);
+    }
+  }
+  return block_join(refd, body);
+}
+
 block gen_function(const char* name, block formals, block body) {
  block_bind_each(formals, body, OP_IS_CALL_PSEUDO);
  inst* i = inst_new(CLOSURE_CREATE);
@@ -262,12 +272,16 @@ block gen_function(const char* name, block formals, block body) {
  return b;
 }

+block gen_param(const char* name) {
+  return gen_op_unbound(CLOSURE_PARAM, name);
+}
+
 block gen_lambda(block body) {
  return gen_function("@lambda", gen_noop(), body);
 }

 block gen_call(const char* name, block args) {
-  block b = gen_op_block_unbound(CALL_JQ, name);
+  block b = gen_op_unbound(CALL_JQ, name);
  b.first->arglist = args;
  return b;
 }
@@ -275,7 +289,7 @@ block gen_call(const char* name, block args) {


 block gen_subexp(block a) {
-  return BLOCK(gen_op_simple(DUP), a, gen_op_simple(SWAP));
+  return BLOCK(gen_op_simple(SUBEXP_BEGIN), a, gen_op_simple(SUBEXP_END));
 }

 block gen_both(block a, block b) {
@@ -288,50 +302,46 @@ block gen_both(block a, block b) {


 block gen_collect(block expr) {
-  block array_var = block_bind(gen_op_var_unbound(STOREV, "collect"),
-                               gen_noop(), OP_HAS_VARIABLE);
+  block array_var = gen_op_var_fresh(STOREV, "collect");
  block c = BLOCK(gen_op_simple(DUP), gen_const(jv_array()), array_var);

-  block tail = BLOCK(gen_op_simple(DUP),
-                     gen_op_var_bound(LOADV, array_var),
-                     gen_op_simple(SWAP),
-                     gen_op_simple(APPEND),
-                     gen_op_var_bound(STOREV, array_var),
+  block tail = BLOCK(gen_op_bound(APPEND, array_var),
                     gen_op_simple(BACKTRACK));

  return BLOCK(c,
               gen_op_target(FORK, tail),
               expr, 
               tail,
-               gen_op_var_bound(LOADV, array_var));
+               gen_op_bound(LOADVN, array_var));
 }

-block gen_assign(block expr) {
-  block result_var = block_bind(gen_op_var_unbound(STOREV, "result"),
-                                gen_noop(), OP_HAS_VARIABLE);
-
+block gen_reduce(const char* varname, block source, block init, block body) {
+  block res_var = gen_op_var_fresh(STOREV, "reduce");
  block loop = BLOCK(gen_op_simple(DUP),
-                     expr,
-                     gen_op_var_bound(ASSIGN, result_var),
+                     source,
+                     block_bind(gen_op_unbound(STOREV, varname),
+                                BLOCK(gen_op_bound(LOADVN, res_var),
+                                      body,
+                                      gen_op_bound(STOREV, res_var)),
+                                OP_HAS_VARIABLE),
                     gen_op_simple(BACKTRACK));
-
  return BLOCK(gen_op_simple(DUP),
-               result_var,
+               init,
+               res_var,
               gen_op_target(FORK, loop),
               loop,
-               gen_op_var_bound(LOADV, result_var));
+               gen_op_bound(LOADVN, res_var));
 }

 block gen_definedor(block a, block b) {
  // var found := false
-  block found_var = block_bind(gen_op_var_unbound(STOREV, "found"),
-                               gen_noop(), OP_HAS_VARIABLE);
+  block found_var = gen_op_var_fresh(STOREV, "found");
  block init = BLOCK(gen_op_simple(DUP), gen_const(jv_false()), found_var);

  // if found, backtrack. Otherwise execute b
  block backtrack = gen_op_simple(BACKTRACK);
  block tail = BLOCK(gen_op_simple(DUP),
-                     gen_op_var_bound(LOADV, found_var),
+                     gen_op_bound(LOADV, found_var),
                     gen_op_target(JUMP_F, backtrack),
                     backtrack,
                     gen_op_simple(POP),
@@ -343,7 +353,7 @@ block gen_definedor(block a, block b) {
  // found := true, produce result
  block if_found = BLOCK(gen_op_simple(DUP),
                         gen_const(jv_true()),
-                         gen_op_var_bound(STOREV, found_var),
+                         gen_op_bound(STOREV, found_var),
                         gen_op_target(JUMP, tail));

  return BLOCK(init,
@@ -380,16 +390,22 @@ block gen_or(block a, block b) {
                                                   gen_const(jv_false())))));
 }

+block gen_var_binding(block var, const char* name, block body) {
+  return BLOCK(gen_op_simple(DUP), var,
+               block_bind(gen_op_unbound(STOREV, name),
+                          body, OP_HAS_VARIABLE));
+}
+
 block gen_cond(block cond, block iftrue, block iffalse) {
  return BLOCK(gen_op_simple(DUP), cond, 
               gen_condbranch(BLOCK(gen_op_simple(POP), iftrue),
                              BLOCK(gen_op_simple(POP), iffalse)));
 }

-block gen_cbinding(struct symbol_table* t, block code) {
-  for (int cfunc=0; cfunc<t->ncfunctions; cfunc++) {
+block gen_cbinding(const struct cfunction* cfunctions, int ncfunctions, block code) {
+  for (int cfunc=0; cfunc<ncfunctions; cfunc++) {
    inst* i = inst_new(CLOSURE_CREATE_C);
-    i->imm.cfunc = &t->cfunctions[cfunc];
+    i->imm.cfunc = &cfunctions[cfunc];
    i->symbol = strdup(i->imm.cfunc->name);
    code = block_bind(inst_block(i), code, OP_IS_CALL_PSEUDO);
  }
@@ -451,7 +467,7 @@ static int expand_call_arglist(struct locfile* locations, block* b) {
            break;
          case CLOSURE_CREATE:
            block_append(&prelude, b);
-            block_append(&callargs, gen_op_block_bound(CLOSURE_REF, b));
+            block_append(&callargs, gen_op_bound(CLOSURE_REF, b));
            break;
          }
          actual_args++;
@@ -510,13 +526,8 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
  int var_frame_idx = 0;
  bc->nsubfunctions = 0;
  errors += expand_call_arglist(locations, &b);
-  if (bc->parent) {
-    // functions should end in a return
-    b = BLOCK(b, gen_op_simple(RET));
-  } else {
-    // the toplevel should YIELD;BACKTRACK; when it finds an answer
-    b = BLOCK(b, gen_op_simple(YIELD), gen_op_simple(BACKTRACK));
-  }
+  b = BLOCK(b, gen_op_simple(RET));
+  jv localnames = jv_array();
  for (inst* curr = b.first; curr; curr = curr->next) {
    if (!curr->next) assert(curr == b.last);
    int length = opcode_describe(curr->op)->length;
@@ -534,6 +545,7 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
    if ((opcode_describe(curr->op)->flags & OP_HAS_VARIABLE) &&
        curr->bound_by == curr) {
      curr->imm.intval = var_frame_idx++;
+      localnames = jv_array_append(localnames, jv_string(curr->symbol));
    }

    if (curr->op == CLOSURE_CREATE) {
@@ -543,10 +555,13 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
    if (curr->op == CLOSURE_CREATE_C) {
      assert(curr->bound_by == curr);
      int idx = bc->globals->ncfunctions++;
+      bc->globals->cfunc_names = jv_array_append(bc->globals->cfunc_names,
+                                                 jv_string(curr->symbol));
      bc->globals->cfunctions[idx] = *curr->imm.cfunc;
      curr->imm.intval = idx;
    }
  }
+  bc->debuginfo = jv_object_set(bc->debuginfo, jv_string("locals"), localnames);
  if (bc->nsubfunctions) {
    bc->subfunctions = jv_mem_alloc(sizeof(struct bytecode*) * bc->nsubfunctions);
    for (inst* curr = b.first; curr; curr = curr->next) {
@@ -556,12 +571,16 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
        subfn->globals = bc->globals;
        subfn->parent = bc;
        subfn->nclosures = 0;
+        subfn->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_string(curr->symbol));
+        jv params = jv_array();
        for (inst* param = curr->arglist.first; param; param = param->next) {
          assert(param->op == CLOSURE_PARAM);
          assert(param->bound_by == param);
          param->imm.intval = subfn->nclosures++;
          param->compiled = subfn;
+          params = jv_array_append(params, jv_string(param->symbol));
        }
+        subfn->debuginfo = jv_object_set(subfn->debuginfo, jv_string("params"), params);
        errors += compile(locations, subfn, curr->subfn);
        curr->subfn = gen_noop();
      }
@@ -629,6 +648,8 @@ int block_compile(block b, struct locfile* locations, struct bytecode** out) {
  int ncfunc = count_cfunctions(b);
  bc->globals->ncfunctions = 0;
  bc->globals->cfunctions = jv_mem_alloc(sizeof(struct cfunction) * ncfunc);
+  bc->globals->cfunc_names = jv_array();
+  bc->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_null());
  int nerrors = compile(locations, bc, b);
  assert(bc->globals->ncfunctions == ncfunc);
  if (nerrors > 0) {
@@ -2,12 +2,9 @@
 #define COMPILE_H
 #include <stdint.h>
 #include "jv.h"
-#include "opcode.h"
+#include "bytecode.h"
 #include "locfile.h"

-struct bytecode;
-struct symbol_table;
-
 struct inst;
 typedef struct inst inst;

@@ -23,30 +20,34 @@ block gen_noop();
 block gen_op_simple(opcode op);
 block gen_const(jv constant);
 block gen_op_target(opcode op, block target);
-block gen_op_var_unbound(opcode op, const char* name);
-block gen_op_var_bound(opcode op, block binder);
-block gen_op_block_unbound(opcode op, const char* name);
+block gen_op_unbound(opcode op, const char* name);
+block gen_op_bound(opcode op, block binder);
+block gen_op_var_fresh(opcode op, const char* name);

 block gen_function(const char* name, block formals, block body);
+block gen_param(const char* name);
 block gen_lambda(block body);
 block gen_call(const char* name, block body);
 block gen_subexp(block a);
 block gen_both(block a, block b);
 block gen_collect(block expr);
-block gen_assign(block expr);
+block gen_reduce(const char* varname, block source, block init, block body);
 block gen_definedor(block a, block b);
 block gen_condbranch(block iftrue, block iffalse);
 block gen_and(block a, block b);
 block gen_or(block a, block b);

+block gen_var_binding(block var, const char* name, block body);
+
 block gen_cond(block cond, block iftrue, block iffalse);

-block gen_cbinding(struct symbol_table* functions, block b);
+block gen_cbinding(const struct cfunction* functions, int nfunctions, block b);

 void block_append(block* b, block b2);
 block block_join(block a, block b);
 int block_has_only_binders(block, int bindflags);
 block block_bind(block binder, block body, int bindflags);
+block block_bind_referenced(block binder, block body, int bindflags);

 int block_compile(block, struct locfile*, struct bytecode**);

@@ -0,0 +1,8 @@
+config.guess
+config.sub
+compile
+depcomp
+install-sh
+ltmain.sh
+missing
+ylwrap
@@ -0,0 +1,2 @@
+libtool.m4
+lt*.m4
@@ -0,0 +1,74 @@
+AC_INIT([jq], [1.3], [mu@netsoc.tcd.ie],
+             [jq], [http://stedolan.github.com/jq/])
+
+dnl Created autoconf implementation thompson@dtosolutions, 26NOV12
+AC_PREREQ([2.61])
+AC_CONFIG_AUX_DIR([config])
+AM_INIT_AUTOMAKE([parallel-tests foreign])
+AM_SILENT_RULES([yes])
+AC_PROG_CC
+AC_PROG_CC_STDC
+AC_PROG_CPP_WERROR
+AC_PROG_YACC
+AC_OBJEXT
+AC_EXEEXT
+LT_INIT([shared static win32-dll])
+AM_PROG_CC_C_O
+
+dnl couldn't use AM_PROG_LEX as it doesn't support header files like the
+dnl AC_PROG_YACC macros...
+
+dnl
+dnl these program checks should probably be deleted
+dnl
+
+AC_PROG_LEX
+if test "x$LEX" != xflex; then
+  LEX="$SHELL $missing_dir/missing flex"
+fi
+
+
+dnl Check for valgrind
+AC_CHECK_PROGS(valgrind_cmd, valgrind)
+if test "x$valgrind_cmd" = "x" ; then
+    AC_MSG_WARN([valgrind is required to test jq.])
+fi
+
+
+dnl Don't attempt to build docs if there's no Ruby lying around
+AC_ARG_ENABLE([docs],
+   AC_HELP_STRING([--disable-docs], [don't build docs]))
+
+AS_IF([test "x$enable_docs" != "xno"],[
+   AC_CHECK_PROGS(bundle_cmd, bundle)
+
+   AC_CACHE_CHECK([for Ruby dependencies], [jq_cv_ruby_deps],
+     [jq_cv_ruby_deps=yes;
+        AS_IF([test "x$bundle_cmd" = "x" || \
+        ! bmsg="`cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null`"],[
+           AC_MSG_WARN([$bmsg])
+           cat <<EOF
+*****************************************************************
+*  Ruby dependencies for building jq documentation not found.   *
+*  You can still build, install and hack on jq, but the manpage *
+*  will not be rebuilt and some of the tests won't run.         *
+*  See docs/README.md for how to install the docs dependencies. *
+*****************************************************************
+EOF
+           jq_cv_ruby_deps=no
+     ])])
+
+   if test "x$jq_cv_ruby_deps" != "xyes"; then
+     enable_docs=no
+   fi
+])
+
+AM_CONDITIONAL([ENABLE_DOCS], [test "x$enable_docs" != xno])
+AC_SUBST([BUNDLER], ["$bundle_cmd"])
+
+AC_CONFIG_MACRO_DIR([config/m4])
+AM_INIT_AUTOMAKE([-Wall])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+
+
@@ -1,5 +1,6 @@
-source :rubygems
+source "https://rubygems.org"

+gem "rake"
 gem "bonsai"
 gem "maruku"
 gem "ronn"
@@ -1,11 +1,15 @@
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
  specs:
-    activesupport (3.2.9)
+    POpen4 (0.1.4)
+      Platform (>= 0.4.0)
+      open4
+    Platform (0.4.0)
+    activesupport (3.2.12)
      i18n (~> 0.6)
      multi_json (~> 1.0)
-    addressable (2.3.2)
-    bonsai (1.4.7)
+    addressable (2.3.4)
+    bonsai (1.4.8)
      activesupport (>= 3.0.3)
      builder (>= 3.0.0)
      i18n (>= 0.5.0)
@@ -17,32 +21,37 @@ GEM
      sinatra (>= 1.0)
      tilt (>= 1.3)
      watch (>= 0.1.0)
-    builder (3.1.4)
+      yui-compressor
+    builder (3.2.0)
    hpricot (0.8.6)
-    i18n (0.6.1)
-    launchy (2.1.2)
+    i18n (0.6.4)
+    launchy (2.3.0)
      addressable (~> 2.3)
-    liquid (2.4.1)
+    liquid (2.5.0)
    maruku (0.6.1)
      syntax (>= 1.0.0)
-    multi_json (1.5.0)
+    multi_json (1.7.3)
    mustache (0.99.4)
-    rack (1.4.1)
-    rack-protection (1.3.2)
+    open4 (1.3.0)
+    rack (1.5.2)
+    rack-protection (1.5.0)
      rack
-    rdiscount (1.6.8)
+    rake (10.0.3)
+    rdiscount (2.0.7.2)
    ronn (0.7.3)
      hpricot (>= 0.8.2)
      mustache (>= 0.7.0)
      rdiscount (>= 1.5.8)
-    sass (3.2.3)
-    sinatra (1.3.3)
-      rack (~> 1.3, >= 1.3.6)
-      rack-protection (~> 1.2)
-      tilt (~> 1.3, >= 1.3.3)
+    sass (3.2.8)
+    sinatra (1.4.2)
+      rack (~> 1.5, >= 1.5.2)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
    syntax (1.0.0)
-    tilt (1.3.3)
+    tilt (1.4.0)
    watch (0.1.0)
+    yui-compressor (0.9.6)
+      POpen4 (>= 0.1.4)

 PLATFORMS
  ruby
@@ -50,4 +59,5 @@ PLATFORMS
 DEPENDENCIES
  bonsai
  maruku
+  rake
  ronn
@@ -0,0 +1,25 @@
+Documentation
+=============
+
+The jq website, manpages and some of the tests are generated from this
+directory. The directory holds a [Bonsai](http://tinytree.info)
+website, and the manual is a YAML file in `content/3.manual`.
+
+To build the documentation (including building the jq manpage), you'll
+need a working Ruby setup. The easiest way to get one is to install
+RVM and Ruby 1.9.3 like so:
+
+    \curl -L https://get.rvm.io | bash -s stable --ruby=1.9.3
+
+After that finishes installing, you'll need to make sure RVM is on
+your path by doing `source $HOME/.rvm/scripts/rvm`, or just opening a
+new shell. See <http://rvm.io> for more info on RVM.
+
+Once RVM is installed, you can install all the dependencies for jq's
+documentation build by running this from the `docs` directory:
+
+    bundle install
+
+When bundle manages to install the dependencies, rerun `./configure`
+in the jq root directory and then the Makefile will be able to
+generate the jq manpage.
@@ -1,9 +1,11 @@
+require 'yaml'
 require 'bonsai'
 require 'liquid'
 require 'maruku'
 require 'json'
 require 'ronn'
 require 'tempfile'
+require 'yaml'

 module ExtraFilters
  def markdownify(input)
@@ -46,7 +48,7 @@ task :serve do
        run Bonsai::DevelopmentServer
      }
      Rack::Handler.default.run(app, :Port => 5000) do
-        Launchy.open("http://localhost:5000/jq")
+        Launchy.open("http://localhost:5000/jq/")
      end
    }    
    Watch.new("{content,templates,public}/**/*") { Bonsai::Exporter.process! }
@@ -62,6 +64,28 @@ task :build do
  Bonsai::Exporter.publish!
 end

+$BINARIES = {
+  :osx32 => "--host=i686-apple-darwin10 CFLAGS='-m32 -g -O'",
+  :osx64 => "--host=i686-apple-darwin10 CFLAGS='-m64 -g -O'",
+  :win32 => "--host=i686-w64-mingw32 CFLAGS='-g -O'",
+  :win64 => "--host=x86_64-w64-mingw32 CFLAGS='-g -O'",
+  :linux32 => "--host=x86_64-linux-gnu CFLAGS='-m32 -g -O'",
+  :linux64 => "--host=x86_64-linux-gnu CFLAGS='-m64 -g -O'"
+}
+
+$BINARIES.each do |name, args|
+  file "../build/#{name}" do |t|
+    sh "MAKEFLAGS=-j4 ../scripts/crosscompile #{name} #{args}"
+  end
+  task :binaries => ["../build/#{name}"]
+end
+
+task :binaries => ["output/download"] do
+  $BINARIES.each do |name, args|
+    FileUtils.cp_r "../build/#{name}", "output/download/"
+  end
+end
+
 def load_manual
  YAML::ENGINE.yamler = 'syck'
  YAML::load(File.open("content/3.manual/manual.yml"))
@@ -96,6 +120,10 @@ task :manpage do
  end  
 end

+task :manpage_default => ["default_manpage.md"] do
+  puts Ronn::Document.new("default_manpage.md").convert('roff').gsub(/<\/?code>/,"")
+end
+
 task :mantests do
  load_manual['sections'].each do |section|
    (section['entries'] || []).each do |entry|
@@ -108,3 +136,11 @@ task :mantests do
    end
  end
 end
+
+directory "output/download/source"
+task :tarball => ["output/download/source"] do
+  sh "cd ..; ./configure && make dist && make distclean"
+  sh "cp ../jq-*.tar.gz output/download/source"
+end
+
+task :dist => [:build, :binaries, :tarball]
@@ -237,5 +237,5 @@ body:
      - - -

      Here endeth the tutorial! There's lots more to play with, go
-      read [the manual](../manual) if you're interested, and [download
-      jq](../download) if you haven't already.
+      read [the manual](../manual/) if you're interested, and [download
+      jq](../download/) if you haven't already.
@@ -27,7 +27,7 @@ body:
         manually by following instructions on [Arch Linux's
         Wiki](https://wiki.archlinux.org/index.php/Arch_UseRepository).

-       * Debian/Ubuntu packages are coming Real Soon Now.
+       * jq 1.2 is [in Debian](http://packages.debian.org/jq)

      ### OS X

@@ -38,22 +38,32 @@ body:

      ### Windows

-       * Executables for [64-bit](win64/jq) or [32-bit](win32/jq)
+       * Executables for [64-bit](win64/jq.exe) or [32-bit](win32/jq.exe)

      ### From source on Linux or OS X

+       * [Source tarball for jq $JQ_VERSION](source/jq-$JQ_VERSION.tar.gz)
+
+      You can build it using the usual `./configure && make && sudo
+      make install` rigmarole.
+
+      If you're interested in using the lastest development version, try:
+
          git clone https://github.com/stedolan/jq.git
          cd jq
-          make && sudo make install
+          autoreconf
+          ./configure
+          make
+          sudo make install

      To build it from a git clone, you'll need to install a few
      packages first:

       * [Flex](http://www.gnu.org/software/flex/)
       * [Bison](http://www.gnu.org/software/bison/)
-       * [Python](http://www.python.org)
       * [GCC](http://gcc.gnu.org)
       * [Make](http://www.gnu.org/software/make)
+       * [Autotools](http://www.gnu.org/software/automake)

      For Linux systems, these will all be in your system's package
      manager, and if you do development on the machine they're most
@@ -73,6 +83,8 @@ body:

      jq's documentation is compiled into static HTML using
      [Bonsai](http://www.tinytree.info). To view the documentation
-      locally, run `rake serve` from the docs/ subdirectory.
+      locally, run `rake serve` from the docs/ subdirectory. You'll
+      need a few Ruby dependencies, which can be installed by
+      following the instructions in `docs/README.md`.


@@ -75,51 +75,69 @@ sections:
      You can affect how jq reads and writes its input and output
      using some command-line options:

-        * `--slurp`/`-s`:
+      * `--slurp`/`-s`:
  
-          Instead of running the filter for each JSON object in the
-          input, read the entire input stream into a large array and run
-          the filter just once.
+        Instead of running the filter for each JSON object in the
+        input, read the entire input stream into a large array and run
+        the filter just once.
+      
+      * `--raw-input`/`-R`:
+      
+        Don't parse the input as JSON. Instead, each line of text is
+        passed to the filter as a string. If combined with `--slurp`,
+        then the entire input is passed to the filter as a single long
+        string.
+  
+      * `--null-input`/`-n`:
+  
+        Don't read any input at all! Instead, the filter is run once
+        using `null` as the input. This is useful when using jq as a
+        simple calculator or to construct JSON data from scratch.
+  
+      * `--compact-output` / `-c`:
+  
+        By default, jq pretty-prints JSON output. Using this option
+        will result in more compact output by instead putting each
+        JSON object on a single line.
+  
+      * `--colour-output` / `-C` and `--monochrome-output` / `-M`:
+      
+        By default, jq outputs colored JSON if writing to a
+        terminal. You can force it to produce color even if writing to
+        a pipe or a file using `-C`, and disable color with `-M`.
+  
+      * `--ascii-output` / `-a`:
+  
+        jq usually outputs non-ASCII Unicode codepoints as UTF-8, even
+        if the input specified them as escape sequences (like
+        "\u03bc"). Using this option, you can force jq to produce pure
+        ASCII output with every non-ASCII character replaced with the
+        equivalent escape sequence.
+
+      * `--sort-keys` / `-S`:
        
-        * `--raw-input`/`-R`:
-        
-          Don't parse the input as JSON. Instead, each line of text is
-          passed to the filter as a string. If combined with `--slurp`,
-          then the entire input is passed to the filter as a single long
-          string.
+        Output the fields of each object with the keys in sorted order.
+
+      * `--raw-output` / `-r`:
  
-        * `--null-input`/`-n`:
-  
-          Don't read any input at all! Instead, the filter is run once
-          using `null` as the input. This is useful when using jq as a
-          simple calculator or to construct JSON data from scratch.
-  
-        * `--compact-output` / `-c`:
-  
-          By default, jq pretty-prints JSON output. Using this option
-          will result in more compact output by instead putting each
-          JSON object on a single line.
-  
-        * `--colour-output` / `-C` and `--monochrome-output` / `-M`:
-        
-          By default, jq outputs colored JSON if writing to a
-          terminal. You can force it to produce color even if writing to
-          a pipe or a file using `-C`, and disable color with `-M`.
-  
-        * `--ascii-output` / `-a`:
-  
-          jq usually outputs non-ASCII Unicode codepoints as UTF-8, even
-          if the input specified them as escape sequences (like
-          "\u03bc"). Using this option, you can force jq to produce pure
-          ASCII output with every non-ASCII character replaced with the
-          equivalent escape sequence.
-  
-        * `--raw-output` / `-r`:
-  
-          With this option, if the filter's result is a string then it
-          will be written directly to standard output rather than being
-          formatted as a JSON string with quotes. This can be useful for
-          making jq filters talk to non-JSON-based systems.
+        With this option, if the filter's result is a string then it
+        will be written directly to standard output rather than being
+        formatted as a JSON string with quotes. This can be useful for
+        making jq filters talk to non-JSON-based systems.
+
+      * `--arg name value`:
+
+        This option passes a value to the jq program as a predefined
+        variable. If you run jq with `--arg foo bar`, then `$foo` is
+        available in the program and has the value `"bar"`.
+
+      * `--argfile name filename`:
+
+        This option passes the first value from the named file as a
+        value to the jq program as a predefined variable. If you run jq
+        with `--argfile foo bar`, then `$foo` is available in the
+        program and has the value resulting from parsing the content of
+        the file named `bar`.

  - title: Basic filters
    entries:
@@ -142,10 +160,13 @@ sections:
      - title: "`.foo`"
        body: |
          
-          The simplest *useful* filter is .foo. When given a
+          The simplest *useful* filter is `.foo`. When given a
          JSON object (aka dictionary or hash) as input, it produces
-          the value at the key "foo", or null if there\'s none present.
+          the value at the key "foo", or null if there's none present.

+          If the key contains special characters, you need to surround
+          it with double quotes like this: `."foo$"`.
+          
        examples:
          - program: '.foo'
            input: '{"foo": 42, "bar": "less interesting data"}'
@@ -153,16 +174,26 @@ sections:
          - program: '.foo'
            input: '{"notfoo": true, "alsonotfoo": false}'
            output: ['null']
+          - program: '."foo"'
+            input: '{"foo": 42}'
+            output: [42]

-      - title: "`.[foo]`"
+      - title: "`.[foo]`, `.[2]`, `.[10:15]`"
        body: |
          
          You can also look up fields of an object using syntax like
          `.["foo"]` (.foo above is a shorthand version of this). This
          one works for arrays as well, if the key is an
-          integer. Arrays are zero-based (like javascript), so .[2]
+          integer. Arrays are zero-based (like javascript), so `.[2]`
          returns the third element of the array.

+          The `.[10:15]` syntax can be used to return a subarray of an
+          array. The array returned by `.[10:15]` will be of length 5,
+          containing the elements from index 10 (inclusive) to index
+          15 (exclusive). Either index may be negative (in which case
+          it counts backwards from the end of the array), or omitted
+          (in which case it refers to the start or end of the array).
+
        examples:
          - program: '.[0]'
            input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
@@ -172,15 +203,30 @@ sections:
            input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
            output: ['null']

+          - program: '.[2:4]'
+            input: '["a","b","c","d","e"]'
+            output: ['["c", "d"]']
+          
+          - program: '.[:3]'
+            input: '["a","b","c","d","e"]'
+            output: ['["a", "b", "c"]']
+
+          - program: '.[-2:]'
+            input: '["a","b","c","d","e"]'
+            output: ['["d", "e"]']
+
      - title: "`.[]`"
        body: |
          
          If you use the `.[foo]` syntax, but omit the index
          entirely, it will return *all* of the elements of an
          array. Running `.[]` with the input `[1,2,3]` will produce the
-          numbers as three seperate results, rather than as a single
+          numbers as three separate results, rather than as a single
          array.

+          You can also use this on an object, and it will return all
+          the values of the object.
+
        examples:
          - program: '.[]'
            input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
@@ -192,6 +238,10 @@ sections:
            input: '[]'
            output: []

+          - program: '.[]'
+            input: '{"a": 1, "b": 1}'
+            output: ['1', '1']
+
      - title: "`,`"
        body: |
          
@@ -218,8 +268,8 @@ sections:
      - title: "`|`"
        body: |
          The | operator combines two filters by feeding the output(s) of
-          the one on the left into the input of the one on the right. It\'s
-          pretty much the same as the Unix shell\'s pipe, if you\'re used to
+          the one on the left into the input of the one on the right. It's
+          pretty much the same as the Unix shell's pipe, if you're used to
          that. 

          If the one on the left produces multiple results, the one on
@@ -257,8 +307,8 @@ sections:
          in `[.foo, .bar, .baz]`) or to "collect" all the results of a
          filter into an array (as in `[.items[].name]`)
        
-          Once you understand the "," operator, you can look at jq\'s array
-          syntax in a different light: the expression [1,2,3] is not using a
+          Once you understand the "," operator, you can look at jq's array
+          syntax in a different light: the expression `[1,2,3]` is not using a
          built-in syntax for comma-separated arrays, but is instead applying
          the `[]` operator (collect results) to the expression 1,2,3 (which
          produces three different results).
@@ -279,8 +329,8 @@ sections:
          
          If the keys are "sensible" (all alphabetic characters), then
          the quotes can be left off. The value can be any expression
-          (although you may need to wrap it in parentheses if it\'s a
-          complicated one), which gets applied to the {} expression\'s
+          (although you may need to wrap it in parentheses if it's a
+          complicated one), which gets applied to the {} expression's
          input (remember, all filters have an input and an
          output).
          
@@ -294,10 +344,10 @@ sections:
          
              {user: .user, title: .title}
          
-          Because that\'s so common, there\'s a shortcut syntax: `{user, title}`.
+          Because that's so common, there's a shortcut syntax: `{user, title}`.
          
          If one of the expressions produces multiple results,
-          multiple dictionaries will be produced. If the input\'s
+          multiple dictionaries will be produced. If the input's
          
              {"user":"stedolan","titles":["JQ Primer", "More JQ"]}
          
@@ -356,6 +406,9 @@ sections:
              the key-value pairs from both objects into a single
              combined object. If both objects contain a value for the
              same key, the object on the right of the `+` wins.
+          
+          `null` can be added to any value, and returns the other
+          value unchanged.

        examples:
          - program: '.a + 1'
@@ -364,6 +417,12 @@ sections:
          - program: '.a + .b'
            input: '{"a": [1,2], "b": [3,4]}'
            output: ['[1,2,3,4]']
+          - program: '.a + null'
+            input: '{"a": 1}'
+            output: ['1']
+          - program: '.a + 1'
+            input: '{}'
+            output: ['1']
          - program: '{a: 1} + {b: 2} + {c: 3} + {a: 42}'
            input: 'null'
            output: ['{"a": 42, "b": 2, "c": 3}']
@@ -437,6 +496,50 @@ sections:
            input: '[42,3,35]'
            output: ['[0,1,2]']

+      - title: `has`
+        body: |
+          
+          The builtin function `has` returns whether the input object
+          has the given key, or the input array has an element at the
+          given index.
+
+          `has($key)` has the same effect as checking whether `$key`
+          is a member of the array returned by `keys`, although `has`
+          will be faster.
+
+        examples:
+          - program: 'map(has("foo"))'
+            input: '[{"foo": 42}, {}]'
+            output: ['[true, false]']
+          - program: 'map(has(2))'
+            input: '[[0,1], ["a","b","c"]]'
+            output: ['[false, true]']
+
+      - title: `to_entries`, `from_entries`, `with_entries`
+        body: |
+          
+          These functions convert between an object and an array of
+          key-value pairs. If `to_entries` is passed an object, then
+          for each `k: v` entry in the input, the output array
+          includes `{"key": k, "value": v}`. 
+
+          `from_entries` does the opposite conversion, and
+          `with_entries(foo)` is a shorthand for `to_entries |
+          map(foo) | from_entries`, useful for doing some operation to
+          all keys and values of an object.
+
+        examples:
+          - program: 'to_entries'
+            input: '{"a": 1, "b": 2}'
+            output: ['[{"key":"a", "value":1}, {"key":"b", "value":2}]']
+          - program: 'from_entries'
+            input: '[{"key":"a", "value":1}, {"key":"b", "value":2}]'
+            output: ['{"a": 1, "b": 2}']
+          - program: 'with_entries(.key |= "KEY_" + .)'
+            input: '{"a": 1, "b": 2}'
+            output: ['{"KEY_a": 1, "KEY_b": 2}']
+
+
      - title: `select`
        body: |
          
@@ -503,6 +606,22 @@ sections:
          - program: add
            input: '[]'
            output: ["null"]
+
+      - title: `range`
+        body: |
+          
+          The `range` function produces a range of numbers. `range(4;10)`
+          produces 6 numbers, from 4 (inclusive) to 10 (exclusive). The numbers
+          are produced as separate outputs. Use `[range(4;10)]` to get a range as 
+          an array.
+
+        examples:
+          - program: 'range(2;4)'
+            input: 'null'
+            output: ['2', '3']
+          - program: '[range(2;4)]'
+            input: 'null'
+            output: ['[2,3]']
            
      - title: `tonumber`
        body: |
@@ -528,6 +647,18 @@ sections:
            input: '[1, "1", [1]]'
            output: ['"1"', '"1"', '"[1]"']

+      - title: `type`
+        body: |
+          
+          The `type` function returns the type of its argument as a
+          string, which is one of null, boolean, number, string, array
+          or object.
+
+        examples:
+          - program: 'map(type)'
+            input: '[0, false, [], {}, null, "hello"]'
+            output: ['["number", "boolean", "array", "object", "null", "string"]']
+
      - title: `sort, sort_by`
        body: |
          
@@ -605,6 +736,16 @@ sections:
            input: '[1,2,5,3,5,3,1,3]'
            output: ['[1,2,3,5]']

+      - title: `reverse`
+        body: |
+          
+          This function reverses an array.
+
+        examples:
+          - program: 'reverse'
+            input: '[1,2,3,4]'
+            output: ['[4,3,2,1]']
+
      - title: `contains`
        body: |

@@ -634,6 +775,38 @@ sections:
            input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
            output: ['false']

+      - title: `recurse`
+        body: |
+          
+          The `recurse` function allows you to search through a
+          recursive structure, and extract interesting data from all
+          levels. Suppose your input represents a filesystem:
+
+              {"name": "/", "children": [
+                {"name": "/bin", "children": [
+                  {"name": "/bin/ls", "children": []},
+                  {"name": "/bin/sh", "children": []}]},
+                {"name": "/home", "children": [
+                  {"name": "/home/stephen", "children": [
+                    {"name": "/home/stephen/jq", "children": []}]}]}]}
+          
+          Now suppose you want to extract all of the filenames
+          present. You need to retrieve `.name`, `.children[].name`,
+          `.children[].children[].name`, and so on. You can do this
+          with:
+
+              recurse(.children[]) | .name
+
+        examples:
+          - program: 'recurse(.foo[])'
+            input: '{"foo":[{"foo": []}, {"foo":[{"foo":[]}]}]}'
+            output: 
+              - '{"foo":[{"foo":[]},{"foo":[{"foo":[]}]}]}'
+              - '{"foo":[]}'
+              - '{"foo":[{"foo":[]}]}'
+              - '{"foo":[]}'
+
+
      - title: "String interpolation - `\(foo)`"
        body: |

@@ -646,10 +819,77 @@ sections:
            input: '42'
            output: ['"The input was 42, which is one less than 43"']
          
-              
+      - title: "Format strings and escaping"
+        body: |

-              
+          The `@foo` syntax is used to format and escape strings,
+          which is useful for building URLs, documents in a language
+          like HTML or XML, and so forth. `@foo` can be used as a
+          filter on its own, the possible escapings are:

+          * `@text`:
+
+            Calls `tostring`, see that function for details.
+
+          * `@json`:
+
+            Serialises the input as JSON.
+
+          * `@html`:
+
+            Applies HTML/XML escaping, by mapping the characters
+            `<>&'"` to their entity equivalents `&lt;`, `&gt;`,
+            `&amp;`, `&apos;`, `&quot;`.
+
+          * `@uri`:
+
+            Applies percent-encoding, by mapping all reserved URI
+            characters to a `%xx` sequence.
+
+          * `@csv`:
+            
+            The input must be an array, and it is rendered as CSV
+            with double quotes for strings, and quotes escaped by
+            repetition.
+
+          * `@sh`:
+            
+            The input is escaped suitable for use in a command-line
+            for a POSIX shell. If the input is an array, the output
+            will be a series of space-separated strings.
+            
+          * `@base64`:
+
+            The input is converted to base64 as specified by RFC 4648.
+
+          This syntax can be combined with string interpolation in a
+          useful way. You can follow a `@foo` token with a string
+          literal. The contents of the string literal will *not* be
+          escaped. However, all interpolations made inside that string
+          literal will be escaped. For instance,
+
+              @uri "http://www.google.com/search?q=\(.search)"
+
+          will produce the following output for the input
+          `{"search":"jq!"}`:
+
+              http://www.google.com/search?q=jq%21
+
+          Note that the slashes, question mark, etc. in the URL are
+          not escaped, as they were part of the string literal.
+
+        examples:
+          - program: '@html'
+            input: '"This works if x < y"'
+            output: ['"This works if x &lt; y"']
+
+#          - program: '@html "<span>Anonymous said: \(.)</span>"'
+#            input: '"<script>alert(\"lol hax\");</script>"'
+#            output: ["<span>Anonymous said: &lt;script&gt;alert(&quot;lol hax&quot;);&lt;/script&gt;</span>"]
+
+          - program: '@sh "echo \(.)"'
+            input: "\"O'Hara's Ale\""
+            output: ["\"echo 'O'\\\\''Hara'\\\\''s Ale'\""]
        
  - title: Conditionals and Comparisons
    entries:
@@ -681,8 +921,8 @@ sections:
          means that you'll sometimes have to be more explicit about
          the condition you want: you can't test whether, e.g. a
          string is empty using `if .name then A else B end`, you'll
-          need something more like 'if (.name | count) > 0 then A else
-          B end' instead.
+          need something more like `if (.name | count) > 0 then A else
+          B end` instead.
          
          If the condition A produces multiple results, it is
          considered "true" if any of those results is not false or
@@ -746,9 +986,12 @@ sections:
          - program: '(true, false) or false'
            input: 'null'
            output: ['true', 'false']
-          - program: '(true, false) and (true, false)'
+#          - program: '(true, false) and (true, false)'
+#            input: 'null'
+#            output: ['true', 'false', 'false', 'false']
+          - program: '(true, true) and (true, false)'
            input: 'null'
-            output: ['true', 'false', 'false', 'false']
+            output: ['true', 'false', 'true', 'false']
          - program: '[true, false | not]'
            input: 'null'
            output: ['[false, true]']
@@ -774,7 +1017,7 @@ sections:
            input: '{}'
            output: [42]
          
-  - title: Variables and Functions
+  - title: Advanced features
    body: |
      Variables are an absolute necessity in most programming languages, but
      they're relegated to an "advanced feature" in jq.
@@ -791,6 +1034,10 @@ sections:
      (many jq functions such as `map` and `find` are in fact written
      in jq).

+      Finally, jq has a `reduce` operation, which is very powerful but a
+      bit tricky. Again, it's mostly used internally, to define some
+      useful bits of jq's standard library.
+
    entries:
      - title: Variables
        body: |
@@ -805,8 +1052,8 @@ sections:
          For instance, calculating the average value of an array of numbers
          requires a few variables in most languages - at least one to hold the
          array, perhaps one for each element or for a loop counter. In jq, it's
-          simply `add / length` - the `sum` expression is given the array and
-          produces its sum, and the `count` expression is given the array and
+          simply `add / length` - the `add` expression is given the array and
+          produces its sum, and the `length` expression is given the array and
          produces its length.
          
          So, there's generally a cleaner way to solve most problems in jq that
@@ -841,10 +1088,11 @@ sections:
          
              .realnames as $names | .posts[] | {title, author: $names[.author]}
          
-          The expression "foo as $x" runs foo, puts the result in $x,
-          and returns the original input. Apart from the side-effect
-          of binding the variable, it has the same effect as ".".
-          
+          The expression `exp as $x | ...` means: for each value of expression
+          `exp`, run the rest of the pipeline with the entire original input, and
+          with `$x` set to that value.  Thus `as` functions as something of a 
+          foreach loop.
+
          Variables are scoped over the rest of the expression that defines
          them, so 
          
@@ -883,7 +1131,7 @@ sections:
          If you want the value-argument behaviour for defining simple
          functions, you can just use a variable:

-              def addvalue(f): f as $value | map(, + $value);
+              def addvalue(f): f as $value | map(. + $value);

          With that definition, `addvalue(.foo)` will add the current
          input's `.foo` field to each element of the array.
@@ -895,6 +1143,29 @@ sections:
          - program: 'def addvalue(f): f as $x | map(. + $x); addvalue(.[0])'
            input: '[[1,2],[10,20]]'
            output: ['[[1,2,1,2], [10,20,1,2]]']
+
+      - title: Reduce
+        body: |
+        
+          The `reduce` syntax in jq allows you to combine all of the
+          results of an expression by accumulating them into a single
+          answer. As an example, we'll pass `[3,2,1]` to this expression:
+
+              reduce .[] as $item (0; . + $item)
+          
+          For each result that `.[]` produces, `. + $item` is run to
+          accumulate a running total, starting from 0. In this
+          example, `.[]` produces the results 3, 2, and 1, so the
+          effect is similar to running something like this:
+
+              0 | (3 as $item | . + $item) | 
+                  (2 as $item | . + $item) |
+                  (1 as $item | . + $item)
+
+        examples:
+          - program: 'reduce .[] as $item (0; . + $item)'
+            input: '[10,2,5,3]'
+            output: ['20']
      

  - title: Assignment
@@ -23,5 +23,12 @@ body3: |
  shorter and simpler than you'd expect.
  
 tail: |
-  Go read the [tutorial](tutorial) for more, or the [manual](manual)
-  for *way* more.
+  Go read the [tutorial](tutorial/) for more, or the [manual](manual/)
+  for *way* more.
+
+news:
+  - date: 2013-05-19
+    body: |
+      
+      jq 1.3 (finally) released! Get it on the [download](download/) page.
+
@@ -0,0 +1,22 @@
+jq(1) -- Command-line JSON processor
+====================================
+
+## DESCRIPTION
+
+`jq` can transform JSON in various ways, by selecting, iterating,
+reducing and otherwise mangling JSON documents. 
+
+This version of `jq` was built without a manual, so this manpage is a
+stub. For full documentation of the `jq` language, see:
+
+    http://stedolan.github.com/jq
+    
+## BUGS
+
+Presumably. Report them or discuss them at:
+
+    https://github.com/stedolan/jq/issues
+
+## AUTHOR
+
+Stephen Dolan `<mu@netsoc.tcd.ie>`
@@ -89,4 +89,11 @@ h3 code {

 footer {
    text-align: center;
+}
+
+#news {
+    font-size: 12pt;
+    .date {
+          font-style: italic;
+    }
 }
@@ -1,7 +1,11 @@
 # The key value pairs found below are available within the templates.

+:url: http://stedolan.github.io/jq
+
+# This line is modified by the Makefile. To change the version number,
+# edit the Autoconf version number at the top of configure.ac
+jq_version: "1.3"

-jq_version: 1.2
 root: '/jq'
 footer: |
  
@@ -11,4 +15,4 @@ footer: |

  jq is licensed under the MIT license (code) and the
  [CC-BY-3.0](http://creativecommons.org/licenses/by/3.0/) license
-  (docs).
+  (docs).
@@ -10,7 +10,7 @@
        <h1>{{headline}}</h1>
        {% for item in body %}
          {% if item.text %}
-            {{ item.text | markdownify }}
+            {{ item.text | replace: '$JQ_VERSION', jq_version | markdownify }}
          {% endif %}
        
          {% if item.command %}
@@ -18,7 +18,7 @@
              <ul class="dropdown-menu">
                <li><a href="{{root}}/download/linux64/jq">Linux (64-bit)</a></li>
                <li><a href="{{root}}/download/osx64/jq">OS X (64-bit)</a></li>
-                <li><a href="{{root}}/download">Other platforms and source</a></li>
+                <li><a href="{{root}}/download/">Other platforms and source</a></li>
              </ul>
            </div>

@@ -35,6 +35,16 @@
      <div class="row" style="text-align:center; margin-top: 30px">
        {{tail | markdownify}}
      </div>
+      <div class="row">
+        <h2>News</h2>
+        <div id="news">
+          <ul>
+            {% for item in news %}
+            <li><span class="date">{{ item.date | date: '%m %B %Y' }}</span>{{item.body | markdownify}}</li>
+            {% endfor %}
+          </ul>
+        </div>
+      </div>
    </div>
    {% include "shared/footer" %}    
  </body>
@@ -0,0 +1,82 @@
+#ifndef EXEC_STACK_H
+#define EXEC_STACK_H
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct determine_alignment {
+  char x;
+  union { int i; double d; uint64_t u64; size_t sz; void* ptr; } u;
+};
+enum {ALIGNMENT = offsetof(struct determine_alignment, u)};
+
+static size_t align_round_up(size_t sz) {
+  return ((sz + (ALIGNMENT - 1)) / ALIGNMENT) * ALIGNMENT;
+}
+
+typedef int stack_ptr;
+
+struct stack {
+  char* mem_end; // one-past-the-end of allocated region
+  stack_ptr bound;
+  stack_ptr limit; // 0 - stack is empty
+};
+
+static void stack_init(struct stack* s) {
+  s->mem_end = 0;
+  s->bound = ALIGNMENT;
+  s->limit = 0;
+}
+
+static void stack_reset(struct stack* s) {
+  assert(s->limit == 0 && "stack freed while not empty");
+  char* mem_start = s->mem_end - ( -s->bound + ALIGNMENT);
+  free(mem_start);
+  stack_init(s);
+}
+
+static int stack_pop_will_free(struct stack* s, stack_ptr p) {
+  return p == s->limit;
+}
+
+static void* stack_block(struct stack* s, stack_ptr p) {
+  return (void*)(s->mem_end + p);
+}
+
+static stack_ptr* stack_block_next(struct stack* s, stack_ptr p) {
+  return &((stack_ptr*)stack_block(s, p))[-1];
+}
+
+static void stack_reallocate(struct stack* s, size_t sz) {
+  int old_mem_length = -(s->bound) + ALIGNMENT;
+  char* old_mem_start = s->mem_end - old_mem_length;
+  
+  int new_mem_length = align_round_up((old_mem_length + sz + 256) * 2);
+  char* new_mem_start = realloc(old_mem_start, new_mem_length);
+  memmove(new_mem_start + (new_mem_length - old_mem_length),
+            new_mem_start, old_mem_length);
+  s->mem_end = new_mem_start + new_mem_length;
+  s->bound = -(new_mem_length - ALIGNMENT);
+}
+
+static stack_ptr stack_push_block(struct stack* s, stack_ptr p, size_t sz) {
+  int alloc_sz = align_round_up(sz) + ALIGNMENT;
+  stack_ptr r = s->limit - alloc_sz;
+  if (r < s->bound) {
+    stack_reallocate(s, alloc_sz);
+  }
+  s->limit = r;
+  *stack_block_next(s, r) = p;
+  return r;
+}
+
+static stack_ptr stack_pop_block(struct stack* s, stack_ptr p, size_t sz) {
+  stack_ptr r = *stack_block_next(s, p);
+  if (p == s->limit) {
+    int alloc_sz = align_round_up(sz) + ALIGNMENT;
+    s->limit += alloc_sz;
+  }
+  return r;
+}
+#endif
@@ -1,13 +0,0 @@
-#ifndef EXECUTE_H
-#define EXECUTE_H
-#include "bytecode.h"
-
-
-struct bytecode* jq_compile(const char* str);
-
-void jq_init(struct bytecode* bc, jv value);
-jv jq_next();
-void jq_teardown();
-
-
-#endif
@@ -1,163 +0,0 @@
-#ifndef FORKABLE_STACK_H
-#define FORKABLE_STACK_H
-#include <stddef.h>
-#include <assert.h>
-#include <string.h>
-#include "jv_alloc.h"
-
-struct forkable_stack_header {
-  /* distance in bytes between this header and the header
-     of the next object on the stack */
-  int next_delta;
-};
-
-#define FORKABLE_STACK_HEADER struct forkable_stack_header fk_header_
-
-struct forkable_stack {
-  // Stack grows down from stk+length
-  char* stk;
-
-  // stk+length is just past end of allocated area
-  // stk = jv_mem_alloc(length)
-  int length;
-
-  // stk+pos is header of top object, or stk+length if empty
-  int pos;
-
-  // everything after-or-including stk+savedlimit must be preserved
-  int savedlimit;
-};
-
-static void forkable_stack_check(struct forkable_stack* s) {
-  assert(s->stk);
-  assert(s->length > 0);
-  assert(s->pos >= 0 && s->pos <= s->length);
-  assert(s->savedlimit >= 0 && s->savedlimit <= s->length);
-}
-
-static int forkable_stack_empty(struct forkable_stack* s) {
-  return s->pos == s->length;
-}
-
-static void forkable_stack_init(struct forkable_stack* s, size_t sz) {
-  s->stk = jv_mem_alloc(sz);
-  s->length = sz;
-  s->pos = s->length;
-  s->savedlimit = s->length;
-  forkable_stack_check(s);
-}
-
-static void forkable_stack_free(struct forkable_stack* s) {
-  assert(forkable_stack_empty(s));
-  assert(s->savedlimit == s->length);
-  jv_mem_free(s->stk);
-  s->stk = 0;
-}
-
-static void* forkable_stack_push(struct forkable_stack* s, size_t sz_size) {
-  assert(sz_size > 0 && sz_size % sizeof(struct forkable_stack_header) == 0);
-  int size = (int)sz_size;
-  forkable_stack_check(s);
-  int curr = s->pos < s->savedlimit ? s->pos : s->savedlimit;
-  if (curr - size < 0) {
-    int oldlen = s->length;
-    s->length = (size + oldlen + 1024) * 2;
-    s->stk = jv_mem_realloc(s->stk, s->length);
-    int shift = s->length - oldlen;
-    memmove(s->stk + shift, s->stk, oldlen);
-    s->pos += shift;
-    s->savedlimit += shift;
-    curr += shift;
-  }
-  int newpos = curr - size;
-  assert(newpos < s->pos);
-  void* ret = (void*)(s->stk + newpos);
-  ((struct forkable_stack_header*)ret)->next_delta = s->pos - newpos;
-  s->pos = newpos;
-  forkable_stack_check(s);
-  return ret;
-}
-
-static void* forkable_stack_peek(struct forkable_stack* s) {
-  assert(!forkable_stack_empty(s));
-  forkable_stack_check(s);
-  return (void*)(s->stk + s->pos);
-}
-
-static void* forkable_stack_peek_next(struct forkable_stack* s, void* top) {
-  forkable_stack_check(s);
-  struct forkable_stack_header* elem = top;
-  int elempos = (char*)top - s->stk;
-  assert(elempos >= 0 && elempos < s->length);
-  assert(elempos + elem->next_delta <= s->length);
-  if (elempos + elem->next_delta < s->length) {
-    return (void*)(s->stk + elempos + elem->next_delta);
-  } else {
-    return 0;
-  }
-}
-
-// Returns 1 if the next forkable_stack_pop will permanently remove an
-// object from the stack (i.e. the top object was not saved with a fork)
-static int forkable_stack_pop_will_free(struct forkable_stack* s) {
-  return s->pos < s->savedlimit ? 1 : 0;
-}
-
-static void forkable_stack_pop(struct forkable_stack* s) {
-  forkable_stack_check(s);
-  struct forkable_stack_header* elem = forkable_stack_peek(s);
-  s->pos += elem->next_delta;
-}
-
-
-
-struct forkable_stack_state {
-  // We save the previous pos, savedlimit as
-  // length-pos, length-savedlimit since these
-  // values are stable across stack reallocations.
-  int prev_pos_delta, prev_limit_delta;
-};
-
-static void forkable_stack_save(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  state->prev_pos_delta = s->length - s->pos;
-  state->prev_limit_delta = s->length - s->savedlimit;
-  if (s->pos < s->savedlimit) s->savedlimit = s->pos;
-}
-
-static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  int curr_pos = s->pos;
-  s->pos = s->length - state->prev_pos_delta;
-  state->prev_pos_delta = s->length - curr_pos;
-
-  int curr_limit = s->savedlimit;
-  if (curr_pos < curr_limit) s->savedlimit = curr_pos;
-  //state->prevlimit = curr_limit; FIXME clean up
-  forkable_stack_check(s);
-}
-
-static void forkable_stack_restore(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  assert(s->savedlimit <= s->length - state->prev_pos_delta);
-  assert(s->savedlimit <= s->length - state->prev_limit_delta);
-  s->pos = s->length - state->prev_pos_delta;
-  s->savedlimit = s->length - state->prev_limit_delta;
-  forkable_stack_check(s);
-}
-
-typedef int stack_idx;
-
-static stack_idx forkable_stack_to_idx(struct forkable_stack* s, void* ptr) {
-  char* item = ptr;
-  int pos = item - s->stk;
-  assert(pos >= 0 && pos < s->length);
-  return s->length - pos;
-}
-
-static void* forkable_stack_from_idx(struct forkable_stack* s, stack_idx idx) {
-  assert(idx >= 1 && idx <= s->length);
-  return &s->stk[s->length - idx];
-}
-
-#endif
@@ -1,118 +0,0 @@
-#ifndef FRAME_LAYOUT_H
-#include "forkable_stack.h"
-#include "bytecode.h"
-
-struct closure {
-  struct bytecode* bc;
-  stack_idx env;
-};
-
-struct continuation {
-  struct bytecode* bc;
-  stack_idx env;
-  uint16_t* retaddr;
-  // FIXME: probably not necessary as a separate field
-  int is_backtrack_frame;
-};
-
-typedef union frame_elem {
-  FORKABLE_STACK_HEADER;
-  struct continuation cont;
-  struct closure closure;
-  jv jsonval;
-} *frame_ptr;
-
-/*
- * Frame layout
- * fr[0] - FORKABLE_STACK_HEADER (next pointer)
- * fr[1] - self (used to store return addresses, etc)
- * fr[2...nclosures+2] - closure params
- * fr[nclosures+2..nclosures+nlocals+2] - local variables
- */
-
-static int frame_size(struct bytecode* bc) {
-  return sizeof(union frame_elem) * (bc->nclosures + bc->nlocals + 2);
-}
-
-static struct continuation* frame_self(frame_ptr fr) {
-  return &fr[1].cont;
-}
-
-static struct closure* frame_closure_arg(frame_ptr fr, int closure) {
-  assert(closure >= 0);
-  assert(closure < frame_self(fr)->bc->nclosures);
-  return &fr[2+closure].closure;
-}
-
-static jv* frame_local_var(frame_ptr fr, int var) {
-  assert(var >= 0);
-  assert(var < frame_self(fr)->bc->nlocals);
-  return &fr[2 + frame_self(fr)->bc->nclosures + var].jsonval;
-}
-
-
-static frame_ptr frame_current(struct forkable_stack* stk) {
-  frame_ptr fp = forkable_stack_peek(stk);
-  if (forkable_stack_peek_next(stk, fp)) {
-    struct bytecode* bc = frame_self(forkable_stack_peek_next(stk, fp))->bc;
-    assert(frame_self(fp)->retaddr >= bc->code && frame_self(fp)->retaddr < bc->code + bc->codelen);
-  } else {
-    assert(frame_self(fp)->retaddr == 0);
-  }
-  return fp;
-}
-
-static struct bytecode* frame_current_bytecode(struct forkable_stack* stk) {
-  return frame_self(frame_current(stk))->bc;
-}
-static uint16_t** frame_current_retaddr(struct forkable_stack* stk) {
-  return &frame_self(frame_current(stk))->retaddr;
-}
-
-static frame_ptr frame_get_parent(struct forkable_stack* stk, frame_ptr fr) {
-  return forkable_stack_from_idx(stk, frame_self(fr)->env);
-}
-
-static frame_ptr frame_get_level(struct forkable_stack* stk, frame_ptr fr, int level) {
-  for (int i=0; i<level; i++) {
-    fr = frame_get_parent(stk, fr);
-  }
-  return fr;
-}
-
-static frame_ptr frame_push(struct forkable_stack* stk, struct closure cl, uint16_t* retaddr) {
-  frame_ptr fp = forkable_stack_push(stk, frame_size(cl.bc));
-  struct continuation* cc = frame_self(fp);
-  cc->bc = cl.bc;
-  cc->env = cl.env;
-  cc->retaddr = retaddr;
-  cc->is_backtrack_frame = 0;
-  for (int i=0; i<cl.bc->nlocals; i++) {
-    *frame_local_var(fp, i) = jv_invalid();
-  }
-  return fp;
-}
-
-static frame_ptr frame_push_backtrack(struct forkable_stack* stk, uint16_t* retaddr) {
-  struct continuation cc = *frame_self(frame_current(stk));
-  frame_ptr fp = forkable_stack_push(stk, sizeof(union frame_elem) * 2);
-  assert(!cc.is_backtrack_frame);
-  cc.is_backtrack_frame = 1;
-  cc.retaddr = retaddr;
-  *frame_self(fp) = cc;
-  return fp;
-}
-
-static void frame_pop(struct forkable_stack* stk) {
-  frame_ptr fp = frame_current(stk);
-  if (forkable_stack_pop_will_free(stk) && 
-      !frame_self(fp)->is_backtrack_frame) {
-    int nlocals = frame_self(fp)->bc->nlocals;
-    for (int i=0; i<nlocals; i++) {
-      jv_free(*frame_local_var(fp, i));
-    }
-  }
-  forkable_stack_pop(stk);
-}
-
-#endif
@@ -0,0 +1,39 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "JQ" "1" "May 2013" "" ""
+.
+.SH "NAME"
+\fBjq\fR \- Command\-line JSON processor
+.
+.SH "DESCRIPTION"
+\fBjq\fR can transform JSON in various ways, by selecting, iterating, reducing and otherwise mangling JSON documents\.
+.
+.P
+This version of \fBjq\fR was built without a manual, so this manpage is a stub\. For full documentation of the \fBjq\fR language, see:
+.
+.IP "" 4
+.
+.nf
+
+http://stedolan\.github\.com/jq
+.
+.fi
+.
+.IP "" 0
+.
+.SH "BUGS"
+Presumably\. Report them or discuss them at:
+.
+.IP "" 4
+.
+.nf
+
+https://github\.com/stedolan/jq/issues
+.
+.fi
+.
+.IP "" 0
+.
+.SH "AUTHOR"
+Stephen Dolan \fB<mu@netsoc\.tcd\.ie>\fR
@@ -0,0 +1,18 @@
+#ifndef _JQ_H_
+#define _JQ_H_
+
+#include <jv.h>
+
+enum {JQ_DEBUG_TRACE = 1};
+
+typedef struct jq_state jq_state;
+jq_state *jq_init(void);
+void jq_set_nomem_handler(jq_state *, void (*)(void *), void *);
+int jq_compile(jq_state *, const char* str);
+int jq_compile_args(jq_state *, const char* str, jv args);
+void jq_dump_disassembly(jq_state *, int);
+void jq_start(jq_state *, jv value, int flags);
+jv jq_next(jq_state *);
+void jq_teardown(jq_state **);
+
+#endif /* !_JQ_H_ */
@@ -0,0 +1,65 @@
+# This is spec file maintained by developers of JQ, not by a OS distro.
+# Your OS of choice will likely ignore this RPM spec file.
+Summary: Command-line JSON processor
+Name: jq
+Version: %{myver}
+Release: %{myrel}%{?dist}
+Source0: jq-%{myver}.tar.gz
+URL: https://github.com/stedolan/jq
+License: BSD
+AutoReqProv: no
+#BuildPrereq: autoconf, libtool, automake, flex, bison, python
+
+Group: Applications/System
+# Requires:
+
+# Disables debug packages and stripping of binaries:
+%global _enable_debug_package 0
+%global debug_package %{nil}
+%global __os_install_post %{nil}
+
+# Crank up the compression
+%define _binary_payload w7.lzdio
+
+%description
+jq is a command-line JSON processor
+
+%prep
+%setup
+
+%build
+echo "Building in: \"$(pwd)\""
+%if "%{devbuild}" == "yes"
+./configure --prefix=%{_prefix} --enable-devel
+%else
+./configure --prefix=%{_prefix}
+%endif
+make
+
+%install
+echo "Installing to: \"%{buildroot}\""
+make install DESTDIR=%{buildroot}
+
+%clean
+rm -rf %{buildroot}
+
+%files
+%defattr(-,root,root)
+%{_bindir}/jq
+%if "%{devbuild}" == "yes"
+%{_libexecdir}/%{name}/jq_test
+%{_libexecdir}/%{name}/testdata
+%endif
+%{_datadir}/doc/%{name}/AUTHORS
+%{_datadir}/doc/%{name}/COPYING
+%{_datadir}/doc/%{name}/INSTALL
+%{_datadir}/doc/%{name}/NEWS
+%{_datadir}/doc/%{name}/README
+%{_datadir}/doc/%{name}/README.md
+%{_datadir}/man/man1/jq.1
+
+%changelog
+
+%pre
+
+%post
@@ -1,5 +1,7 @@
-#ifndef PARSER_H
-#define PARSER_H
+#ifndef JQ_PARSER_H
+#define JQ_PARSER_H
+#include "locfile.h"
+#include "compile.h"

 int jq_parse(struct locfile* source, block* answer);
 int jq_parse_library(struct locfile* locations, block* answer);
@@ -3,34 +3,26 @@
 #include <string.h>
 #include <stdlib.h>
 #include "jv.h"
-#include "execute.h"
+#include "jq.h"

 static void jv_test();
 static void run_jq_tests();

-FILE* testdata;

-int main(int argc, char* argv[]) {
+int jq_testsuite(int argc, char* argv[]) {
+  FILE *testdata = stdin;
  jv_test();
-  if (argc == 1) {
-    testdata = fopen("testdata", "r");
-  } else if (argc == 2) {
-    if (!strcmp(argv[1], "-")) {
-      testdata = stdin;
-    } else {
-      testdata = fopen(argv[1], "r");
+  if (argc > 2) {
+    testdata = fopen(argv[2], "r");
+    if (!testdata) {
+      perror("fopen");
+      exit(1);
    }
-  } else {
-    printf("usage: %s OR cat testdata | %s - OR %s testdata\n", argv[0], argv[0], argv[0]);
-    return 127;
  }
-  run_jq_tests();
-  if (testdata != stdin) fclose(testdata);
+  run_jq_tests(testdata);
+  return 0;
 }

-
-
-
 static int skipline(const char* buf) {
  int p = 0;
  while (buf[p] == ' ' || buf[p] == '\t') p++;
@@ -38,9 +30,13 @@ static int skipline(const char* buf) {
  return 0;
 }

-static void run_jq_tests() {
+static void run_jq_tests(FILE *testdata) {
  char buf[4096];
  int tests = 0, passed = 0, invalid = 0;
+  jq_state *jq = NULL;
+
+  jq = jq_init();
+  assert(jq);

  while (1) {
    if (!fgets(buf, sizeof(buf), testdata)) break;
@@ -49,23 +45,21 @@ static void run_jq_tests() {
    printf("Testing %s\n", buf);
    int pass = 1;
    tests++;
-    struct bytecode* bc = jq_compile(buf);
-    if (!bc) {invalid++; continue;}
-#if JQ_DEBUG
+    int compiled = jq_compile(jq, buf);
+    if (!compiled) {invalid++; continue;}
    printf("Disassembly:\n");
-    dump_disassembly(2, bc);
+    jq_dump_disassembly(jq, 2);
    printf("\n");
-#endif
-    fgets(buf, sizeof(buf), testdata);
+    if (!fgets(buf, sizeof(buf), testdata)) { invalid++; break; }
    jv input = jv_parse(buf);
    if (!jv_is_valid(input)){ invalid++; continue; }
-    jq_init(bc, input);
+    jq_start(jq, input, JQ_DEBUG_TRACE);

    while (fgets(buf, sizeof(buf), testdata)) {
      if (skipline(buf)) break;
      jv expected = jv_parse(buf);
      if (!jv_is_valid(expected)){ invalid++; continue; }
-      jv actual = jq_next();
+      jv actual = jq_next(jq);
      if (!jv_is_valid(actual)) {
        jv_free(actual);
        printf("*** Insufficient results\n");
@@ -80,7 +74,7 @@ static void run_jq_tests() {
        pass = 0;
      }
      jv as_string = jv_dump_string(jv_copy(expected), rand() & ~JV_PRINT_COLOUR);
-      jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length(jv_copy(as_string)));
+      jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length_bytes(jv_copy(as_string)));
      assert(jv_equal(jv_copy(expected), jv_copy(reparsed)));
      jv_free(as_string);
      jv_free(reparsed);
@@ -88,7 +82,7 @@ static void run_jq_tests() {
      jv_free(actual);
    }
    if (pass) {
-      jv extra = jq_next();
+      jv extra = jq_next(jq);
      if (jv_is_valid(extra)) {
        printf("*** Superfluous result: ");
        jv_dump(extra, 0);
@@ -98,10 +92,9 @@ static void run_jq_tests() {
        jv_free(extra);
      }
    }
-    jq_teardown();
-    bytecode_free(bc);
    passed+=pass;
  }
+  jq_teardown(&jq);
  printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
  if (passed != tests) exit(1);
 }
@@ -132,9 +125,9 @@ static void jv_test() {
    jv_free(a2);


-    assert(a.val.complex.ptr->count == 1);
+    assert(jv_get_refcnt(a) == 1);
    a = jv_array_append(a, jv_copy(a));
-    assert(a.val.complex.ptr->count == 1);
+    assert(jv_get_refcnt(a) == 1);

    assert(jv_array_length(jv_copy(a)) == 2);
    assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
@@ -167,9 +160,9 @@ static void jv_test() {

    jv_free(subarray);

-    void* before = sub2.val.complex.ptr;
+    void* before = sub2.val.nontrivial.ptr;
    sub2 = jv_array_append(sub2, jv_number(200));
-    void* after = sub2.val.complex.ptr;
+    void* after = sub2.val.nontrivial.ptr;
    assert(before == after);
    jv_free(sub2);

@@ -181,6 +174,18 @@ static void jv_test() {
    jv_free(a3);


+    jv a4 = jv_array();
+    a4 = jv_array_append(a4, jv_number(1));
+    a4 = jv_array_append(a4, jv_number(2));
+    jv a5 = jv_copy(a4);
+    a4 = jv_array_append(a4, jv_number(3));
+    a4 = jv_array_slice(a4, 0, 1);
+    assert(jv_array_length(jv_copy(a4)) == 1);
+    a4 = jv_array_append(a4, jv_number(4));
+    assert(jv_array_length(a4) == 2);
+    assert(jv_array_length(a5) == 2);
+
+
    assert(jv_array_length(jv_copy(a)) == 2);
    assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
    assert(jv_array_length(jv_array_get(jv_copy(a), 1)) == 1);
@@ -196,8 +201,8 @@ static void jv_test() {
    assert(jv_equal(jv_string("foo"), jv_string_sized("foo", 3)));
    char nasty[] = "foo\0";
    jv shortstr = jv_string(nasty), longstr = jv_string_sized(nasty, sizeof(nasty));
-    assert(jv_string_length(shortstr) == (int)strlen(nasty));
-    assert(jv_string_length(longstr) == (int)sizeof(nasty));
+    assert(jv_string_length_bytes(shortstr) == (int)strlen(nasty));
+    assert(jv_string_length_bytes(longstr) == (int)sizeof(nasty));

  
    char a1s[] = "hello", a2s[] = "hello", bs[] = "goodbye";
@@ -218,7 +223,7 @@ static void jv_test() {
    for (int i=0; i<(int)sizeof(big); i++) big[i] = 'a';
    big[sizeof(big)-1] = 0;
    jv str = jv_string_fmt("%s", big);
-    assert(jv_string_length(jv_copy(str)) == sizeof(big) - 1);
+    assert(jv_string_length_bytes(jv_copy(str)) == sizeof(big) - 1);
    assert(!strcmp(big, jv_string_value(str)));
    jv_free(str);
  }
@@ -8,25 +8,31 @@

 #include "jv_alloc.h"
 #include "jv.h"
+#include "jv_unicode.h"

 /*
 * Internal refcounting helpers
 */

-static void jvp_refcnt_init(jv_complex* c) {
+typedef struct jv_refcnt {
+  size_t count;
+} jv_refcnt;
+
+
+static void jvp_refcnt_init(jv_nontrivial* c) {
  c->ptr->count = 1;
 }

-static void jvp_refcnt_inc(jv_complex* c) {
+static void jvp_refcnt_inc(jv_nontrivial* c) {
  c->ptr->count++;
 }

-static int jvp_refcnt_dec(jv_complex* c) {
+static int jvp_refcnt_dec(jv_nontrivial* c) {
  c->ptr->count--;
  return c->ptr->count == 0;
 }

-static int jvp_refcnt_unshared(jv_complex* c) {
+static int jvp_refcnt_unshared(jv_nontrivial* c) {
  assert(c->ptr->count > 0);
  return c->ptr->count == 1;
 }
@@ -86,9 +92,9 @@ typedef struct {
 jv jv_invalid_with_msg(jv err) {
  jv x;
  x.kind = JV_KIND_INVALID;
-  x.val.complex.i[0] = x.val.complex.i[1] = 0;
+  x.val.nontrivial.i[0] = x.val.nontrivial.i[1] = 0;
  jvp_invalid* i = jv_mem_alloc(sizeof(jvp_invalid));
-  x.val.complex.ptr = &i->refcnt;
+  x.val.nontrivial.ptr = &i->refcnt;
  i->refcnt.count = 1;
  i->errmsg = err;
  return x;
@@ -99,7 +105,7 @@ jv jv_invalid() {
 }

 jv jv_invalid_get_msg(jv inv) {
-  jv x = jv_copy(((jvp_invalid*)inv.val.complex.ptr)->errmsg);
+  jv x = jv_copy(((jvp_invalid*)inv.val.nontrivial.ptr)->errmsg);
  jv_free(inv);
  return x;
 }
@@ -111,7 +117,7 @@ int jv_invalid_has_msg(jv inv) {
  return r;
 }

-static void jvp_invalid_free(jv_complex* x) {
+static void jvp_invalid_free(jv_nontrivial* x) {
  if (jvp_refcnt_dec(x)) {
    jv_free(((jvp_invalid*)x->ptr)->errmsg);
    jv_mem_free(x->ptr);
@@ -153,7 +159,7 @@ typedef struct {
  jv elements[];
 } jvp_array;

-static jvp_array* jvp_array_ptr(jv_complex* a) {
+static jvp_array* jvp_array_ptr(jv_nontrivial* a) {
  return (jvp_array*)a->ptr;
 }

@@ -165,12 +171,12 @@ static jvp_array* jvp_array_alloc(unsigned size) {
  return a;
 }

-static jv_complex jvp_array_new(unsigned size) {
-  jv_complex r = {&jvp_array_alloc(size)->refcnt, {0, 0}};
+static jv_nontrivial jvp_array_new(unsigned size) {
+  jv_nontrivial r = {&jvp_array_alloc(size)->refcnt, {0, 0}};
  return r;
 }

-static void jvp_array_free(jv_complex* a) {
+static void jvp_array_free(jv_nontrivial* a) {
  if (jvp_refcnt_dec(a)) {
    jvp_array* array = jvp_array_ptr(a);
    for (int i=0; i<array->length; i++) {
@@ -180,11 +186,11 @@ static void jvp_array_free(jv_complex* a) {
  }
 }

-static int jvp_array_length(jv_complex* a) {
+static int jvp_array_length(jv_nontrivial* a) {
  return a->i[1] - a->i[0];
 }

-static jv* jvp_array_read(jv_complex* a, int i) {
+static jv* jvp_array_read(jv_nontrivial* a, int i) {
  if (i >= 0 && i < jvp_array_length(a)) {
    jvp_array* array = jvp_array_ptr(a);
    assert(i + a->i[0] < array->length);
@@ -194,7 +200,7 @@ static jv* jvp_array_read(jv_complex* a, int i) {
  }
 }

-static jv* jvp_array_write(jv_complex* a, int i) {
+static jv* jvp_array_write(jv_nontrivial* a, int i) {
  assert(i >= 0);
  jvp_array* array = jvp_array_ptr(a);

@@ -214,7 +220,7 @@ static jv* jvp_array_write(jv_complex* a, int i) {
        array->elements[j] = JV_NULL;
      }
      array->length = imax(pos + 1, array->length);
-      a->i[1] = imax(pos + 1, array->length);
+      a->i[1] = imax(pos + 1, a->i[1]);
      return &array->elements[pos];
    }
  }
@@ -237,7 +243,7 @@ static jv* jvp_array_write(jv_complex* a, int i) {
  return &new_array->elements[i];
 }

-static int jvp_array_equal(jv_complex* a, jv_complex* b) {
+static int jvp_array_equal(jv_nontrivial* a, jv_nontrivial* b) {
  if (jvp_array_length(a) != jvp_array_length(b)) 
    return 0;
  if (jvp_array_ptr(a) == jvp_array_ptr(b) &&
@@ -251,12 +257,11 @@ static int jvp_array_equal(jv_complex* a, jv_complex* b) {
  return 1;
 }

-static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
+static jv_nontrivial jvp_array_slice(jv_nontrivial* a, int start, int end) {
  // FIXME: maybe slice should reallocate if the slice is small enough
  assert(start <= end);
-  jvp_array* array = jvp_array_ptr(a);
-  assert(a->i[1] + end < array->length);
-  jv_complex slice = *a;
+  assert(a->i[0] + end <= a->i[1]);
+  jv_nontrivial slice = *a;
  slice.i[0] += start;
  slice.i[1] = slice.i[0] + (end - start);
  return slice;
@@ -269,7 +274,7 @@ static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
 jv jv_array_sized(int n) {
  jv j;
  j.kind = JV_KIND_ARRAY;
-  j.val.complex = jvp_array_new(n);
+  j.val.nontrivial = jvp_array_new(n);
  return j;
 }

@@ -279,14 +284,14 @@ jv jv_array() {

 int jv_array_length(jv j) {
  assert(jv_get_kind(j) == JV_KIND_ARRAY);
-  int len = jvp_array_length(&j.val.complex);
+  int len = jvp_array_length(&j.val.nontrivial);
  jv_free(j);
  return len;
 }

 jv jv_array_get(jv j, int idx) {
  assert(jv_get_kind(j) == JV_KIND_ARRAY);
-  jv* slot = jvp_array_read(&j.val.complex, idx);
+  jv* slot = jvp_array_read(&j.val.nontrivial, idx);
  jv val;
  if (slot) {
    val = jv_copy(*slot);
@@ -300,7 +305,7 @@ jv jv_array_get(jv j, int idx) {
 jv jv_array_set(jv j, int idx, jv val) {
  assert(jv_get_kind(j) == JV_KIND_ARRAY);
  // copy/free of val,j coalesced
-  jv* slot = jvp_array_write(&j.val.complex, idx);
+  jv* slot = jvp_array_write(&j.val.nontrivial, idx);
  jv_free(*slot);
  *slot = val;
  return j;
@@ -316,8 +321,8 @@ jv jv_array_concat(jv a, jv b) {
  assert(jv_get_kind(b) == JV_KIND_ARRAY);

  // FIXME: could be much faster
-  for (int i=0; i<jv_array_length(jv_copy(b)); i++) {
-    a = jv_array_append(a, jv_array_get(jv_copy(b), i));
+  jv_array_foreach(b, i, elem) {
+    a = jv_array_append(a, elem);
  }
  jv_free(b);
  return a;
@@ -326,22 +331,21 @@ jv jv_array_concat(jv a, jv b) {
 jv jv_array_slice(jv a, int start, int end) {
  assert(jv_get_kind(a) == JV_KIND_ARRAY);
  // copy/free of a coalesced
-  a.val.complex = jvp_array_slice(&a.val.complex, start, end);
+  a.val.nontrivial = jvp_array_slice(&a.val.nontrivial, start, end);
  return a;
 }

 int jv_array_contains(jv a, jv b) {
  int r = 1;
-  int a_length = jv_array_length(jv_copy(a));
-  int b_length = jv_array_length(jv_copy(b));
-  for (int bi = 0; bi < b_length; bi++) {
+  jv_array_foreach(b, bi, belem) {
    int ri = 0;
-    for (int ai = 0; ai < a_length; ai++) {
-      if (jv_contains(jv_array_get(jv_copy(a), ai), jv_array_get(jv_copy(b), bi))) {
+    jv_array_foreach(a, ai, aelem) {
+      if (jv_contains(aelem, jv_copy(belem))) {
        ri = 1;
        break;
      }
    }
+    jv_free(belem);
    if (!ri) {
      r = 0;
      break;
@@ -367,7 +371,7 @@ typedef struct {
  char data[];
 } jvp_string;

-static jvp_string* jvp_string_ptr(jv_complex* a) {
+static jvp_string* jvp_string_ptr(jv_nontrivial* a) {
  return (jvp_string*)a->ptr;
 }

@@ -378,16 +382,16 @@ static jvp_string* jvp_string_alloc(uint32_t size) {
  return s;
 }

-static jv_complex jvp_string_new(const char* data, uint32_t length) {
+static jv_nontrivial jvp_string_new(const char* data, uint32_t length) {
  jvp_string* s = jvp_string_alloc(length);
  s->length_hashed = length << 1;
  memcpy(s->data, data, length);
  s->data[length] = 0;
-  jv_complex r = {&s->refcnt, {0,0}};
+  jv_nontrivial r = {&s->refcnt, {0,0}};
  return r;
 }

-static void jvp_string_free(jv_complex* s) {
+static void jvp_string_free(jv_nontrivial* s) {
  if (jvp_refcnt_dec(s)) {
    jvp_string* str = jvp_string_ptr(s);
    jv_mem_free(str);
@@ -395,12 +399,12 @@ static void jvp_string_free(jv_complex* s) {
 }

 static void jvp_string_free_p(jvp_string* s) {
-  jv_complex p = {&s->refcnt,{0,0}};
+  jv_nontrivial p = {&s->refcnt,{0,0}};
  jvp_string_free(&p);
 }

 static jvp_string* jvp_string_copy_p(jvp_string* s) {
-  jv_complex p = {&s->refcnt,{0,0}};
+  jv_nontrivial p = {&s->refcnt,{0,0}};
  jvp_refcnt_inc(&p);
  return s;
 }
@@ -415,7 +419,7 @@ static uint32_t jvp_string_remaining_space(jvp_string* s) {
  return r;
 }

-static void jvp_string_append(jv_complex* string, const char* data, uint32_t len) {
+static void jvp_string_append(jv_nontrivial* string, const char* data, uint32_t len) {
  jvp_string* s = jvp_string_ptr(string);
  uint32_t currlen = jvp_string_length(s);
    
@@ -435,7 +439,7 @@ static void jvp_string_append(jv_complex* string, const char* data, uint32_t len
    memcpy(news->data + currlen, data, len);
    news->data[currlen + len] = 0;
    jvp_string_free(string);
-    jv_complex r = {&news->refcnt, {0,0}};
+    jv_nontrivial r = {&news->refcnt, {0,0}};
    *string = r;
  }
 }
@@ -510,7 +514,7 @@ static int jvp_string_equal_hashed(jvp_string* a, jvp_string* b) {
  return memcmp(a->data, b->data, jvp_string_length(a)) == 0;
 }

-static int jvp_string_equal(jv_complex* a, jv_complex* b) {
+static int jvp_string_equal(jv_nontrivial* a, jv_nontrivial* b) {
  jvp_string* stra = jvp_string_ptr(a);
  jvp_string* strb = jvp_string_ptr(b);
  if (jvp_string_length(stra) != jvp_string_length(strb)) return 0;
@@ -524,7 +528,7 @@ static int jvp_string_equal(jv_complex* a, jv_complex* b) {
 jv jv_string_sized(const char* str, int len) {
  jv j;
  j.kind = JV_KIND_STRING;
-  j.val.complex = jvp_string_new(str, len);
+  j.val.nontrivial = jvp_string_new(str, len);
  return j;
 }

@@ -532,34 +536,44 @@ jv jv_string(const char* str) {
  return jv_string_sized(str, strlen(str));
 }

-int jv_string_length(jv j) {
+int jv_string_length_bytes(jv j) {
  assert(jv_get_kind(j) == JV_KIND_STRING);
-  int r = jvp_string_length(jvp_string_ptr(&j.val.complex));
+  int r = jvp_string_length(jvp_string_ptr(&j.val.nontrivial));
  jv_free(j);
  return r;
 }

-uint32_t jv_string_hash(jv j) {
+int jv_string_length_codepoints(jv j) {
  assert(jv_get_kind(j) == JV_KIND_STRING);
-  uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.complex));
+  const char* i = jv_string_value(j);
+  const char* end = i + jv_string_length_bytes(jv_copy(j));
+  int c = 0, len = 0;
+  while ((i = jvp_utf8_next(i, end, &c))) len++;
+  jv_free(j);
+  return len;
+}
+
+unsigned long jv_string_hash(jv j) {
+  assert(jv_get_kind(j) == JV_KIND_STRING);
+  uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.nontrivial));
  jv_free(j);
  return hash;
 }

 const char* jv_string_value(jv j) {
  assert(jv_get_kind(j) == JV_KIND_STRING);
-  return jvp_string_ptr(&j.val.complex)->data;
+  return jvp_string_ptr(&j.val.nontrivial)->data;
 }

 jv jv_string_concat(jv a, jv b) {
-  jvp_string* sb = jvp_string_ptr(&b.val.complex);
-  jvp_string_append(&a.val.complex, sb->data, jvp_string_length(sb));
+  jvp_string* sb = jvp_string_ptr(&b.val.nontrivial);
+  jvp_string_append(&a.val.nontrivial, sb->data, jvp_string_length(sb));
  jv_free(b);
  return a;
 }

 jv jv_string_append_buf(jv a, const char* buf, int len) {
-  jvp_string_append(&a.val.complex, buf, len);
+  jvp_string_append(&a.val.nontrivial, buf, len);
  return a;
 }

@@ -604,8 +618,8 @@ typedef struct {
 } jvp_object;


-/* warning: complex justification of alignment */
-static jv_complex jvp_object_new(int size) {
+/* warning: nontrivial justification of alignment */
+static jv_nontrivial jvp_object_new(int size) {
  // Allocates an object of (size) slots and (size*2) hash buckets.

  // size must be a power of two
@@ -622,7 +636,7 @@ static jv_complex jvp_object_new(int size) {
  }
  obj->first_free = size - 1;
  int* hashbuckets = (int*)(&obj->elements[size]);
-  jv_complex r = {&obj->refcnt, 
+  jv_nontrivial r = {&obj->refcnt, 
                  {size*2 - 1, (char*)hashbuckets - (char*)obj}};
  for (int i=0; i<size*2; i++) {
    hashbuckets[i] = -1;
@@ -630,39 +644,39 @@ static jv_complex jvp_object_new(int size) {
  return r;
 }

-static jvp_object* jvp_object_ptr(jv_complex* o) {
+static jvp_object* jvp_object_ptr(jv_nontrivial* o) {
  return (jvp_object*)o->ptr;
 }

-static uint32_t jvp_object_mask(jv_complex* o) {
+static uint32_t jvp_object_mask(jv_nontrivial* o) {
  return o->i[0];
 }

-static int jvp_object_size(jv_complex* o) {
+static int jvp_object_size(jv_nontrivial* o) {
  return (o->i[0] + 1) >> 1;
 }

-static int* jvp_object_buckets(jv_complex* o) {
+static int* jvp_object_buckets(jv_nontrivial* o) {
  int* buckets = (int*)((char*)o->ptr + o->i[1]);
  assert(buckets == (int*)&jvp_object_ptr(o)->elements[jvp_object_size(o)]);
  return buckets;
 }

-static int* jvp_object_find_bucket(jv_complex* object, jvp_string* key) {
+static int* jvp_object_find_bucket(jv_nontrivial* object, jvp_string* key) {
  return jvp_object_buckets(object) + (jvp_object_mask(object) & jvp_string_hash(key));
 }

-static struct object_slot* jvp_object_get_slot(jv_complex* object, int slot) {
+static struct object_slot* jvp_object_get_slot(jv_nontrivial* object, int slot) {
  assert(slot == -1 || (slot >= 0 && slot < jvp_object_size(object)));
  if (slot == -1) return 0;
  else return &jvp_object_ptr(object)->elements[slot];
 }

-static struct object_slot* jvp_object_next_slot(jv_complex* object, struct object_slot* slot) {
+static struct object_slot* jvp_object_next_slot(jv_nontrivial* object, struct object_slot* slot) {
  return jvp_object_get_slot(object, slot->next);
 }

-static struct object_slot* jvp_object_find_slot(jv_complex* object, jvp_string* keystr, int* bucket) {
+static struct object_slot* jvp_object_find_slot(jv_nontrivial* object, jvp_string* keystr, int* bucket) {
  for (struct object_slot* curr = jvp_object_get_slot(object, *bucket); 
       curr; 
       curr = jvp_object_next_slot(object, curr)) {
@@ -673,7 +687,7 @@ static struct object_slot* jvp_object_find_slot(jv_complex* object, jvp_string*
  return 0;
 }

-static struct object_slot* jvp_object_add_slot(jv_complex* object, jvp_string* key, int* bucket) {
+static struct object_slot* jvp_object_add_slot(jv_nontrivial* object, jvp_string* key, int* bucket) {
  jvp_object* o = jvp_object_ptr(object);
  int newslot_idx = o->first_free;
  struct object_slot* newslot = jvp_object_get_slot(object, newslot_idx);
@@ -686,7 +700,7 @@ static struct object_slot* jvp_object_add_slot(jv_complex* object, jvp_string* k
  return newslot;
 }

-static void jvp_object_free_slot(jv_complex* object, struct object_slot* slot) {
+static void jvp_object_free_slot(jv_nontrivial* object, struct object_slot* slot) {
  jvp_object* o = jvp_object_ptr(object);
  slot->next = o->first_free;
  assert(slot->string);
@@ -696,14 +710,14 @@ static void jvp_object_free_slot(jv_complex* object, struct object_slot* slot) {
  o->first_free = slot - jvp_object_get_slot(object, 0);
 }

-static jv* jvp_object_read(jv_complex* object, jvp_string* key) {
+static jv* jvp_object_read(jv_nontrivial* object, jvp_string* key) {
  int* bucket = jvp_object_find_bucket(object, key);
  struct object_slot* slot = jvp_object_find_slot(object, key, bucket);
  if (slot == 0) return 0;
  else return &slot->value;
 }

-static void jvp_object_free(jv_complex* o) {
+static void jvp_object_free(jv_nontrivial* o) {
  if (jvp_refcnt_dec(o)) {
    for (int i=0; i<jvp_object_size(o); i++) {
      struct object_slot* slot = jvp_object_get_slot(o, i);
@@ -716,10 +730,10 @@ static void jvp_object_free(jv_complex* o) {
  }
 }

-static void jvp_object_rehash(jv_complex* object) {
+static void jvp_object_rehash(jv_nontrivial* object) {
  assert(jvp_refcnt_unshared(object));
  int size = jvp_object_size(object);
-  jv_complex new_object = jvp_object_new(size * 2);
+  jv_nontrivial new_object = jvp_object_new(size * 2);
  for (int i=0; i<size; i++) {
    struct object_slot* slot = jvp_object_get_slot(object, i);
    if (!slot->string) continue;
@@ -735,11 +749,11 @@ static void jvp_object_rehash(jv_complex* object) {
  *object = new_object;
 }

-static void jvp_object_unshare(jv_complex* object) {
+static void jvp_object_unshare(jv_nontrivial* object) {
  if (jvp_refcnt_unshared(object))
    return;

-  jv_complex new_object = jvp_object_new(jvp_object_size(object));
+  jv_nontrivial new_object = jvp_object_new(jvp_object_size(object));
  jvp_object_ptr(&new_object)->first_free = jvp_object_ptr(object)->first_free;
  for (int i=0; i<jvp_object_size(&new_object); i++) {
    struct object_slot* old_slot = jvp_object_get_slot(object, i);
@@ -760,7 +774,7 @@ static void jvp_object_unshare(jv_complex* object) {
  assert(jvp_refcnt_unshared(object));
 }

-static jv* jvp_object_write(jv_complex* object, jvp_string* key) {
+static jv* jvp_object_write(jv_nontrivial* object, jvp_string* key) {
  jvp_object_unshare(object);
  int* bucket = jvp_object_find_bucket(object, key);
  struct object_slot* slot = jvp_object_find_slot(object, key, bucket);
@@ -783,7 +797,7 @@ static jv* jvp_object_write(jv_complex* object, jvp_string* key) {
  return &slot->value;
 }

-static int jvp_object_delete(jv_complex* object, jvp_string* key) {
+static int jvp_object_delete(jv_nontrivial* object, jvp_string* key) {
  jvp_object_unshare(object);
  int* bucket = jvp_object_find_bucket(object, key);
  int* prev_ptr = bucket;
@@ -800,7 +814,7 @@ static int jvp_object_delete(jv_complex* object, jvp_string* key) {
  return 0;
 }

-static int jvp_object_length(jv_complex* object) {
+static int jvp_object_length(jv_nontrivial* object) {
  int n = 0;
  for (int i=0; i<jvp_object_size(object); i++) {
    struct object_slot* slot = jvp_object_get_slot(object, i);
@@ -809,7 +823,7 @@ static int jvp_object_length(jv_complex* object) {
  return n;
 }

-static int jvp_object_equal(jv_complex* o1, jv_complex* o2) {
+static int jvp_object_equal(jv_nontrivial* o1, jv_nontrivial* o2) {
  int len2 = jvp_object_length(o2);
  int len1 = 0;
  for (int i=0; i<jvp_object_size(o1); i++) {
@@ -831,14 +845,14 @@ static int jvp_object_equal(jv_complex* o1, jv_complex* o2) {
 jv jv_object() {
  jv j;
  j.kind = JV_KIND_OBJECT;
-  j.val.complex = jvp_object_new(8);
+  j.val.nontrivial = jvp_object_new(8);
  return j;
 }

 jv jv_object_get(jv object, jv key) {
  assert(jv_get_kind(object) == JV_KIND_OBJECT);
  assert(jv_get_kind(key) == JV_KIND_STRING);
-  jv* slot = jvp_object_read(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jv* slot = jvp_object_read(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
  jv val;
  if (slot) {
    val = jv_copy(*slot);
@@ -854,7 +868,7 @@ jv jv_object_set(jv object, jv key, jv value) {
  assert(jv_get_kind(object) == JV_KIND_OBJECT);
  assert(jv_get_kind(key) == JV_KIND_STRING);
  // copy/free of object, key, value coalesced
-  jv* slot = jvp_object_write(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jv* slot = jvp_object_write(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
  jv_free(*slot);
  *slot = value;
  return object;
@@ -863,24 +877,22 @@ jv jv_object_set(jv object, jv key, jv value) {
 jv jv_object_delete(jv object, jv key) {
  assert(jv_get_kind(object) == JV_KIND_OBJECT);
  assert(jv_get_kind(key) == JV_KIND_STRING);
-  jvp_object_delete(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jvp_object_delete(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
  jv_free(key);
  return object;
 }

 int jv_object_length(jv object) {
  assert(jv_get_kind(object) == JV_KIND_OBJECT);
-  int n = jvp_object_length(&object.val.complex);
+  int n = jvp_object_length(&object.val.nontrivial);
  jv_free(object);
  return n;
 }

 jv jv_object_merge(jv a, jv b) {
  assert(jv_get_kind(a) == JV_KIND_OBJECT);
-  jv_object_foreach(i, b) {
-    a = jv_object_set(a, 
-                      jv_object_iter_key(b, i),
-                      jv_object_iter_value(b, i));
+  jv_object_foreach(b, k, v) {
+    a = jv_object_set(a, k, v);
  }
  jv_free(b);
  return a;
@@ -891,10 +903,8 @@ int jv_object_contains(jv a, jv b) {
  assert(jv_get_kind(b) == JV_KIND_OBJECT);
  int r = 1;

-  jv_object_foreach(i, b) {
-    jv key = jv_object_iter_key(b, i);
+  jv_object_foreach(b, key, b_val) {
    jv a_val = jv_object_get(jv_copy(a), jv_copy(key));
-    jv b_val = jv_object_get(jv_copy(b), jv_copy(key));

    r = jv_contains(a_val, b_val);
    jv_free(key);
@@ -925,7 +935,7 @@ int jv_object_iter(jv object) {
 int jv_object_iter_next(jv object, int iter) {
  assert(jv_get_kind(object) == JV_KIND_OBJECT);
  assert(iter != ITER_FINISHED);
-  jv_complex* o = &object.val.complex;
+  jv_nontrivial* o = &object.val.nontrivial;
  struct object_slot* slot;
  do {
    iter++;
@@ -937,18 +947,18 @@ int jv_object_iter_next(jv object, int iter) {
 }

 jv jv_object_iter_key(jv object, int iter) {
-  jvp_string* s = jvp_object_get_slot(&object.val.complex, iter)->string;
+  jvp_string* s = jvp_object_get_slot(&object.val.nontrivial, iter)->string;
  assert(s);
  jv j;
  j.kind = JV_KIND_STRING;
-  j.val.complex.ptr = &s->refcnt;
-  j.val.complex.i[0] = 0;
-  j.val.complex.i[1] = 0;
+  j.val.nontrivial.ptr = &s->refcnt;
+  j.val.nontrivial.i[0] = 0;
+  j.val.nontrivial.i[1] = 0;
  return jv_copy(j);
 }

 jv jv_object_iter_value(jv object, int iter) {
-  return jv_copy(jvp_object_get_slot(&object.val.complex, iter)->value);
+  return jv_copy(jvp_object_get_slot(&object.val.nontrivial, iter)->value);
 }

 /*
@@ -959,20 +969,20 @@ jv jv_copy(jv j) {
      jv_get_kind(j) == JV_KIND_STRING || 
      jv_get_kind(j) == JV_KIND_OBJECT ||
      jv_get_kind(j) == JV_KIND_INVALID) {
-    jvp_refcnt_inc(&j.val.complex);
+    jvp_refcnt_inc(&j.val.nontrivial);
  }
  return j;
 }

 void jv_free(jv j) {
  if (jv_get_kind(j) == JV_KIND_ARRAY) {
-    jvp_array_free(&j.val.complex);
+    jvp_array_free(&j.val.nontrivial);
  } else if (jv_get_kind(j) == JV_KIND_STRING) {
-    jvp_string_free(&j.val.complex);
+    jvp_string_free(&j.val.nontrivial);
  } else if (jv_get_kind(j) == JV_KIND_OBJECT) {
-    jvp_object_free(&j.val.complex);
+    jvp_object_free(&j.val.nontrivial);
  } else if (jv_get_kind(j) == JV_KIND_INVALID) {
-    jvp_invalid_free(&j.val.complex);
+    jvp_invalid_free(&j.val.nontrivial);
  }
 }

@@ -981,7 +991,7 @@ int jv_get_refcnt(jv j) {
  case JV_KIND_ARRAY:
  case JV_KIND_STRING:
  case JV_KIND_OBJECT:
-    return j.val.complex.ptr->count;
+    return j.val.nontrivial.ptr->count;
  default:
    return 1;
  }
@@ -997,20 +1007,20 @@ int jv_equal(jv a, jv b) {
    r = 0;
  } else if (jv_get_kind(a) == JV_KIND_NUMBER) {
    r = jv_number_value(a) == jv_number_value(b);
-  } else if (a.val.complex.ptr == b.val.complex.ptr &&
-             a.val.complex.i[0] == b.val.complex.i[0] &&
-             a.val.complex.i[1] == b.val.complex.i[1]) {
+  } else if (a.val.nontrivial.ptr == b.val.nontrivial.ptr &&
+             a.val.nontrivial.i[0] == b.val.nontrivial.i[0] &&
+             a.val.nontrivial.i[1] == b.val.nontrivial.i[1]) {
    r = 1;
  } else {
    switch (jv_get_kind(a)) {
    case JV_KIND_ARRAY:
-      r = jvp_array_equal(&a.val.complex, &b.val.complex);
+      r = jvp_array_equal(&a.val.nontrivial, &b.val.nontrivial);
      break;
    case JV_KIND_STRING:
-      r = jvp_string_equal(&a.val.complex, &b.val.complex);
+      r = jvp_string_equal(&a.val.nontrivial, &b.val.nontrivial);
      break;
    case JV_KIND_OBJECT:
-      r = jvp_object_equal(&a.val.complex, &b.val.complex);
+      r = jvp_object_equal(&a.val.nontrivial, &b.val.nontrivial);
      break;
    default:
      r = 1;
@@ -1,12 +1,6 @@
 #ifndef JV_H
 #define JV_H

-#include <stdint.h>
-#include <assert.h>
-#include <stddef.h>
-
-
-
 typedef enum {
  JV_KIND_INVALID,
  JV_KIND_NULL,
@@ -18,20 +12,17 @@ typedef enum {
  JV_KIND_OBJECT
 } jv_kind;

-typedef struct {
-  size_t count;
-} jv_refcnt;
-
+struct jv_refcnt;
 typedef struct{
-  jv_refcnt* ptr;
+  struct jv_refcnt* ptr;
  int i[2];
-} jv_complex;
+} jv_nontrivial;

 typedef struct {
  jv_kind kind;
  union {
    double number;
-    jv_complex complex;
+    jv_nontrivial nontrivial;
  } val;
 } jv;

@@ -47,6 +38,8 @@ static int jv_is_valid(jv x) { return jv_get_kind(x) != JV_KIND_INVALID; }
 jv jv_copy(jv);
 void jv_free(jv);

+int jv_get_refcnt(jv);
+
 int jv_equal(jv, jv);
 int jv_contains(jv, jv);

@@ -72,12 +65,19 @@ jv jv_array_set(jv, int, jv);
 jv jv_array_append(jv, jv);
 jv jv_array_concat(jv, jv);
 jv jv_array_slice(jv, int, int);
-
+#define jv_array_foreach(a, i, x) \
+  for (int jv_len__ = jv_array_length(jv_copy(a)), i=0, jv_j__ = 1;     \
+       jv_j__; jv_j__ = 0)                                              \
+    for (jv x;                                                          \
+         i < jv_len__ ?                                                 \
+           (x = jv_array_get(jv_copy(a), i), 1) : 0;                    \
+         i++)

 jv jv_string(const char*);
 jv jv_string_sized(const char*, int);
-int jv_string_length(jv);
-uint32_t jv_string_hash(jv);
+int jv_string_length_bytes(jv);
+int jv_string_length_codepoints(jv);
+unsigned long jv_string_hash(jv);
 const char* jv_string_value(jv);
 jv jv_string_concat(jv, jv);
 jv jv_string_fmt(const char*, ...);
@@ -96,29 +96,48 @@ int jv_object_iter_next(jv, int);
 int jv_object_iter_valid(jv, int);
 jv jv_object_iter_key(jv, int);
 jv jv_object_iter_value(jv, int);
-#define jv_object_foreach(i,t) \
-  for (int i = jv_object_iter(t);               \
-       jv_object_iter_valid(t, i);              \
-       i = jv_object_iter_next(t, i))           \
+#define jv_object_foreach(t, k, v)                                      \
+  for (int jv_i__ = jv_object_iter(t), jv_j__ = 1; jv_j__; jv_j__ = 0)  \
+    for (jv k, v;                                                       \
+         jv_object_iter_valid((t), jv_i__) ?                            \
+           (k = jv_object_iter_key(t, jv_i__),                          \
+            v = jv_object_iter_value(t, jv_i__),                        \
+            1)                                                          \
+           : 0;                                                         \
+         jv_i__ = jv_object_iter_next(t, jv_i__))                       \
 


 int jv_get_refcnt(jv);

-enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4 };
+enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4, JV_PRINT_SORTED = 8 };
 void jv_dump(jv, int flags);
 jv jv_dump_string(jv, int flags);

 jv jv_parse(const char* string);
 jv jv_parse_sized(const char* string, int length);

+typedef void (*jv_nomem_handler_f)(void *);
+void jv_nomem_handler(jv_nomem_handler_f, void *);

+jv jv_load_file(const char *, int);

+struct jv_parser;
+struct jv_parser* jv_parser_new();
+void jv_parser_set_buf(struct jv_parser*, const char*, int, int);
+jv jv_parser_next(struct jv_parser*);
+void jv_parser_free(struct jv_parser*);

-
-
-
-
+jv jv_get(jv, jv);
+jv jv_set(jv, jv, jv);
+jv jv_has(jv, jv);
+jv jv_setpath(jv, jv, jv);
+jv jv_getpath(jv, jv);
+jv jv_delpaths(jv, jv);
+jv jv_keys(jv /*object or array*/);
+int jv_cmp(jv, jv);
+jv jv_group(jv, jv);
+jv jv_sort(jv, jv);


 #endif
@@ -2,11 +2,89 @@
 #include <stdio.h>
 #include "jv_alloc.h"

+struct nomem_handler {
+    jv_nomem_handler_f handler;
+    void *data;
+};
+
+#ifndef USE_PTHREAD_KEY
+#ifdef _MSC_VER
+static __declspec(thread) struct nomem_handler nomem_handler;
+#else
+static __thread struct nomem_handler nomem_handler;
+#endif
+
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  nomem_handler.handler = handler;
+}
+
+static void memory_exhausted() {
+  if (nomem_handler.handler)
+    nomem_handler.handler(nomem_handler.data); // Maybe handler() will longjmp() to safety
+  // Or not
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+#else
+#ifdef HAVE_PTHREAD_KEY_CREATE
+#include <pthread.h>
+
+pthread_key_t nomem_handler_key;
+pthread_once_t mem_once = PTHREAD_ONCE_INIT;
+
+static void tsd_init(void) {
+  if (pthread_key_create(&nomem_handler_key, NULL) != 0) {
+    fprintf(stderr, "error: cannot create thread specific key");
+    abort();
+  }
+}
+
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  pthread_once(&mem_once, tsd_init); // cannot fail
+  struct nomem_handler *nomem_handler = calloc(1, sizeof (nomem_handler));
+  if (nomem_handler == NULL) {
+    handler(data);
+    fprintf(stderr, "error: cannot allocate memory\n");
+    abort();
+  }
+  nomem_handler.handler = handler;
+  nomem_handler.data = data;
+  if (pthread_setspecific(nomem_handler_key, nomem_handler) != 0) {
+    handler(data);
+    fprintf(stderr, "error: cannot set thread specific data");
+    abort();
+  }
+}
+
+static void memory_exhausted() {
+  jv_nomem_handler_f handler;
+
+  pthread_once(&mem_once, tsd_init);
+  handler = pthread_getspecific(nomem_handler_key);
+  if (handler)
+    handler(); // Maybe handler() will longjmp() to safety
+  // Or not
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+
+#else
+
+static struct nomem_handler nomem_handler;
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  nomem_handler.handler = handler;
+  nomem_handler.data = data;
+}
+
 static void memory_exhausted() {
  fprintf(stderr, "error: cannot allocate memory\n");
  abort();
 }

+#endif /* HAVE_PTHREAD_KEY_CREATE */
+#endif /* !USE_PTHREAD_KEY */
+
+
 void* jv_mem_alloc(size_t sz) {
  void* p = malloc(sz);
  if (!p) {
@@ -15,6 +93,10 @@ void* jv_mem_alloc(size_t sz) {
  return p;
 }

+void* jv_mem_alloc_unguarded(size_t sz) {
+  return malloc(sz);
+}
+
 void jv_mem_free(void* p) {
  free(p);
 }
@@ -26,3 +108,12 @@ void* jv_mem_realloc(void* p, size_t sz) {
  }
  return p;
 }
+
+#ifndef NDEBUG
+volatile char jv_mem_uninitialised;
+__attribute__((constructor)) void jv_mem_uninit_setup(){
+  char* p = malloc(1);
+  jv_mem_uninitialised = *p;
+  free(p);
+}
+#endif
@@ -2,8 +2,21 @@
 #define JV_ALLOC_H

 #include <stddef.h>
+#include "jv.h"
+
+#ifndef NDEBUG
+extern volatile char jv_mem_uninitialised;
+#endif
+
+static void jv_mem_invalidate(void* mem, size_t n) {
+#ifndef NDEBUG
+  char* m = mem;
+  while (n--) *m++ ^= jv_mem_uninitialised ^ jv_mem_uninitialised;
+#endif
+}

 void* jv_mem_alloc(size_t);
+void* jv_mem_alloc_unguarded(size_t);
 void jv_mem_free(void*);
 __attribute__((warn_unused_result)) void* jv_mem_realloc(void*, size_t);

@@ -1,9 +1,44 @@
-#include "jv_aux.h"
 #include <string.h>
 #include <stdlib.h>
+#include <assert.h>
 #include "jv_alloc.h"

-jv jv_lookup(jv t, jv k) {
+static int parse_slice(jv array, jv slice, int* pstart, int* pend) {
+  // Array slices
+  int len = jv_array_length(jv_copy(array));
+  jv start_jv = jv_object_get(jv_copy(slice), jv_string("start"));
+  jv end_jv = jv_object_get(slice, jv_string("end"));
+  if (jv_get_kind(start_jv) == JV_KIND_NULL) {
+    jv_free(start_jv);
+    start_jv = jv_number(0);
+  }
+  if (jv_get_kind(end_jv) == JV_KIND_NULL) {
+    jv_free(end_jv);
+    end_jv = jv_number(len);
+  }
+  if (jv_get_kind(start_jv) != JV_KIND_NUMBER ||
+      jv_get_kind(end_jv) != JV_KIND_NUMBER) {
+    jv_free(start_jv);
+    jv_free(end_jv);
+    return 0;
+  } else {
+    int start = (int)jv_number_value(start_jv);
+    int end = (int)jv_number_value(end_jv);
+    if (start < 0) start = len + start;
+    if (end < 0) end = len + end;
+
+    if (start < 0) start = 0;
+    if (start > len) start = len;
+    if (end > len) end = len;
+    if (end < start) end = start;
+    assert(0 <= start && start <= end && end <= len);
+    *pstart = start;
+    *pend = end;
+    return 1;
+  }
+}
+
+jv jv_get(jv t, jv k) {
  jv v;
  if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) {
    v = jv_object_get(t, k);
@@ -18,8 +53,18 @@ jv jv_lookup(jv t, jv k) {
      jv_free(v);
      v = jv_null();
    }
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_OBJECT) {
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      v = jv_array_slice(t, start, end);
+    } else {
+      v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+      jv_free(t);
+    }
  } else if (jv_get_kind(t) == JV_KIND_NULL && 
-             (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER)) {
+             (jv_get_kind(k) == JV_KIND_STRING || 
+              jv_get_kind(k) == JV_KIND_NUMBER || 
+              jv_get_kind(k) == JV_KIND_OBJECT)) {
    jv_free(t);
    jv_free(k);
    v = jv_null();
@@ -33,7 +78,12 @@ jv jv_lookup(jv t, jv k) {
  return v;
 }

-jv jv_modify(jv t, jv k, jv v) {
+jv jv_set(jv t, jv k, jv v) {
+  if (!jv_is_valid(v)) {
+    jv_free(t);
+    jv_free(k);
+    return v;
+  }
  int isnull = jv_get_kind(t) == JV_KIND_NULL;
  if (jv_get_kind(k) == JV_KIND_STRING && 
      (jv_get_kind(t) == JV_KIND_OBJECT || isnull)) {
@@ -43,10 +93,49 @@ jv jv_modify(jv t, jv k, jv v) {
             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
    if (isnull) t = jv_array();
    t = jv_array_set(t, (int)jv_number_value(k), v);
+  } else if (jv_get_kind(k) == JV_KIND_OBJECT &&
+             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
+    if (isnull) t = jv_array();
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      if (jv_get_kind(v) == JV_KIND_ARRAY) {
+        int array_len = jv_array_length(jv_copy(t));
+        assert(0 <= start && start <= end && end <= array_len);
+        int slice_len = end - start;
+        int insert_len = jv_array_length(jv_copy(v));
+        if (slice_len < insert_len) {
+          // array is growing
+          int shift = insert_len - slice_len;
+          for (int i = array_len - 1; i >= end; i--) {
+            t = jv_array_set(t, i + shift, jv_array_get(jv_copy(t), i));
+          }
+        } else if (slice_len > insert_len) {
+          // array is shrinking
+          int shift = slice_len - insert_len;
+          for (int i = end; i < array_len; i++) {
+            t = jv_array_set(t, i - shift, jv_array_get(jv_copy(t), i));
+          }
+          t = jv_array_slice(t, 0, array_len - shift);
+        }
+        for (int i=0; i < insert_len; i++) {
+          t = jv_array_set(t, start + i, jv_array_get(jv_copy(v), i));
+        }
+        jv_free(v);
+      } else {
+        jv_free(t);
+        jv_free(v);
+        t = jv_invalid_with_msg(jv_string_fmt("A slice of an array can only be assigned another array"));
+      }
+    } else {
+      jv_free(t);
+      jv_free(k);
+      jv_free(v);
+      t = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+    }
  } else {
    jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s",
-                                               jv_kind_name(jv_get_kind(t)),
-                                               jv_kind_name(jv_get_kind(v))));
+                                               jv_kind_name(jv_get_kind(k)),
+                                               jv_kind_name(jv_get_kind(t))));
    jv_free(t);
    jv_free(k);
    jv_free(v);
@@ -55,21 +144,244 @@ jv jv_modify(jv t, jv k, jv v) {
  return t;
 }

-jv jv_insert(jv root, jv value, jv* path, int pathlen) {
-  if (pathlen == 0) {
+jv jv_has(jv t, jv k) {
+  assert(jv_is_valid(t));
+  assert(jv_is_valid(k));
+  jv ret;
+  if (jv_get_kind(t) == JV_KIND_NULL) {
+    jv_free(t);
+    jv_free(k);
+    ret = jv_false();
+  } else if (jv_get_kind(t) == JV_KIND_OBJECT &&
+             jv_get_kind(k) == JV_KIND_STRING) {
+    jv elem = jv_object_get(t, k);
+    ret = jv_bool(jv_is_valid(elem));
+    jv_free(elem);
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY &&
+             jv_get_kind(k) == JV_KIND_NUMBER) {
+    jv elem = jv_array_get(t, (int)jv_number_value(k));
+    ret = jv_bool(jv_is_valid(elem));
+    jv_free(elem);
+  } else {
+    ret = jv_invalid_with_msg(jv_string_fmt("Cannot check whether %s has a %s key",
+                                            jv_kind_name(jv_get_kind(t)),
+                                            jv_kind_name(jv_get_kind(k))));
+    jv_free(t);
+    jv_free(k);
+  }
+  return ret;
+}
+
+// assumes keys is a sorted array
+jv jv_dels(jv t, jv keys) {
+  assert(jv_get_kind(keys) == JV_KIND_ARRAY);
+  assert(jv_is_valid(t));
+  
+  if (jv_get_kind(t) == JV_KIND_NULL || jv_array_length(jv_copy(keys)) == 0) {
+    // no change
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY) {
+    // extract slices, they must be handled differently
+    jv orig_keys = keys;
+    keys = jv_array();
+    jv new_array = jv_array();
+    jv starts = jv_array(), ends = jv_array();
+    jv_array_foreach(orig_keys, i, key) {
+      if (jv_get_kind(key) == JV_KIND_NUMBER) {
+        keys = jv_array_append(keys, key);
+      } else if (jv_get_kind(key) == JV_KIND_OBJECT) {
+        int start, end;
+        if (parse_slice(t, key, &start, &end)) {
+          starts = jv_array_append(starts, jv_number(start));
+          ends = jv_array_append(ends, jv_number(end));
+        } else {
+          jv_free(new_array);
+          jv_free(key);
+          new_array = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+          goto arr_out;
+        }
+      } else {
+        jv_free(new_array);
+        new_array = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array",
+                                                      jv_kind_name(jv_get_kind(key))));
+        jv_free(key);
+        goto arr_out;
+      }
+    }
+
+    int kidx = 0;
+    jv_array_foreach(t, i, elem) {
+      int del = 0;
+      while (kidx < jv_array_length(jv_copy(keys))) {
+        int delidx = (int)jv_number_value(jv_array_get(jv_copy(keys), kidx));
+        if (i == delidx) {
+          del = 1;
+        }
+        if (i < delidx) {
+          break;
+        }
+        kidx++;
+      }
+      for (int sidx=0; !del && sidx<jv_array_length(jv_copy(starts)); sidx++) {
+        if ((int)jv_number_value(jv_array_get(jv_copy(starts), sidx)) <= i &&
+            i < (int)jv_number_value(jv_array_get(jv_copy(ends), sidx))) {
+          del = 1;
+        }
+      }
+      if (!del)
+        new_array = jv_array_append(new_array, elem);
+      else
+        jv_free(elem);
+    }
+  arr_out:
+    jv_free(starts);
+    jv_free(ends);
+    jv_free(orig_keys);
+    jv_free(t);
+    t = new_array;
+  } else if (jv_get_kind(t) == JV_KIND_OBJECT) {
+    jv_array_foreach(keys, i, k) {
+      if (jv_get_kind(k) != JV_KIND_STRING) {
+        jv_free(t);
+        t = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s field of object",
+                                              jv_kind_name(jv_get_kind(k))));
+        jv_free(k);
+        break;
+      }
+      t = jv_object_delete(t, k);      
+    }
+  } else {
+    jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete fields from %s",
+                                               jv_kind_name(jv_get_kind(t))));
+    jv_free(t);
+    t = err;
+  }
+  jv_free(keys);
+  return t;
+}
+
+jv jv_setpath(jv root, jv path, jv value) {
+  if (jv_get_kind(path) != JV_KIND_ARRAY) {
+    jv_free(value);
+    jv_free(root);
+    jv_free(path);
+    return jv_invalid_with_msg(jv_string("Path must be specified as an array"));
+  }
+  if (!jv_is_valid(root)){
+    jv_free(value);
+    jv_free(path);
+    return root;
+  }
+  if (jv_array_length(jv_copy(path)) == 0) {
+    jv_free(path);
    jv_free(root);
    return value;
  }
-  return jv_modify(root, jv_copy(*path), 
-                   jv_insert(jv_lookup(jv_copy(root), jv_copy(*path)), value, path+1, pathlen-1));
+  jv pathcurr = jv_array_get(jv_copy(path), 0);
+  jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path)));
+  return jv_set(root, pathcurr, 
+                jv_setpath(jv_get(jv_copy(root), jv_copy(pathcurr)), pathrest, value));
+}
+
+jv jv_getpath(jv root, jv path) {
+  if (jv_get_kind(path) != JV_KIND_ARRAY) {
+    jv_free(root);
+    jv_free(path);
+    return jv_invalid_with_msg(jv_string("Path must be specified as an array"));
+  }
+  if (!jv_is_valid(root)) {
+    jv_free(path);
+    return root;
+  }
+  if (jv_array_length(jv_copy(path)) == 0) {
+    jv_free(path);
+    return root;
+  }
+  jv pathcurr = jv_array_get(jv_copy(path), 0);
+  jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path)));
+  return jv_getpath(jv_get(root, pathcurr), pathrest);
+}
+
+// assumes paths is a sorted array of arrays
+static jv delpaths_sorted(jv object, jv paths, int start) {
+  jv delkeys = jv_array();
+  for (int i=0; i<jv_array_length(jv_copy(paths));) {
+    int j = i;
+    assert(jv_array_length(jv_array_get(jv_copy(paths), i)) > start);
+    int delkey = jv_array_length(jv_array_get(jv_copy(paths), i)) == start + 1;
+    jv key = jv_array_get(jv_array_get(jv_copy(paths), i), start);
+    while (j < jv_array_length(jv_copy(paths)) &&
+           jv_equal(jv_copy(key), jv_array_get(jv_array_get(jv_copy(paths), j), start)))
+      j++;
+    // if i <= entry < j, then entry starts with key
+    if (delkey) {
+      // deleting this entire key, we don't care about any more specific deletions
+      delkeys = jv_array_append(delkeys, key);
+    } else {
+      // deleting certain sub-parts of this key
+      jv subobject = jv_get(jv_copy(object), jv_copy(key));
+      if (!jv_is_valid(subobject)) {
+        jv_free(key);
+        jv_free(object);
+        object = subobject;
+        break;
+      } else if (jv_get_kind(subobject) == JV_KIND_NULL) {
+        jv_free(key);
+        jv_free(subobject);
+      } else {
+        jv newsubobject = delpaths_sorted(subobject, jv_array_slice(jv_copy(paths), i, j), start+1);
+        if (!jv_is_valid(newsubobject)) {
+          jv_free(key);
+          jv_free(object);
+          object = newsubobject;
+          break;
+        }
+        object = jv_set(object, key, newsubobject);
+      }
+      if (!jv_is_valid(object)) break;
+    }
+    i = j;
+  }
+  jv_free(paths);
+  if (jv_is_valid(object))
+    object = jv_dels(object, delkeys);
+  else 
+    jv_free(delkeys);
+  return object;
+}
+
+jv jv_delpaths(jv object, jv paths) {
+  paths = jv_sort(paths, jv_copy(paths));
+  jv_array_foreach(paths, i, elem) {
+    if (jv_get_kind(elem) != JV_KIND_ARRAY) {
+      jv_free(object);
+      jv_free(paths);
+      jv err = jv_invalid_with_msg(jv_string_fmt("Path must be specified as array, not %s",
+                                                 jv_kind_name(jv_get_kind(elem))));
+      jv_free(elem);
+      return err;
+    }
+    jv_free(elem);
+  }
+  if (jv_array_length(jv_copy(paths)) == 0) {
+    // nothing is being deleted
+    jv_free(paths);
+    return object;
+  }
+  if (jv_array_length(jv_array_get(jv_copy(paths), 0)) == 0) {
+    // everything is being deleted
+    jv_free(paths);
+    jv_free(object);
+    return jv_null();
+  }
+  return delpaths_sorted(object, paths, 0);
 }


 static int string_cmp(const void* pa, const void* pb){
  const jv* a = pa;
  const jv* b = pb;
-  int lena = jv_string_length(jv_copy(*a));
-  int lenb = jv_string_length(jv_copy(*b));
+  int lena = jv_string_length_bytes(jv_copy(*a));
+  int lenb = jv_string_length_bytes(jv_copy(*b));
  int minlen = lena < lenb ? lena : lenb;
  int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen);
  if (r == 0) r = lena - lenb;
@@ -81,8 +393,9 @@ jv jv_keys(jv x) {
    int nkeys = jv_object_length(jv_copy(x));
    jv* keys = jv_mem_alloc(sizeof(jv) * nkeys);
    int kidx = 0;
-    jv_object_foreach(i, x) {
-      keys[kidx++] = jv_object_iter_key(x, i);
+    jv_object_foreach(x, key, value) {
+      keys[kidx++] = key;
+      jv_free(value);
    }
    qsort(keys, nkeys, sizeof(jv), string_cmp);
    jv answer = jv_array_sized(nkeys);
@@ -163,8 +476,7 @@ int jv_cmp(jv a, jv b) {
    jv keys_b = jv_keys(jv_copy(b));
    r = jv_cmp(jv_copy(keys_a), keys_b);
    if (r == 0) {
-      for (int i=0; i<jv_array_length(jv_copy(keys_a)); i++) {
-        jv key = jv_array_get(jv_copy(keys_a), i);
+      jv_array_foreach(keys_a, i, key) {
        jv xa = jv_object_get(jv_copy(a), jv_copy(key));
        jv xb = jv_object_get(jv_copy(b), key);
        r = jv_cmp(xa, xb);
@@ -1,17 +0,0 @@
-#ifndef JV_AUX_H
-#define JV_AUX_H
-
-#include "jv.h"
-
-jv jv_lookup(jv t, jv k);
-jv jv_modify(jv t, jv k, jv v);
-jv jv_insert(jv root, jv value, jv* path, int pathlen);
-
-jv jv_keys(jv /*object or array*/);
-int jv_cmp(jv, jv);
-
-jv jv_group(jv objects, jv keys);
-jv jv_sort(jv objects, jv keys);
-
-
-#endif
@@ -0,0 +1,46 @@
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "jv.h"
+
+jv jv_load_file(const char* filename, int raw) {
+  FILE* file = fopen(filename, "r");
+  struct jv_parser* parser;
+  jv data;
+  if (!file) {
+    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
+                                             filename,
+                                             strerror(errno)));
+  }
+  if (raw) {
+    data = jv_string("");
+  } else {
+    data = jv_array();
+    parser = jv_parser_new();
+  }
+  while (!feof(file) && !ferror(file)) {
+    char buf[4096];
+    size_t n = fread(buf, 1, sizeof(buf), file);
+    if (raw) {
+      data = jv_string_concat(data, jv_string_sized(buf, (int)n));
+    } else {
+      jv_parser_set_buf(parser, buf, n, !feof(file));
+      jv value;
+      while (jv_is_valid((value = jv_parser_next(parser))))
+        data = jv_array_append(data, value);
+      jv_free(value);
+    }
+  }
+  if (!raw)
+      jv_parser_free(parser);
+  int badread = ferror(file);
+  fclose(file);
+  if (badread) {
+    jv_free(data);
+    return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
+                                             filename));
+  }
+  return data;
+}
@@ -1,11 +1,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 #include "jv.h"
 #include "jv_dtoa.h"
-#include "jv_parse.h"
 #include "jv_unicode.h"
 #include "jv_alloc.h"
+#include "jv_dtoa.h"

 typedef const char* presult;

@@ -16,7 +17,35 @@ typedef const char* presult;
 #define pfunc presult
 #endif

-void jv_parser_init(struct jv_parser* p) {
+struct jv_parser {
+  const char* curr_buf;
+  int curr_buf_length;
+  int curr_buf_pos;
+  int curr_buf_is_partial;
+  unsigned bom_strip_position;
+
+  jv* stack;
+  int stackpos;
+  int stacklen;
+  jv next;
+  
+  char* tokenbuf;
+  int tokenpos;
+  int tokenlen;
+
+  int line, column;
+  
+  struct dtoa_context dtoa;
+
+  enum {
+    JV_PARSER_NORMAL,
+    JV_PARSER_STRING,
+    JV_PARSER_STRING_ESCAPE
+  } st;
+};
+
+
+static void parser_init(struct jv_parser* p) {
  p->stack = 0;
  p->stacklen = p->stackpos = 0;
  p->next = jv_invalid();
@@ -26,10 +55,12 @@ void jv_parser_init(struct jv_parser* p) {
  p->curr_buf = 0;
  p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0;
  p->bom_strip_position = 0;
+  p->line = 1;
+  p->column = 0;
  jvp_dtoa_context_init(&p->dtoa);
 }

-void jv_parser_free(struct jv_parser* p) {
+static void parser_free(struct jv_parser* p) {
  jv_free(p->next);
  for (int i=0; i<p->stackpos; i++) 
    jv_free(p->stack[i]);
@@ -281,7 +312,7 @@ static chclass classify(char c) {
 }


-static presult OK = "output produced";
+static const presult OK = "output produced";

 static int check_done(struct jv_parser* p, jv* out) {
  if (p->stackpos == 0 && jv_is_valid(p->next)) {
@@ -294,6 +325,11 @@ static int check_done(struct jv_parser* p, jv* out) {
 }

 static pfunc scan(struct jv_parser* p, char ch, jv* out) {
+  p->column++;
+  if (ch == '\n') {
+    p->line++;
+    p->column = 0;
+  }
  presult answer = 0;
  if (p->st == JV_PARSER_NORMAL) {
    chclass cls = classify(ch);
@@ -334,7 +370,18 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
  return answer;
 }

-static unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};
+struct jv_parser* jv_parser_new() {
+  struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser));
+  parser_init(p);
+  return p;
+}
+
+void jv_parser_free(struct jv_parser* p) {
+  parser_free(p);
+  jv_mem_free(p);
+}
+
+static const unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};

 void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) {
  assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length)
@@ -373,7 +420,7 @@ jv jv_parser_next(struct jv_parser* p) {
  if (msg == OK) {
    return value;
  } else if (msg) {
-    return jv_invalid_with_msg(jv_string(msg));
+    return jv_invalid_with_msg(jv_string_fmt("%s at line %d, column %d", msg, p->line, p->column));
  } else if (p->curr_buf_is_partial) {
    assert(p->curr_buf_pos == p->curr_buf_length);
    // need another buffer
@@ -397,7 +444,7 @@ jv jv_parser_next(struct jv_parser* p) {

 jv jv_parse_sized(const char* string, int length) {
  struct jv_parser parser;
-  jv_parser_init(&parser);
+  parser_init(&parser);
  jv_parser_set_buf(&parser, string, length, 0);
  jv value = jv_parser_next(&parser);
  if (jv_is_valid(value)) {
@@ -422,7 +469,7 @@ jv jv_parse_sized(const char* string, int length) {
    jv_free(value);
    value = jv_invalid_with_msg(jv_string("Expected JSON value"));
  }
-  jv_parser_free(&parser);
+  parser_free(&parser);

  if (!jv_is_valid(value) && jv_invalid_has_msg(jv_copy(value))) {
    jv msg = jv_invalid_get_msg(value);
@@ -1,35 +0,0 @@
-#ifndef JV_PARSE_H
-#define JV_PARSE_H
-#include "jv_dtoa.h"
-struct jv_parser {
-  const char* curr_buf;
-  int curr_buf_length;
-  int curr_buf_pos;
-  int curr_buf_is_partial;
-  unsigned bom_strip_position;
-
-  jv* stack;
-  int stackpos;
-  int stacklen;
-  jv next;
-  
-  char* tokenbuf;
-  int tokenpos;
-  int tokenlen;
-  
-  struct dtoa_context dtoa;
-
-  enum {
-    JV_PARSER_NORMAL,
-    JV_PARSER_STRING,
-    JV_PARSER_STRING_ESCAPE
-  } st;
-};
-
-void jv_parser_init(struct jv_parser* p);
-void jv_parser_free(struct jv_parser* p);
-
-void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial);
-
-jv jv_parser_next(struct jv_parser* p);
-#endif
@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <float.h>
 #include <string.h>
+#include <assert.h>

 #include "jv_dtoa.h"
 #include "jv_unicode.h"
@@ -12,10 +13,10 @@

 // Colour table. See http://en.wikipedia.org/wiki/ANSI_escape_code#Colors
 // for how to choose these.
-static jv_kind colour_kinds[] = 
+static const jv_kind colour_kinds[] = 
  {JV_KIND_NULL,   JV_KIND_FALSE, JV_KIND_TRUE, JV_KIND_NUMBER,
   JV_KIND_STRING, JV_KIND_ARRAY, JV_KIND_OBJECT};
-static const char* colours[] =
+static const char* const colours[] =
  {COL("30;1"),    COL("0"),      COL("0"),     COL("0"),
   COL("32"),      COL("37"),     COL("37")};
 #define FIELD_COLOUR COL("34;1")
@@ -45,7 +46,7 @@ static void put_space(int n, FILE* fout, jv* strout) {
 static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) {
  assert(jv_get_kind(str) == JV_KIND_STRING);
  const char* i = jv_string_value(str);
-  const char* end = i + jv_string_length(jv_copy(str));
+  const char* end = i + jv_string_length_bytes(jv_copy(str));
  const char* cstart;
  int c = 0;
  char buf[32];
@@ -124,6 +125,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
    }
  }
  switch (jv_get_kind(x)) {
+  default:
  case JV_KIND_INVALID:
    assert(0 && "Invalid value");
    break;
@@ -162,7 +164,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
      put_char('\n', F, S);
      put_space(indent+INDENT, F, S);
    }
-    for (int i=0; i<jv_array_length(jv_copy(x)); i++) {
+    jv_array_foreach(x, i, elem) {
      if (i!=0) {
        if (flags & JV_PRINT_PRETTY) {
          put_str(",\n", F, S);
@@ -171,7 +173,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
          put_str(",", F, S);
        }
      }
-      jv_dump_term(C, jv_array_get(jv_copy(x), i), flags, indent + INDENT, F, S);
+      jv_dump_term(C, elem, flags, indent + INDENT, F, S);
      if (colour) put_str(colour, F, S);
    }
    if (flags & JV_PRINT_PRETTY) {
@@ -193,7 +195,34 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
      put_space(indent+INDENT, F, S);
    }
    int first = 1;
-    jv_object_foreach(i, x) {
+    int i = 0;
+    jv keyset = jv_null();
+    while (1) {
+      jv key, value;
+      if (flags & JV_PRINT_SORTED) {
+        if (first) {
+          keyset = jv_keys(jv_copy(x));
+          i = 0;
+        } else {
+          i++;
+        }
+        if (i >= jv_array_length(jv_copy(keyset))) {
+          jv_free(keyset);
+          break;
+        }
+        key = jv_array_get(jv_copy(keyset), i);
+        value = jv_object_get(jv_copy(x), jv_copy(key));
+      } else {
+        if (first) {
+          i = jv_object_iter(x);
+        } else {
+          i = jv_object_iter_next(x, i);
+        }
+        if (!jv_object_iter_valid(x, i)) break;
+        key = jv_object_iter_key(x, i);
+        value = jv_object_iter_value(x, i);
+      }
+
      if (!first) {
        if (flags & JV_PRINT_PRETTY){
          put_str(",\n", F, S);
@@ -205,7 +234,6 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
      if (colour) put_str(COLRESET, F, S);

      first = 0;
-      jv key = jv_object_iter_key(x, i);
      if (colour) put_str(FIELD_COLOUR, F, S);
      jvp_dump_string(key, flags & JV_PRINT_ASCII, F, S);
      jv_free(key);
@@ -215,7 +243,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
      put_str((flags & JV_PRINT_PRETTY) ? ": " : ":", F, S);
      if (colour) put_str(COLRESET, F, S);
      
-      jv_dump_term(C, jv_object_iter_value(x, i), flags, indent + INDENT, F, S);
+      jv_dump_term(C, value, flags, indent + INDENT, F, S);
      if (colour) put_str(colour, F, S);
    }
    if (flags & JV_PRINT_PRETTY) {
@@ -1,7 +1,7 @@
 #include <stdio.h>
 #include <assert.h>
 #include "jv_unicode.h"
-#include "jv_utf8_tables.gen.h"
+#include "jv_utf8_tables.h"

 const char* jvp_utf8_next(const char* in, const char* end, int* codepoint) {
  if (in == end) {
@@ -0,0 +1,35 @@
+#define UTF8_CONTINUATION_BYTE ((unsigned char)255)
+static const unsigned char utf8_coding_length[] =
+ {0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+  0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
+  0x04, 0x04, 0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+static const unsigned char utf8_coding_bits[] =
+ {0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+  0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+  0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+  0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+  0x00, 0x00, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f,
+  0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f,
+  0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
+  0x07, 0x07, 0x07, 0x07, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
@@ -1,10 +1,11 @@
 %{
+#include <assert.h>
 #include "jv_alloc.h"
 #include "compile.h"

 struct lexer_param;

-#include "parser.gen.h"  /* Generated by bison. */
+#include "parser.h"  /* Generated by bison. */

 #define YY_USER_ACTION                           \
  do {                                           \
@@ -49,6 +50,7 @@ struct lexer_param;
 "and" { return AND; }
 "or" { return OR; }
 "end" { return END; }
+"reduce" { return REDUCE; }
 "//" { return DEFINEDOR; }
 "|=" { return SETPIPE; }
 "+=" { return SETPLUS; }
@@ -68,8 +70,11 @@ struct lexer_param;
  return try_exit(yytext[0], YY_START, yyscanner);
 }

+"@"[a-zA-Z0-9_]+ {
+  yylval->literal = jv_string_sized(yytext + 1, yyleng - 1); return FORMAT;
+}

-?[0-9.]+([eE][+-]?[0-9]+)? { 
+[0-9.]+([eE][+-]?[0-9]+)? { 
   yylval->literal = jv_parse_sized(yytext, yyleng); return LITERAL; 
 }

@@ -89,7 +94,7 @@ struct lexer_param;
  (\\[^u(]|\\u[a-zA-Z0-9]{0,4})+ {
    /* pass escapes to the json parser */
    jv escapes = jv_string_fmt("\"%.*s\"", yyleng, yytext);
-    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length(jv_copy(escapes)));
+    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length_bytes(jv_copy(escapes)));
    jv_free(escapes);
    return QQSTRING_TEXT;
  }
@@ -104,6 +109,7 @@ struct lexer_param;
  

 [a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext); return IDENT;}
+\.[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext+1); return FIELD;}

 [ \n\t]+  {}

@@ -0,0 +1,65 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <stdarg.h>
+#include "jv_alloc.h"
+#include "locfile.h"
+
+
+void locfile_init(struct locfile* l, const char* data, int length) {
+  l->data = data;
+  l->length = length;
+  l->nlines = 1;
+  for (int i=0; i<length; i++) {
+    if (data[i] == '\n') l->nlines++;
+  }
+  l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
+  l->linemap[0] = 0;
+  int line = 1;
+  for (int i=0; i<length; i++) {
+    if (data[i] == '\n') {
+      l->linemap[line] = i+1;   // at start of line, not of \n
+      line++;
+    }
+  }
+  l->linemap[l->nlines] = length+1;   // virtual last \n
+}
+
+void locfile_free(struct locfile* l) {
+  jv_mem_free(l->linemap);
+}
+
+static int locfile_get_line(struct locfile* l, int pos) {
+  assert(pos < l->length);
+  int line = 1;
+  while (l->linemap[line] <= pos) line++;   // == if pos at start (before, never ==, because pos never on \n)
+  assert(line-1 < l->nlines);
+  return line-1;
+}
+
+static int locfile_line_length(struct locfile* l, int line) {
+  assert(line < l->nlines);
+  return l->linemap[line+1] - l->linemap[line] -1;   // -1 to omit \n
+}
+
+void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
+  va_list fmtargs;
+  va_start(fmtargs, fmt);
+  vfprintf(stderr, fmt, fmtargs);
+  va_end(fmtargs);
+  fprintf(stderr, "\n");
+  if (loc.start == -1) {
+    fprintf(stderr, "<unknown location>\n");
+    return;
+  }
+  int startline = locfile_get_line(l, loc.start);
+  int offset = l->linemap[startline];
+  fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset);
+  fprintf(stderr, "%*s", loc.start - offset, "");
+  for (int i = loc.start; 
+       i < loc.end && i < offset + locfile_line_length(l, startline);
+       i++){
+    fprintf(stderr, "^");
+  }
+  fprintf(stderr, "\n");
+}
@@ -1,10 +1,6 @@
 #ifndef _LOCFILE_H
 #define _LOCFILE_H
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include <stdarg.h>
-#include "jv_alloc.h"
+
 typedef struct {
  int start, end;
 } location;
@@ -18,62 +14,10 @@ struct locfile {
  int nlines;
 };

-static void locfile_init(struct locfile* l, const char* data, int length) {
-  l->data = data;
-  l->length = length;
-  l->nlines = 1;
-  for (int i=0; i<length; i++) {
-    if (data[i] == '\n') l->nlines++;
-  }
-  l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
-  l->linemap[0] = 0;
-  int line = 1;
-  for (int i=0; i<length; i++) {
-    if (data[i] == '\n') {
-      l->linemap[line] = i;
-      line++;
-    }
-  }
-  l->linemap[l->nlines] = length;
-}
+void locfile_init(struct locfile* l, const char* data, int length);

-static void locfile_free(struct locfile* l) {
-  jv_mem_free(l->linemap);
-}
+void locfile_free(struct locfile* l);

-static int locfile_get_line(struct locfile* l, int pos) {
-  assert(pos < l->length);
-  int line = 0;
-  while (l->linemap[line+1] < pos) line++;
-  assert(line < l->nlines);
-  return line;
-}
-
-static int locfile_line_length(struct locfile* l, int line) {
-  assert(line < l->nlines);
-  return l->linemap[line+1] - l->linemap[line];
-}
-
-static void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
-  va_list fmtargs;
-  va_start(fmtargs, fmt);
-  vfprintf(stderr, fmt, fmtargs);
-  va_end(fmtargs);
-  fprintf(stderr, "\n");
-  if (loc.start == -1) {
-    fprintf(stderr, "<unknown location>\n");
-    return;
-  }
-  int startline = locfile_get_line(l, loc.start);
-  int offset = l->linemap[startline];
-  fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset);
-  fprintf(stderr, "%*s", loc.start - offset, "");
-  for (int i = loc.start; 
-       i < loc.end && i < offset + locfile_line_length(l, startline);
-       i++){
-    fprintf(stderr, "^");
-  }
-  fprintf(stderr, "\n");
-}
+void locfile_locate(struct locfile* l, location loc, const char* fmt, ...);

 #endif
@@ -1,18 +1,16 @@
-#define _POSIX_SOURCE
+#include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
 #include <ctype.h>
 #include <unistd.h>
 #include "compile.h"
-#include "builtin.h"
 #include "jv.h"
-#include "jv_parse.h"
-#include "locfile.h"
-#include "parser.h"
-#include "execute.h"
+#include "jq.h"
 #include "jv_alloc.h"
-#include "version.gen.h"
+#include "version.h"
+
+int jq_testsuite(int argc, char* argv[]);

 static const char* progname;

@@ -56,25 +54,32 @@ enum {
  ASCII_OUTPUT = 32,
  COLOUR_OUTPUT = 64,
  NO_COLOUR_OUTPUT = 128,
+  SORTED_OUTPUT = 256,

-  FROM_FILE = 256,
+  FROM_FILE = 512,
+
+  /* debugging only */
+  DUMP_DISASM = 2048,
 };
 static int options = 0;
-static struct bytecode* bc;

-static void process(jv value) {
-  jq_init(bc, value);
+static void process(jq_state *jq, jv value, int flags) {
+  jq_start(jq, value, flags);
  jv result;
-  while (jv_is_valid(result = jq_next())) {
+  while (jv_is_valid(result = jq_next(jq))) {
    if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
-      fwrite(jv_string_value(result), 1, jv_string_length(jv_copy(result)), stdout);
+      fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
+      jv_free(result);
    } else {
      int dumpopts;
-#ifdef JQ_DEFAULT_ENABLE_COLOR
-      dumpopts = JQ_DEFAULT_ENABLE_COLOR ? JV_PRINT_COLOUR : 0;
+      /* Disable colour by default on Windows builds as Windows
+         terminals tend not to display it correctly */
+#ifdef WIN32
+      dumpopts = 0;
 #else
      dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0;
 #endif
+      if (options & SORTED_OUTPUT) dumpopts |= JV_PRINT_SORTED;
      if (!(options & COMPACT_OUTPUT)) dumpopts |= JV_PRINT_PRETTY;
      if (options & ASCII_OUTPUT) dumpopts |= JV_PRINT_ASCII;
      if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR;
@@ -84,30 +89,6 @@ static void process(jv value) {
    printf("\n");
  }
  jv_free(result);
-  jq_teardown();
-}
-
-static jv slurp_file(const char* filename) {
-  FILE* file = fopen(filename, "r");
-  if (!file) {
-    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
-                                             filename,
-                                             strerror(errno)));
-  }
-  jv data = jv_string("");
-  while (!feof(file) && !ferror(file)) {
-    char buf[4096];
-    size_t n = fread(buf, 1, sizeof(buf), file);
-    data = jv_string_concat(data, jv_string_sized(buf, (int)n));
-  }
-  int badread = ferror(file);
-  fclose(file);
-  if (badread) {
-    jv_free(data);
-    return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
-                                             filename));
-  }
-  return data;
 }

 FILE* current_input;
@@ -134,17 +115,33 @@ static int read_more(char* buf, size_t size) {
    next_input_idx++;
  }

-  if (!fgets(buf, sizeof(buf), current_input)) buf[0] = 0;
+  if (!fgets(buf, size, current_input)) buf[0] = 0;
  return 1;
 }

 int main(int argc, char* argv[]) {
+  jq_state *jq;
+  int ret = 0;
+  int compiled = 0;
+
  if (argc) progname = argv[0];

+  if (argc > 1 && !strcmp(argv[1], "--run-tests")) {
+    return jq_testsuite(argc, argv);
+  }
+
+  jq = jq_init();
+  if (jq == NULL) {
+    perror("malloc");
+    return 1;
+  }
+
  const char* program = 0;
  input_filenames = jv_mem_alloc(sizeof(const char*) * argc);
  ninput_files = 0;
  int further_args_are_files = 0;
+  int jq_flags = 0;
+  jv program_arguments = jv_array();
  for (int i=1; i<argc; i++) {
    if (further_args_are_files) {
      input_filenames[ninput_files++] = argv[i];
@@ -169,16 +166,52 @@ int main(int argc, char* argv[]) {
      options |= NO_COLOUR_OUTPUT;
    } else if (isoption(argv[i], 'a', "ascii-output")) {
      options |= ASCII_OUTPUT;
+    } else if (isoption(argv[i], 'S', "sort-keys")) {
+      options |= SORTED_OUTPUT;
    } else if (isoption(argv[i], 'R', "raw-input")) {
      options |= RAW_INPUT;
    } else if (isoption(argv[i], 'n', "null-input")) {
      options |= PROVIDE_NULL;
    } else if (isoption(argv[i], 'f', "from-file")) {
      options |= FROM_FILE;
+    } else if (isoption(argv[i], 0, "arg")) {
+      if (i >= argc - 2) {
+        fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname);
+        die();
+      }
+      jv arg = jv_object();
+      arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
+      arg = jv_object_set(arg, jv_string("value"), jv_string(argv[i+2]));
+      program_arguments = jv_array_append(program_arguments, arg);
+      i += 2; // skip the next two arguments
+    } else if (isoption(argv[i], 0, "argfile")) {
+      if (i >= argc - 2) {
+        fprintf(stderr, "%s: --argfile takes two parameters (e.g. -a varname filename)\n", progname);
+        die();
+      }
+      jv arg = jv_object();
+      arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
+      jv data = jv_load_file(argv[i+2], 0);
+      if (!jv_is_valid(data)) {
+        data = jv_invalid_get_msg(data);
+        fprintf(stderr, "%s: Bad JSON in --argfile %s %s: %s\n", progname,
+                argv[i+1], argv[i+2], jv_string_value(data));
+        jv_free(data);
+        return 1;
+      }
+      if (jv_get_kind(data) == JV_KIND_ARRAY && jv_array_length(jv_copy(data)) == 1)
+          data = jv_array_get(data, 0);
+      arg = jv_object_set(arg, jv_string("value"), data);
+      program_arguments = jv_array_append(program_arguments, arg);
+      i += 2; // skip the next two arguments
+    } else if (isoption(argv[i],  0,  "debug-dump-disasm")) {
+      options |= DUMP_DISASM;
+    } else if (isoption(argv[i],  0,  "debug-trace")) {
+      jq_flags |= JQ_DEBUG_TRACE;
    } else if (isoption(argv[i], 'h', "help")) {
      usage();
    } else if (isoption(argv[i], 'V', "version")) {
-      fprintf(stderr, "jq version %s\n", JQ_VERSION);
+      printf("jq-%s\n", JQ_VERSION);
      return 0;
    } else {
      fprintf(stderr, "%s: Unknown option %s\n", progname, argv[i]);
@@ -194,27 +227,27 @@ int main(int argc, char* argv[]) {
  }
  
  if (options & FROM_FILE) {
-    jv data = slurp_file(program);
+    jv data = jv_load_file(program, 1);
    if (!jv_is_valid(data)) {
      data = jv_invalid_get_msg(data);
      fprintf(stderr, "%s: %s\n", progname, jv_string_value(data));
      jv_free(data);
      return 1;
    }
-    bc = jq_compile(jv_string_value(data));
+    compiled = jq_compile_args(jq, jv_string_value(data), program_arguments);
    jv_free(data);
  } else {
-    bc = jq_compile(program);
+    compiled = jq_compile_args(jq, program, program_arguments);
  }
-  if (!bc) return 1;
+  if (!compiled) return 1;

-#if JQ_DEBUG
-  dump_disassembly(0, bc);
-  printf("\n");
-#endif
+  if (options & DUMP_DISASM) {
+    jq_dump_disassembly(jq, 0);
+    printf("\n");
+  }

  if (options & PROVIDE_NULL) {
-    process(jv_null());
+    process(jq, jv_null(), jq_flags);
  } else {
    jv slurped;
    if (options & SLURP) {
@@ -224,8 +257,7 @@ int main(int argc, char* argv[]) {
        slurped = jv_array();
      }
    }
-    struct jv_parser parser;
-    jv_parser_init(&parser);
+    struct jv_parser* parser = jv_parser_new();
    char buf[4096];
    while (read_more(buf, sizeof(buf))) {
      if (options & RAW_INPUT) {
@@ -235,35 +267,39 @@ int main(int argc, char* argv[]) {
            slurped = jv_string_concat(slurped, jv_string(buf));
          } else {
            if (buf[len-1] == '\n') buf[len-1] = 0;
-            process(jv_string(buf));
+            process(jq, jv_string(buf), jq_flags);
          }
        }
      } else {
-        jv_parser_set_buf(&parser, buf, strlen(buf), !feof(stdin));
+        jv_parser_set_buf(parser, buf, strlen(buf), !feof(stdin));
        jv value;
-        while (jv_is_valid((value = jv_parser_next(&parser)))) {
+        while (jv_is_valid((value = jv_parser_next(parser)))) {
          if (options & SLURP) {
            slurped = jv_array_append(slurped, value);
          } else {
-            process(value);
+            process(jq, value, jq_flags);
          }
        }
        if (jv_invalid_has_msg(jv_copy(value))) {
          jv msg = jv_invalid_get_msg(value);
          fprintf(stderr, "parse error: %s\n", jv_string_value(msg));
          jv_free(msg);
+          ret = 1;
          break;
        } else {
          jv_free(value);
        }
      }
    }
-    jv_parser_free(&parser);
+    jv_parser_free(parser);
+    if (ret != 0)
+      goto out;
    if (options & SLURP) {
-      process(slurped);
+      process(jq, slurped, jq_flags);
    }
  }
+out:
  jv_mem_free(input_filenames);
-  bytecode_free(bc);
-  return 0;
+  jq_teardown(&jq);
+  return ret;
 }
@@ -1,31 +0,0 @@
-#include "opcode.h"
-
-// flags, length
-#define NONE 0, 1
-#define CONSTANT OP_HAS_CONSTANT, 2
-#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
-#define BRANCH OP_HAS_BRANCH, 2
-#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
-#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
-#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
-#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
-
-#define OP(name, imm, in, out) \
-  {name, #name, imm, in, out},
-
-static const struct opcode_description opcode_descriptions[] = {
-#include "opcode_list.h"
-};
-
-static const struct opcode_description invalid_opcode_description = {
-  -1, "#INVALID", 0, 0, 0, 0
-};
-
-
-const struct opcode_description* opcode_describe(opcode op) {
-  if ((int)op >= 0 && (int)op < NUM_OPCODES) {
-    return &opcode_descriptions[op];
-  } else {
-    return &invalid_opcode_description;
-  }
-}
@@ -1,41 +0,0 @@
-#ifndef OPCODE_H
-#define OPCODE_H
-#include <assert.h>
-
-typedef enum {
-#define OP(name, imm, in, out) name,
-#include "opcode_list.h"
-#undef OP
-} opcode;
-
-enum {
-  NUM_OPCODES = 
-#define OP(name, imm, in, out) +1
-#include "opcode_list.h"
-#undef OP
-};
-
-enum {
-  OP_HAS_CONSTANT = 2,
-  OP_HAS_VARIABLE = 4,
-  OP_HAS_BRANCH = 8,
-  OP_HAS_CFUNC = 32,
-  OP_HAS_UFUNC = 64,
-  OP_IS_CALL_PSEUDO = 128,
-  OP_HAS_BINDING = 1024,
-};
-struct opcode_description {
-  opcode op;
-  const char* name;
-
-  int flags;
-
-  // length in 16-bit units
-  int length;
-
-  int stack_in, stack_out;
-};
-
-const struct opcode_description* opcode_describe(opcode op);
-
-#endif
@@ -1,21 +1,25 @@
 OP(LOADK, CONSTANT, 1, 1)
 OP(DUP,   NONE,     1, 2)
 OP(DUP2,  NONE,     2, 3)
-OP(SWAP,  NONE,     2, 2)
 OP(POP,   NONE,     1, 0)
 OP(LOADV, VARIABLE, 1, 1)
+OP(LOADVN, VARIABLE, 1, 1)
 OP(STOREV, VARIABLE, 1, 0)
 OP(INDEX, NONE,     2, 1)
-OP(YIELD, NONE, 1, 0)
 OP(EACH,  NONE,     1, 1)
 OP(FORK,  BRANCH,   0, 0)
 OP(JUMP,  BRANCH,   0, 0)
 OP(JUMP_F,BRANCH,   1, 0)
 OP(BACKTRACK, NONE, 0, 0)
-OP(APPEND, NONE,    2, 1)
+OP(APPEND, VARIABLE,1, 0)
 OP(INSERT, NONE,    4, 2)
+OP(RANGE, VARIABLE, 1, 1)

-OP(ASSIGN, VARIABLE, 3, 0)
+OP(SUBEXP_BEGIN,  NONE,     1, 2)
+OP(SUBEXP_END,    NONE,     2, 2)
+
+OP(PATH_BEGIN, NONE, 1, 2)
+OP(PATH_END,   NONE, 2, 1)

 OP(CALL_BUILTIN, CFUNC, -1, 1)

@@ -1,16 +1,16 @@
 %{
 #include <stdio.h>
 #include <string.h>
+#include <assert.h>
 #include "compile.h"
 #include "jv_alloc.h"
 #define YYMALLOC jv_mem_alloc
 #define YYFREE jv_mem_free
-
-struct lexer_param;
-
 %}
 %code requires {
 #include "locfile.h"
+struct lexer_param;
+
 #define YYLTYPE location
 #define YYLLOC_DEFAULT(Loc, Rhs, N)             \
  do {                                          \
@@ -47,7 +47,9 @@ struct lexer_param;

 %token INVALID_CHARACTER
 %token <literal> IDENT
+%token <literal> FIELD
 %token <literal> LITERAL
+%token <literal> FORMAT
 %token EQ "=="
 %token NEQ "!="
 %token DEFINEDOR "//"
@@ -57,6 +59,7 @@ struct lexer_param;
 %token THEN "then"
 %token ELSE "else"
 %token ELSE_IF "elif"
+%token REDUCE "reduce"
 %token END "end"
 %token AND "and"
 %token OR "or"
@@ -90,7 +93,7 @@ struct lexer_param;

 %type <blk> Exp Term MkDict MkDictPair ExpD ElseBody QQString FuncDef FuncDefs String
 %{
-#include "lexer.gen.h"
+#include "lexer.h"
 struct lexer_param {
  yyscan_t lexer;
 };
@@ -110,25 +113,19 @@ void yyerror(YYLTYPE* loc, block* answer, int* errors,
 int yylex(YYSTYPE* yylval, YYLTYPE* yylloc, block* answer, int* errors, 
          struct locfile* locations, struct lexer_param* lexer_param_ptr) {
  yyscan_t lexer = lexer_param_ptr->lexer;
-  while (1) {
-    int tok = jq_yylex(yylval, yylloc, lexer);
-    if (tok == INVALID_CHARACTER) {
-      FAIL(*yylloc, "Invalid character");
+  int tok = jq_yylex(yylval, yylloc, lexer);
+  if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
+    jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
+    if (jv_get_kind(msg) == JV_KIND_STRING) {
+      FAIL(*yylloc, jv_string_value(msg));
    } else {
-      if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
-        jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
-        if (jv_get_kind(msg) == JV_KIND_STRING) {
-          FAIL(*yylloc, jv_string_value(msg));
-        } else {
-          FAIL(*yylloc, "Invalid literal");
-        }
-        jv_free(msg);
-        jv_free(yylval->literal);
-        yylval->literal = jv_null();
-      }
-      return tok;
+      FAIL(*yylloc, "Invalid literal");
    }
+    jv_free(msg);
+    jv_free(yylval->literal);
+    yylval->literal = jv_null();
  }
+  return tok;
 }

 static block gen_dictpair(block k, block v) {
@@ -139,6 +136,17 @@ static block gen_index(block obj, block key) {
  return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX));
 }

+static block gen_slice_index(block obj, block start, block end) {
+  block key = BLOCK(gen_subexp(gen_const(jv_object())),
+                    gen_subexp(gen_const(jv_string("start"))),
+                    gen_subexp(start),
+                    gen_op_simple(INSERT),
+                    gen_subexp(gen_const(jv_string("end"))),
+                    gen_subexp(end),
+                    gen_op_simple(INSERT));
+  return BLOCK(key, obj, gen_op_simple(INDEX));
+}
+
 static block gen_binop(block a, block b, int op) {
  const char* funcname = 0;
  switch (op) {
@@ -158,15 +166,28 @@ static block gen_binop(block a, block b, int op) {
  return gen_call(funcname, BLOCK(gen_lambda(a), gen_lambda(b)));
 }

-static block gen_format(block a) {
-  return BLOCK(a, gen_call("tostring", gen_noop()));
+static block gen_format(block a, jv fmt) {
+  return BLOCK(a, gen_call("format", BLOCK(gen_lambda(gen_const(fmt)))));
+}
+
+static block gen_definedor_assign(block object, block val) {
+  block tmp = gen_op_var_fresh(STOREV, "tmp");
+  return BLOCK(gen_op_simple(DUP),
+               val, tmp,
+               gen_call("_modify", BLOCK(gen_lambda(object),
+                                         gen_lambda(gen_definedor(gen_noop(), 
+                                                                  gen_op_bound(LOADV, tmp))))));
 }
 
-static block gen_update(block a, block op, int optype) {
-  if (optype) {
-    op = gen_binop(gen_noop(), op, optype);
-  }
-  return gen_assign(BLOCK(a, gen_op_simple(DUP), op));
+static block gen_update(block object, block val, int optype) {
+  block tmp = gen_op_var_fresh(STOREV, "tmp");
+  return BLOCK(gen_op_simple(DUP),
+               val,
+               tmp,
+               gen_call("_modify", BLOCK(gen_lambda(object), 
+                                         gen_lambda(gen_binop(gen_noop(),
+                                                              gen_op_bound(LOADV, tmp),
+                                                              optype)))));
 }

 %}
@@ -194,22 +215,25 @@ FuncDef Exp %prec ';' {
 } |

 Term "as" '$' IDENT '|' Exp {
-  $$ = BLOCK(gen_op_simple(DUP), $1, 
-             block_bind(gen_op_var_unbound(STOREV, jv_string_value($4)), 
-                        $6, OP_HAS_VARIABLE));
+  $$ = gen_var_binding($1, jv_string_value($4), $6);
  jv_free($4);
 } |

+"reduce" Term "as" '$' IDENT '(' Exp ';' Exp ')' {
+  $$ = gen_reduce(jv_string_value($5), $2, $7, $9);
+  jv_free($5);
+} |
+
 "if" Exp "then" Exp ElseBody {
  $$ = gen_cond($2, $4, $5);
 } |
-"if" Exp error {
+"if" Exp "then" error {
  FAIL(@$, "Possibly unterminated 'if' statment");
  $$ = $2;
 } |

 Exp '=' Exp {
-  $$ = gen_assign(BLOCK(gen_op_simple(DUP), $3, gen_op_simple(SWAP), $1, gen_op_simple(SWAP)));
+  $$ = gen_call("_assign", BLOCK(gen_lambda($1), gen_lambda($3)));
 } |

 Exp "or" Exp {
@@ -225,11 +249,11 @@ Exp "//" Exp {
 } |

 Exp "//=" Exp {
-  $$ = gen_update($1, gen_definedor(gen_noop(), $3), 0);
+  $$ = gen_definedor_assign($1, $3);
 } |

 Exp "|=" Exp {
-  $$ = gen_update($1, $3, 0);
+  $$ = gen_call("_modify", BLOCK(gen_lambda($1), gen_lambda($3)));
 } |

 Exp '|' Exp { 
@@ -248,6 +272,10 @@ Exp "+=" Exp {
  $$ = gen_update($1, $3, '+');
 } |

+'-' Exp {
+  $$ = BLOCK($2, gen_call("_negate", gen_noop()));
+} |
+
 Exp '-' Exp {
  $$ = gen_binop($1, $3, '-');
 } |
@@ -308,18 +336,35 @@ FuncDef:

 "def" IDENT '(' IDENT ')' ':' Exp ';' {
  $$ = gen_function(jv_string_value($2), 
-                    gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), 
+                    gen_param(jv_string_value($4)), 
                    $7);
  jv_free($2);
  jv_free($4);
+} |
+
+"def" IDENT '(' IDENT ';' IDENT ')' ':' Exp ';' {
+  $$ = gen_function(jv_string_value($2), 
+                    BLOCK(gen_param(jv_string_value($4)), 
+                          gen_param(jv_string_value($6))),
+                    $9);
+  jv_free($2);
+  jv_free($4);
+  jv_free($6);
 }


+
 String:
-QQSTRING_START QQString QQSTRING_END {
-  $$ = $2;
+QQSTRING_START { $<literal>$ = jv_string("text"); } QQString QQSTRING_END {
+  $$ = $3;
+  jv_free($<literal>2);
+} |
+FORMAT QQSTRING_START { $<literal>$ = $1; } QQString QQSTRING_END {
+  $$ = $4;
+  jv_free($<literal>3);
 }

+
 QQString:
 /* empty */ {
  $$ = gen_const(jv_string(""));
@@ -328,7 +373,7 @@ QQString QQSTRING_TEXT {
  $$ = gen_binop($1, gen_const($2), '+');
 } |
 QQString QQSTRING_INTERP_START Exp QQSTRING_INTERP_END {
-  $$ = gen_binop($1, gen_format($3), '+');
+  $$ = gen_binop($1, gen_format($3, jv_copy($<literal>0)), '+');
 }


@@ -354,12 +399,27 @@ Term:
 '.' {
  $$ = gen_noop(); 
 } |
-Term '.' IDENT {
-  $$ = gen_index($1, gen_const($3)); 
+Term FIELD {
+  $$ = gen_index($1, gen_const($2));
 } |
-'.' IDENT { 
-  $$ = gen_index(gen_noop(), gen_const($2)); 
+FIELD { 
+  $$ = gen_index(gen_noop(), gen_const($1)); 
 } |
+Term '.' String {
+  $$ = gen_index($1, $3);
+} |
+'.' String {
+  $$ = gen_index(gen_noop(), $2);
+} |
+'.' error {
+  FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
+  $$ = gen_noop();
+} |
+'.' IDENT error {
+  jv_free($2);
+  FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
+  $$ = gen_noop();
+} | 
 /* FIXME: string literals */
 Term '[' Exp ']' {
  $$ = gen_index($1, $3); 
@@ -367,12 +427,24 @@ Term '[' Exp ']' {
 Term '[' ']' {
  $$ = block_join($1, gen_op_simple(EACH)); 
 } |
+Term '[' Exp ':' Exp ']' {
+  $$ = gen_slice_index($1, $3, $5);
+} |
+Term '[' Exp ':' ']' {
+  $$ = gen_slice_index($1, $3, gen_const(jv_null()));
+} |
+Term '[' ':' Exp ']' {
+  $$ = gen_slice_index($1, gen_const(jv_null()), $4);
+} |
 LITERAL {
  $$ = gen_const($1); 
 } |
 String {
  $$ = $1;
 } |
+FORMAT {
+  $$ = gen_format(gen_noop(), $1);
+} |
 '(' Exp ')' { 
  $$ = $2; 
 } | 
@@ -386,7 +458,7 @@ String {
  $$ = BLOCK(gen_subexp(gen_const(jv_object())), $2, gen_op_simple(POP));
 } |
 '$' IDENT {
-  $$ = gen_location(@$, gen_op_var_unbound(LOADV, jv_string_value($2)));
+  $$ = gen_location(@$, gen_op_unbound(LOADV, jv_string_value($2)));
  jv_free($2);
 } | 
 IDENT {
@@ -398,6 +470,11 @@ IDENT '(' Exp ')' {
  $$ = gen_location(@1, $$);
  jv_free($1);
 } |
+IDENT '(' Exp ';' Exp ')' {
+  $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5)));
+  $$ = gen_location(@1, $$);
+  jv_free($1);
+} |
 '(' error ')' { $$ = gen_noop(); } |
 '[' error ']' { $$ = gen_noop(); } |
 Term '[' error ']' { $$ = $1; } |
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# This script is used to cross-compile binaries for 
+# platforms other than the current one
+
+# Usage: $0 <platformname> <configure options>
+# <platformname> is arbitrary, it is the name
+# of the directory which will be created to contain
+# the output binaries.
+
+# e.g. $0 win32 --host=i686-w64-mingw32
+
+set -e
+cd `dirname "$0"`/../build
+
+plat="$1"
+[ -z "$plat" ] && exit 1
+shift
+
+[ -d "$plat" ] || mkdir "$plat"
+rm -rf "$plat/tmp"
+mkdir "$plat/tmp"
+cd "$plat/tmp"
+mkdir install_other
+../../../configure \
+    --prefix="`pwd`/install_other" \
+    --bindir="`pwd`/.." \
+    "$@"
+make install-binaries
+cd ..
+rm -rf tmp
@@ -1,5 +1,7 @@
 #!/usr/bin/python

+# This program was used to generate jv_utf8_tables.gen.h
+
 mask = lambda n: (1 << n) - 1


@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+cd `dirname "$0"`
+git rev-parse --verify -q jq-1.0 > /dev/null 2>&1
+git describe --always --match 'jq-*' --dirty | sed 's/^jq-//'
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# RUN ME after pulling the code from git!
+#
+# This script is handy after pulling the code out of git without having
+# to remember all the typing needed to build up and tear down all the
+# crap that autotools puts in your source repo.  The intent is to keep
+# all generated files out of git.
+
+if [ "clean" == "$1" ]; then
+  make distclean || echo "continuing..."
+  rm -rf tmp autom4te.cache config
+  rm -f INSTALL Makefile.in aclocal.m4 configure config.h.in ChangeLog
+  rm -f jv_utf8_tables.gen.h lexer.c lexer.h parser.c parser.h
+elif [ "superclean" == "$1" ]; then
+  # if autoconf errors during distcheck, it leaves files that need chmod'ing
+  ver=$(tr -d '\n' <VERSION)
+  if [ "x${ver}" != "x" ]; then
+    if [ -d jq-${ver} ]; then
+      chmod -R u+w jq-${ver}
+      rm -rf jq-${ver}
+    fi
+    rm -f jq-${ver}*
+  fi
+else
+  autoreconf --install
+  ./configure --prefix=/opt/junk
+  make check
+  [ -d tmp ] && mv tmp tmp-
+  mkdir tmp
+  make DESTDIR=./tmp install
+  make distcheck
+fi
@@ -21,6 +21,11 @@ null
 null
 1

+
+-1
+null
+-1
+
 # FIXME: much more number testing needed

 {}
@@ -52,6 +57,24 @@ null
 null
 "interpolation"

+@text,@json,([1,.] | @csv),@html,@uri,@sh,@base64
+"<>&'\""
+"<>&'\""
+"\"<>&'\\\"\""
+"1,\"<>&'\"\"\""
+"&lt;&gt;&amp;&apos;&quot;"
+"%3c%3e%26'%22"
+"'<>&'\\''\"'"
+"PD4mJyI="
+
+@uri
+"\u03bc"
+"%ce%bc"
+
+@html "<b>\(.)</b>"
+"<script>hax</script>"
+"<b>&lt;script&gt;hax&lt;/script&gt;</b>"
+
 #
 # Dictionary construction syntax
 #
@@ -92,6 +115,10 @@ null
 {"foo": {"bar": 42}, "bar": "badvalue"}
 42

+."foo"."bar"
+{"foo": {"bar": 20}}
+20
+

 #
 # Multiple outputs, iteration
@@ -143,6 +170,25 @@ null
 2
 3

+#
+# Slices
+#
+
+[.[3:2], .[-5:4], .[:-2], .[-2:], .[3:3][1:], .[10:]]
+[0,1,2,3,4,5,6]
+[[], [2,3], [0,1,2,3,4], [5,6], [], []]
+
+del(.[2:4],.[0],.[-2:])
+[0,1,2,3,4,5,6,7]
+[1,4,5]
+
+.[2:4] = ([], ["a","b"], ["a","b","c"])
+[0,1,2,3,4,5,6,7]
+[0,1,4,5,6,7]
+[0,1,"a","b",4,5,6,7]
+[0,1,"a","b","c",4,5,6,7]
+
+
 #
 # Variables
 #
@@ -179,6 +225,14 @@ null
 "wtasdf"
 2.0

+2-1
+null
+1
+
+2-(-1)
+null
+3
+
 1e+0+0.001e3
 "I wonder what this will be?"
 20e-1
@@ -187,6 +241,18 @@ null
 15
 19.0

+.+null
+{"a":42}
+{"a":42}
+
+null+.
+null
+null
+
+.a+.b
+{"a":42}
+42
+
 [1,2,3] + [.]
 null
 [1,2,3,null]
@@ -240,8 +306,8 @@ null
 [false, false, false, false, false, false, false, false, true ]

 [.[] | length]
-[[], {}, [1,2], {"a":42}, "asdf"]
-[0, 0, 2, 1, 4]
+[[], {}, [1,2], {"a":42}, "asdf", "\u03bc"]
+[0, 0, 2, 1, 4, 1]

 map(keys)
 [{}, {"abcd":1,"abc":2,"abcde":3}, {"x":1, "z": 3, "y":2}]
@@ -317,6 +383,60 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
 # []
 # 1001

+reduce .[] as $x (0; . + $x)
+[1,2,4]
+7
+
+#
+# Paths
+#
+
+path(.foo[0,1])
+null
+["foo", 0]
+["foo", 1]
+
+path(.[] | select(.>3))
+[1,5,3]
+[1]
+
+path(.)
+42
+[]
+
+["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
+{"bar": 42, "foo": ["a", "b", "c", "d"]}
+"b"
+{"bar": 42, "foo": ["a", 20, "c", "d"]}
+{"bar": 42, "foo": ["a", "c", "d"]}
+
+map(getpath([2])), map(setpath([2]; 42)), map(delpaths([[2]]))
+[[0], [0,1], [0,1,2]]
+[null, null, 2]
+[[0,null,42], [0,1,42], [0,1,42]]
+[[0], [0,1], [0,1]]
+
+map(delpaths([[0,"foo"]]))
+[[{"foo":2, "x":1}], [{"bar":2}]]
+[[{"x":1}], [{"bar":2}]]
+
+["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
+{"bar":false}
+null
+{"bar":false, "foo": [null, 20]}
+{"bar":false}
+
+delpaths([[-200]])
+[1,2,3]
+[1,2,3]
+
+del(.), del(empty), del((.foo,.bar,.baz) | .[2,3,0]), del(.foo[0], .bar[0], .foo, .baz.bar[0].x)
+{"foo": [0,1,2,3,4], "bar": [0,1]}
+null
+{"foo": [0,1,2,3,4], "bar": [0,1]}
+{"foo": [1,4], "bar": [1]}
+{"bar": [1]}
+
 #
 # Assignment
 #
@@ -339,6 +459,10 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
 [-1,1,3]
 [0.5, 1.5, 2.5]

+.foo += .foo
+{"foo":2}
+{"foo":4}
+
 .[0].a |= {"old":., "new":(.+1)}
 [{"a":1,"b":2}]
 [{"a":{"old":1, "new":2},"b":2}]
@@ -374,10 +498,9 @@ def inc(x): x |= .+1; inc(.[].a)
 [{"foo":[1,2], "bar": 42}, {"foo":[1], "bar": null}, {"foo":[null,false,3], "bar": 18}, {"foo":[], "bar":42}, {"foo": [null,false,null], "bar": 41}]
 [[1,2], [1], [3], [42], [41]]

-# FIXME: behaviour of update operators
-# .[] //= .[0]
-# ["hello",true,false,[false],null]
-# ["hello",true,"hello",[false],"hello"]
+.[] //= .[0]
+["hello",true,false,[false],null]
+["hello",true,"hello",[false],"hello"]

 .[] | [.[0] and .[1], .[0] or .[1]]
 [[true,[]], [false,1], [42,null], [null,false]]
@@ -473,3 +596,11 @@ unique
 .foo[.baz]
 {"foo":{"bar":4},"baz":"bar"}
 4
+
+.[] | .error = "no, it's OK"
+[{"error":true}]
+{"error": "no, it's OK"}
+
+[{a:1}] | .[] | .a=999
+null
+{"a": 999}
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cat $@ | valgrind --error-exitcode=1 -q --leak-check=full ./jq --run-tests
Author	SHA1	Message	Date
Stephen Dolan	37cfc912c1	Remove #includes from jv.h	2013-06-23 14:23:07 +01:00
Stephen Dolan	8c2e228c74	Fix the jv_parser interface.	2013-06-23 12:26:49 +01:00
Stephen Dolan	8630fb02bb	Use libtool's built-in symbol exporting rather than a mapfile.	2013-06-23 12:10:58 +01:00
Stephen Dolan	5052ab4e2b	Move gen_utf8_tables to scripts	2013-06-23 12:08:22 +01:00
Stephen Dolan	a44f739356	Move libtool m4 junk to config/ and delete some autogenerated files.	2013-06-23 12:01:58 +01:00
Stephen Dolan	f44ecf3872	Remove Autoconf-generated config.h.	2013-06-23 12:01:58 +01:00
Stephen Dolan	1c9e03f800	Merge branch 'header-cleanup' into libjq Conflicts: Makefile.am	2013-06-22 23:27:16 +01:00
Stephen Dolan	5d9ec83805	Build libjq only once, and link it statically to ./jq This means ./jq is a real binary rather than a libtool turd.	2013-06-22 20:53:28 +01:00
Stephen Dolan	3a295c659b	Fix distcheck. Update list of files to be distributed.	2013-06-22 20:53:03 +01:00
Nicolas Williams	e8643787cf	Update .gitignore	2013-06-21 12:41:57 -05:00
Nicolas Williams	298b2a6033	Add libjq autoconf goo	2013-06-21 11:57:12 -05:00
Nicolas Williams	c791428905	Quiet setup.sh re: tmp dir	2013-06-21 11:57:11 -05:00
Stephen Dolan	dc2fe63e42	Merge pull request #157 from nicowilliams/argfile-fixes Argfile fixes	2013-06-21 09:40:32 -07:00
Stephen Dolan	7af88962ee	Move cfunction invocation code to the interpreter loop.	2013-06-21 12:06:28 +01:00
Nicolas Williams	3216d7683a	Fix serious bug in handling of --argfile	2013-06-18 19:05:52 -05:00
Nicolas Williams	e4094c8664	Fix leaks in jv_load_file()	2013-06-18 19:05:52 -05:00
Stephen Dolan	b49d65a276	Fold opcode.{c,h} into bytecode.{c,h}	2013-06-18 01:36:18 +01:00
Stephen Dolan	824ce76cec	Simplify block functions for variables	2013-06-18 01:07:18 +01:00
Stephen Dolan	2a18f97d12	Saner build instructions in README.md Closes #144	2013-06-18 00:29:05 +01:00
Stephen Dolan	8c4d29ee38	Remove some initialise-to-zero code. This lets valgrind find more bugs - if a field isn't given a well-defined value valgrind will now find it instead of seeing it set to zero with memset.	2013-06-18 00:17:20 +01:00
Stephen Dolan	2c13f4fe72	Merge branch 'stack-refactor-merge'	2013-06-18 00:13:44 +01:00
Stephen Dolan	94931040a8	Merge branch 'stack-refactor' Conflicts: execute.c	2013-06-18 00:06:00 +01:00
Stephen Dolan	b78a4cdc93	Merge pull request #154 from nicowilliams/no-fopen-re Remove accidentally introduced use of fopen "e"	2013-06-17 15:34:01 -07:00
Nicolas Williams	da09544bb0	Remove accidentally introduced use of fopen "e"	2013-06-17 17:27:25 -05:00
Stephen Dolan	ce17b6bd53	Merge pull request #114 from nicowilliams/nomem_handler Add jv_nomem_handler()	2013-06-16 06:20:27 -07:00
Nicolas Williams	fe03bea191	Remove last remnant of main.h	2013-06-16 08:06:52 -05:00
Nicolas Williams	0ed6347d47	Allow --run-tests to take a file argument	2013-06-15 17:37:15 -05:00
Nicolas Williams	3f86e97f70	Fixup API to get closer to a libjq	2013-06-15 17:37:15 -05:00
Stephen Dolan	69fa279e3c	Merge pull request #150 from nicowilliams/jv_load_file Move slurp_file() into library as jv_load_file()	2013-06-15 05:29:17 -07:00
Nicolas Williams	bbf4e05fde	Move slurp_file() into library as jv_load_file() Needed as part of creating a libjq.	2013-06-15 00:08:59 -05:00
Stephen Dolan	05d90517b0	Clean up lots of stack and frame logic. Move frame defs to execute.c	2013-06-14 22:08:18 +01:00
Stephen Dolan	5f5c1dc5a6	Simplify frame logic.	2013-06-14 01:20:24 +01:00
Stephen Dolan	e0524644f8	Unify all stacks. Passes tests, but needs cleanup.	2013-06-13 21:50:32 +01:00
Stephen Dolan	81e2336ace	Support ."foo" syntax for accessing fields. See #141 .	2013-06-11 22:26:39 +01:00
Stephen Dolan	25bf1a0169	Unify frame and data stacks	2013-06-10 01:17:58 +01:00
Stephen Dolan	c8ab67be27	Speed up cached configure (./configure -C)	2013-06-05 16:40:01 +01:00
Stephen Dolan	6c34288ebf	Clean up flex lines in build	2013-06-05 13:43:23 +01:00
Stephen Dolan	2ffbb9d6b7	Lex and parse .foo better. '.as' is now valid, '. foo' is now invalid. See #141.	2013-06-05 13:41:56 +01:00
Stephen Dolan	dad608d5c0	Merge pull request #142 from lanthaler/master Update README.md	2013-06-04 04:13:24 -07:00
Markus Lanthaler	04b939625b	Update README.md Update the link to the documentation. All GitHub pages are now using the github.io domain.	2013-06-04 13:04:42 +02:00
Stephen Dolan	5dac485458	Make jq --version print to stdout, not stderr	2013-06-03 23:22:59 +01:00
Stephen Dolan	2c4ccd122e	Better error handling for .foo case in parser. See #141 .	2013-06-03 23:12:31 +01:00
Stephen Dolan	b90ae078b8	Let the parser rather than the lexer handle invalid characters.	2013-06-03 23:12:31 +01:00
Stephen Dolan	cc8761dbde	Add command-line option to sort object keys. Closes #79.	2013-06-03 23:12:31 +01:00
Stephen Dolan	08cc591884	Clean up Makefile.am (distcheck, rebuild version.h less often)	2013-06-03 23:12:31 +01:00
Stephen Dolan	1bc4d04f43	Merge pull request #139 from 13ren/stopwarning Stop warning on fgets, jq_test.c:42	2013-06-01 07:30:15 -07:00
Brendan Macmillan	d4018192a9	Stop warning on fgets, simple version	2013-06-01 11:12:28 +10:00
Brendan Macmillan	19ddbfeca9	Stop warning on fgets, complex version	2013-06-01 11:03:15 +10:00
Stephen Dolan	4fed6763fc	Squash a warning on some GCC versions	2013-05-31 12:15:58 +01:00
Stephen Dolan	73446a9cce	Support for printing object keys in sorted order. No command-line option to enable this yet. See #79.	2013-05-29 11:47:12 +01:00
Stephen Dolan	dcf1ac0d1f	Merge pull request #135 from 13ren/locfile_bugfix Locfile bugfix	2013-05-29 03:38:15 -07:00
Stephen Dolan	364c58d13f	Merge pull request #130 from 13ren/libload_2 libload_2 (update to pull request #126, for issue #112)	2013-05-29 03:37:36 -07:00
Brendan Macmillan	8cfe11e4e2	Bugfix multiline off-by-one (locfile.c)	2013-05-29 15:23:08 +10:00
Brendan Macmillan	40378a18f7	locfile.h -> locfile.h + locfile.c clean up includes of a few files	2013-05-29 15:17:01 +10:00
Brendan Macmillan	7fb5d8c973	Hack bugfix for multiline off-by-one (locfile.c)	2013-05-29 15:16:45 +10:00
Brendan Macmillan	6e373942e5	Load library from ~/.jq	2013-05-29 15:05:40 +10:00
Stephen Dolan	f0b982795c	Make jq --version report an actual git revision. Closes #129.	2013-05-24 23:35:38 +01:00
Stephen Dolan	cbd422d56e	Merge pull request #133 from nicowilliams/option_argfile-rebase0 Add --argfile variant of --arg (issue #117)	2013-05-24 04:13:45 -07:00
Nicolas Williams	03598bb5d6	Add --argfile variant of --arg (issue #117 ) This is useful when one has a database (in JSON form) to query using jq input data. % echo '{"a":1, "c":5}' > db.json % echo '"c"'\|./jq --argfile f /tmp/a '$f[.]' 5 % echo '"a"'\|./jq --argfile f /tmp/a '$f[.]' 1 % echo '"b"'\|./jq --argfile f /tmp/a '$f[.]' null %	2013-05-23 13:52:06 -05:00
Stephen Dolan	983a53a643	'make clean' won't delete jq.1 if it can't be rebuilt. See #131	2013-05-23 19:20:19 +01:00
Stephen Dolan	0535e724a2	jq 1.3 release	2013-05-19 11:48:18 +01:00
Stephen Dolan	fd1ac5dd79	EACH need not make a backtrack point on the last iteration	2013-05-17 22:49:08 +01:00
Stephen Dolan	9b91eafac4	Make array construction use LOADVN (slightly faster)	2013-05-16 15:09:59 +01:00
Stephen Dolan	bc42812715	Remove the insane "fold" operation, replace with saner "reduce".	2013-05-16 15:07:53 +01:00
Stephen Dolan	8fbee891b3	Add LOADVN opcode. Does a variable load, but sets the variable to be null afterwards.	2013-05-16 15:02:18 +01:00
Stephen Dolan	919292764a	Reverse function. Closes #94 .	2013-05-16 14:25:28 +01:00
Stephen Dolan	a1e4dfa324	Better debug info for struct bytecode. --debug-dump-disasm produces more comprehensible output now.	2013-05-16 00:51:17 +01:00
Stephen Dolan	f8644c31a3	Only generate code for those builtin functions actually used. Makes output of --debug-dump-disasm much simpler.	2013-05-15 01:37:15 +01:00
Stephen Dolan	1e2851cdb3	Remove the YIELD opcode (use RET instead)	2013-05-15 01:23:06 +01:00
Stephen Dolan	e83e51eb56	'length' function now measures string length in codepoints, not bytes.	2013-05-15 00:37:38 +01:00
Stephen Dolan	c496a924ce	Bugfix for array slices.	2013-05-15 00:22:44 +01:00
Stephen Dolan	81be37b236	Add the range function	2013-05-14 16:09:10 +01:00
Stephen Dolan	4d540b7102	Fix some documentation bugs	2013-05-13 20:26:44 +01:00
Stephen Dolan	33901b74b1	Array slicing. Closes #2 .	2013-05-13 20:16:19 +01:00
Stephen Dolan	a47cfa4757	Delete unnecessary _POSIX_SOURCE. Closes #89	2013-05-13 16:41:11 +01:00
Stephen Dolan	a625d2821e	Add to_entries, from_entries and with_entries functions. Closes #97.	2013-05-13 16:15:49 +01:00
Stephen Dolan	d8706fd460	Fix the //= operator. Closes #62.	2013-05-13 16:04:30 +01:00
Stephen Dolan	e13f24ab45	Better error handling for INSERT opcode jq 'null \| {(.a): 1}' no longer crashes.	2013-05-13 15:42:57 +01:00
Stephen Dolan	34a63246ab	Remove the is_backtrack_frame special case hack.	2013-05-13 15:37:57 +01:00
Stephen Dolan	8c708f3c7a	Refactor path logic.	2013-05-13 15:00:05 +01:00
Stephen Dolan	b0e65d149f	Fix a bug triggered by jv_array_slice.	2013-05-13 14:10:39 +01:00
Stephen Dolan	e29db8d272	Fix bug with path handling (used in assignments). Closes #67	2013-05-11 18:10:21 +01:00
Stephen Dolan	7ca5127fcc	Merge branch 'autotools'	2013-05-11 15:21:23 +01:00
Stephen Dolan	4b1b9c219a	Delete some Autotools junk and update README. Autotools autogenerated files that change frequently have been removed, those that don't are still checked in.	2013-05-11 15:15:56 +01:00
Stephen Dolan	4a5405e4b8	Rake task to build website including source dist and binaries.	2013-05-11 14:57:58 +01:00
Stephen Dolan	d03149f4df	Rake task to build source distribution for website	2013-05-11 14:48:39 +01:00
Stephen Dolan	18cf4ad475	Add the docs to the distributed tarballs	2013-05-09 20:53:51 +01:00
Stephen Dolan	ba4bb136bb	Fix manpage build.	2013-05-09 20:43:26 +01:00
Stephen Dolan	e6494857fc	Add some build instructions for how to install doc dependencies. jq can now build without Ruby, but you won't get a nice manpage.	2013-05-09 20:35:19 +01:00
Stephen Dolan	12d2d4d1f5	Don't build the manpage unless there's a working Ruby setup.	2013-05-09 15:22:38 +01:00
Stephen Dolan	d75414bc94	support for out-of-tree tests	2013-05-09 13:40:00 +01:00
Stephen Dolan	a592c98461	Fail properly when attempting to build manpage without rake	2013-05-09 13:10:09 +01:00
Stephen Dolan	f6089a627a	Autotools hackery so that "make dist" builds lexer and parser.	2013-05-09 11:31:17 +01:00
Stephen Dolan	898dc8978b	Add a redundant intialisation to squash a gcc warning.	2013-05-09 11:30:49 +01:00
Stephen Dolan	d43084086b	Make manpage building more robust.	2013-05-09 02:38:57 +01:00
Stephen Dolan	124bda6993	Add a "binaries" target to the website Rakefile.	2013-05-09 02:18:28 +01:00
Stephen Dolan	7fbc50239c	Minor docs fix to get rid of Bonsai warnings.	2013-05-09 02:01:01 +01:00
Stephen Dolan	282667bbd4	Script for cross-compiling jq binaries for other platforms.	2013-05-09 01:50:20 +01:00
Stephen Dolan	63b689bf54	Use the version number from configure.ac in the website.	2013-05-08 19:57:29 +01:00
Stephen Dolan	c710f91257	Add the parser to BUILT_SOURCES	2013-05-08 19:57:05 +01:00
Stephen Dolan	a511d03d85	Configure whether colour is displayed by default directly in source	2013-05-08 19:37:12 +01:00
Stephen Dolan	9d245c88c4	Check in a pile of Autotools junk, including the configure script.	2013-05-08 18:55:10 +01:00
Stephen Dolan	7813f363a2	Merge pull request #115 from nicowilliams/complex Rename complex to nontrivial for safety (#113)	2013-05-08 07:07:31 -07:00
Stephen Dolan	d121420757	Clean up autotools config. Remove some of @stagrlee's hard work since jq's testsuite is less silly than when he wrote the autotools config.	2013-05-08 02:42:23 +01:00
Stephen Dolan	1ebb382540	Check in jv_utf8_tables.gen.h, it is unlikely to change.	2013-05-08 01:33:04 +01:00
Stephen Dolan	2363246eea	Merge remote-tracking branch 'stagrlee/master' into autotools Conflicts: Makefile jq_test.c	2013-05-08 00:52:24 +01:00
Nicolas Williams	fb3c124af1	Rename complex to nontrivial for safety (#113 )	2013-05-07 18:46:13 -05:00
Stephen Dolan	46814da91a	Merge pull request #119 from 13ren/master tiny typo fix in manual	2013-05-07 03:57:47 -07:00
Nicolas Williams	65deaaacf5	Fix #111 : exit(1) when inputs are not valid JSON	2013-05-06 12:36:53 -05:00
Stephen Dolan	74eb914aba	Add trailing slashes to various internal doc links. Fixes #110	2013-05-06 15:03:14 +01:00
Stephen Dolan	5be97463ec	Add a --arg option to allow variables to be passed from the cmdline. Closes #107	2013-05-06 14:21:00 +01:00
Stephen Dolan	48be23233a	Add the "has" function. Closes #74 .	2013-05-06 13:46:54 +01:00
Stephen Dolan	88a6dc5343	Merge pull request #77 from jkleint/patch-1 Clarify description of `as`	2013-05-05 18:39:59 -07:00
Stephen Dolan	fee38afd8b	Document behaviour of .[] on objects. Closes #57.	2013-05-06 02:37:34 +01:00
Stephen Dolan	cacda19e75	Add documentation for "type". Closes #75.	2013-05-06 02:34:02 +01:00
Stephen Dolan	7f979cf1c7	Update gems and use HTTPS to get them.	2013-05-06 01:15:33 +01:00
Stephen Dolan	8bf4a73f3d	Fix some bad escaping	2013-05-06 01:14:53 +01:00
Stephen Dolan	d04bd897e3	Merge commit 'e45f93f85f22c73025830a6bd0b2807263068996' See #87	2013-05-05 23:58:24 +01:00
Stephen Dolan	1741d8c161	Remove JQ_DEBUG #define and jq_test binary, simplifying build. The debugging features previously available via JQ_DEBUG are now command-line options.	2013-05-05 23:12:10 +01:00
Stephen Dolan	c1748fa633	WIP	2013-05-05 22:59:53 +01:00
Stephen Dolan	a49402c53a	Combine the functionality of jq and jq_test One binary is much simpler. ./jq --run-tests now runs the tests.	2013-05-05 22:59:53 +01:00
Stephen Dolan	47e015e946	Make a few constant globals actually const. This means the .data and .bss sections are empty, a good thing in a shared library (see #98).	2013-05-05 22:59:53 +01:00
Stephen Dolan	ab1a97472c	Merge pull request #109 from nicowilliams/no_mutable_globals-rebased Remove globals/make jq_init/next/teardown() an API	2013-05-05 12:29:07 -07:00
Stephen Dolan	74df082283	Merge pull request #93 from shayel/master Allow manpage to build	2013-05-05 12:13:36 -07:00
Nicolas Williams	a136c3b7c9	Build and install libjq	2013-04-28 20:12:49 -05:00
Nicolas Williams	312ab754a2	Remove unnecessary #includes	2013-04-28 20:12:15 -05:00
Nicolas Williams	87e9c64003	Remove globals/make jq_init/next/teardown() an API	2013-04-28 18:46:21 -05:00
Shay Elkin	054e3f3795	Add missing require to Rakefile Use bundle exec to make manpage Don't set executable bit on manpage	2013-03-24 14:45:10 +02:00
Simon Elsbrock	e45f93f85f	fix typo in manual	2013-02-20 22:38:50 +01:00
Lee Thompson	dc4d3d8cbe	get man pages working	2013-02-04 13:35:42 -06:00
Lee Thompson	1d3d2efc89	add man page to release	2013-02-04 07:01:18 -06:00
Lee Thompson	43fd939f16	rework build to not distribute tests unless --enable-devel is specificied to autoconf	2013-02-03 14:49:03 -06:00
Lee Thompson	d8a072ca01	fix email addresses	2013-02-03 00:35:56 -06:00
Lee Thompson	f62184855b	use AM_SILENT_RULES	2013-02-03 00:09:47 -06:00
Lee Thompson	0370d5b526	fix RPM build process	2013-02-02 23:50:43 -06:00
Lee Thompson	52f906ab43	fix jq_test so that it runs in the distcheck phase of autoconf	2013-02-02 23:05:00 -06:00
Lee Thompson	c7725a8d4d	merging upstream stedolan changes	2013-02-02 20:39:23 -06:00
Lee Thompson	dd70eeb29d	use built in macro to RPM	2013-01-29 19:13:22 -06:00
jkleint	115552f3a6	Clarify description of `as` It's more of a foreach loop than an identity operation. Addresses issue #72.	2013-01-25 12:17:56 -05:00
13ren	d0cd26cd8e	Update docs/content/3.manual/manual.yml There's a , after map ; should be . #65	2013-01-13 17:33:01 +11:00
Stephen Dolan	925ec3751f	Fix negative number syntax. Add a unary '-' operator. Closes #63.	2013-01-03 12:53:23 +00:00
Stephen Dolan	c013b557a2	Change APPEND opcode to directly modify a variable. Avoids a big O(n^2) loop in constructing arrays. Fixes #61.	2013-01-03 12:51:33 +00:00
Stephen Dolan	fb84541e11	Clean up jv_object_foreach and add jv_array_foreach	2012-12-31 23:27:00 +00:00
Stephen Dolan	38fe6fc0cb	Ignore some more stuff.	2012-12-31 23:25:12 +00:00
Stephen Dolan	b3bad59dd7	Add the `recurse` function. See #37 .	2012-12-29 22:59:07 +00:00
Stephen Dolan	e0cda536f3	Make null + foo return foo, rather than an error. This also allows 'add' to be implemented in jq rather than C.	2012-12-29 16:52:26 +00:00
Stephen Dolan	d5fdf70434	Refactor assignment. New version is much more elegant and probably slower. Uses library functions implemented in jq rather than dedicated opcodes.	2012-12-29 16:43:36 +00:00
Stephen Dolan	465a4ec565	Improvements to del(foo). del(foo,bar) is now very different from del(foo),del(bar). See #37.	2012-12-29 16:13:06 +00:00
Stephen Dolan	3a5377e183	Fix a bug in URI-encoding of certain non-ASCII characters. Oh how I hate C casting rules.	2012-12-29 01:37:22 +00:00
Stephen Dolan	79c9c418c6	Path manipulation (path/getpath/setpath/delpath) and docs. del function should fix #37.	2012-12-28 16:08:29 +00:00
Stephen Dolan	f1e23448e7	Support for two-argument functions. Should probably support n args instead. This is a quick hack. Also, ';' is a slightly ugly argument separator.	2012-12-28 15:07:27 +00:00
Stephen Dolan	417899f9a0	Fold operation (code/docs/test)	2012-12-28 15:05:34 +00:00
Stephen Dolan	9302b16247	Docs typos.	2012-12-28 14:23:28 +00:00
Stephen Dolan	89e26969ae	@foo syntax for encoding of strings into various formats. Fixes part of #47 and #48.	2012-12-28 01:23:09 +00:00
Stephen Dolan	52db8000c6	Fix an embarassing I/O bug. Pulling some I/O out to a function meant that buf changed from "char buf[4096]" to "char* buf", and "sizeof(buf)" got a lot less interesting. The upshot of this is that jq read input eight bytes at a time, which is not the fastest.	2012-12-27 01:57:09 +00:00
Stephen Dolan	d327772dfc	JSON parsing error messages now specify a location of the error. Should help with #53.	2012-12-27 01:56:23 +00:00
Stephen Dolan	cedda2084d	Sneaky valgrind trick to detect stack memory issues. After something is popped from a stack, we overwrite the memory with uninitialised data (if JQ_DEBUG is on). This means that valgrind reports use-after-pop as an uninitialised memory error.	2012-12-24 17:11:18 +00:00
Stephen Dolan	0fed03f35a	Fix a bad memory leak when using --raw-output	2012-12-23 15:41:20 +00:00
Stephen Dolan	5578ec0f69	Give Windows builds a proper .exe suffix.	2012-12-20 15:12:14 +00:00
Lee Thompson	a156fb19de	get testing working in autoconf	2012-12-04 19:50:10 -06:00
Lee Thompson	ca862a594c	kill prereq on RPM spec file	2012-12-04 19:00:59 -06:00
Lee Thompson	dfa9e61aeb	fixes jq_test core dump	2012-12-04 18:12:53 -06:00
Lee Thompson	99353d8c84	remove obsolete line	2012-12-04 16:50:11 -06:00
Lee Thompson	76c8706145	fix rpm mv command when rpm stashes the package in a subdirectory	2012-12-04 16:29:01 -06:00
Lee Thompson	277249ffe6	change rpmbuild target to rpm	2012-12-04 14:49:39 -06:00
Lee Thompson	00e4feba81	get rpm building from tarball with autoconf	2012-12-03 17:27:40 -06:00
Lee Thompson	26bbed29f7	start work on getting RPM to work with tarball	2012-12-03 16:19:45 -06:00
Lee Thompson	ddeec45b2e	revert back to make invoking flex, fix a few bugs	2012-11-28 01:08:23 -06:00
Lee Thompson	006357f954	initial attempt at autoconf implementation, removed all generated code from git	2012-11-27 16:02:59 -06:00
Anthony Shortland	a33c6f3df9	Fixed to work under CentOS 5.4	2012-10-30 14:30:36 -07:00
Anthony Shortland	04640aeaeb	Added initial support for packaging jq as an RPM	2012-10-30 10:26:08 -07:00