Delete some Autotools junk and update README.

Autotools autogenerated files that change frequently have been
removed, those that don't are still checked in.

.gitignore

@@ -1,15 +1,28 @@
 *.o
 *~
 
-# Autogenerated
-*.gen.*
-
-# Test binaries
-jv_test
-jv_parse
-parsertest*~
-
-build
+jq
+jq.1
 
 # Something delightfully recursive happens otherwise
-docs/content/2.download/source/*
+docs/content/2.download/source/*
+
+# Autogenerated
+lexer.c
+lexer.h
+parser.c
+parser.h
+
+# Autotools junk
+.deps
+*.log
+stamp-h1
+config.log
+config.status
+autom4te.cache
+config.h
+Makefile
+jq-*.tar.gz
+configure
+aclocal.m4
+Makefile.in

AUTHORS

@@ -0,0 +1,6 @@
+Created By:
+Stephen Dolan        <mu@netsoc.tcd.ie>
+
+Contributions by
+Anthony Shortland    <anthony.shortland@me.com>      - rpmbuild target
+Lee Thompson         <stagr.lee@gmail.com>           - autoconf stuff

Makefile

@@ -1,102 +0,0 @@
-CC=gcc
-CFLAGS=-Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function $(EXTRA_CFLAGS)
-
-prefix=/usr/local
-mandir=$(prefix)/share/man
-
-
-.PHONY: all clean releasedep tarball install uninstall test releasetag
-all: jq
-
-lexer.gen.c: lexer.l
-	flex -o lexer.gen.c --header-file=lexer.gen.h lexer.l
-lexer.gen.h: lexer.gen.c
-
-parser.gen.c: parser.y lexer.gen.h
-	bison -W -d parser.y -v --report-file=parser.gen.info -o $@
-parser.gen.h: parser.gen.c
-
-jv_utf8_tables.gen.h: gen_utf8_tables.py
-	python $^ > $@
-jv_unicode.c: jv_utf8_tables.gen.h
-
-version.gen.h: VERSION
-	sed 's/.*/#define JQ_VERSION "&"/' $^ > $@
-main.c: version.gen.h
-
-JQ_SRC=parser.gen.c lexer.gen.c opcode.c bytecode.c compile.c execute.c builtin.c jv.c jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_alloc.c
-
-jq_test: CFLAGS += -DJQ_DEBUG=1
-jq_test: $(JQ_SRC) jq_test.c
-	$(CC) $(CFLAGS) $(CFLAGS_DEBUG) -o $@ $^
-
-jq: CFLAGS += -O -DJQ_DEBUG=0
-jq: $(JQ_SRC) main.c
-	$(CC) $(CFLAGS) $(CFLAGS_OPT) -o $@ $^
-
-test: jq_test
-	valgrind --error-exitcode=1 -q --leak-check=full ./jq_test >/dev/null
-
-BINARIES=jq jq_test
-PLATFORMS=linux32 linux64 osx32 osx64 win32 win64
-
-build/linux32%: CC='x86_64-linux-gnu-gcc -m32'
-build/linux64%: CC='x86_64-linux-gnu-gcc -m64'
-
-# OS X cross compilers can be gotten from
-# https://launchpad.net/~flosoft/+archive/cross-apple
-build/osx32%:   CC='i686-apple-darwin10-gcc -m32'
-build/osx64%:   CC='i686-apple-darwin10-gcc -m64'
-
-# On Debian, you can get windows compilers in the
-# gcc-mingw-w64-i686 and gcc-mingw-w64-x86-64 packages.
-build/win32%:   CC='i686-w64-mingw32-gcc -m32'   EXTRA_CFLAGS=-DJQ_DEFAULT_ENABLE_COLOR=0
-build/win64%:   CC='x86_64-w64-mingw32-gcc -m64' EXTRA_CFLAGS=-DJQ_DEFAULT_ENABLE_COLOR=0
-
-ALL_BINARIES=$(foreach platform, $(PLATFORMS), $(foreach binary, $(BINARIES), build/$(platform)/$(binary)))
-
-$(ALL_BINARIES): build/%:
-	mkdir -p $(@D)
-	make -B $(BINARIES) CC=$(CC)
-	cp $(BINARIES) $(@D)
-
-binaries: $(ALL_BINARIES)
-
-clean:
-	rm -rf build
-	rm -f $(BINARIES) *.gen.*
-
-releasedep: lexer.gen.c parser.gen.c jv_utf8_tables.gen.h
-
-releasetag:
-	git tag -s "jq-$$(cat VERSION)" -m "jq release $$(cat VERSION)"
-
-docs/content/2.download/source/jq.tgz: jq
-	mkdir -p `dirname $@`
-	tar -czvf $@ `git ls-files; ls *.gen.*`
-
-tarball: docs/content/2.download/source/jq.tgz
-
-jq.1: docs/content/3.manual/manual.yml
-	( cd docs; rake manpage; ) > $@
-
-install: jq jq.1
-	install -d -m 0755 $(prefix)/bin
-	install -m 0755 jq $(prefix)/bin
-	install -d -m 0755 $(mandir)/man1
-	install -m 0755 jq.1 $(mandir)/man1
-
-uninstall:
-	rm -vf $(prefix)/bin/jq
-	rm -vf $(mandir)/man1/jq.1
-
-
-www: docs/output www_binaries
-
-docs/output:
-	cd docs; rake build
-
-www_binaries: docs/output binaries
-	$(foreach platform, $(PLATFORMS), $(foreach binary, $(BINARIES), \
-	  mkdir -p docs/output/download/$(platform); \
-	  cp build/$(platform)/$(binary) docs/output/download/$(platform)/$(binary); ))

Makefile.am

@@ -0,0 +1,123 @@
+### C source files to be built and distributed.
+
+JQ_INCS = jq_parser.h builtin.h bytecode.h compile.h execute.h		\
+  forkable_stack.h frame_layout.h jv.h jv_alloc.h jv_aux.h jv_dtoa.h	\
+  jv_parse.h jv_unicode.h locfile.h opcode.h opcode_list.h parser.y	\
+  jv_utf8_tables.h lexer.l
+
+JQ_SRC = opcode.c bytecode.c compile.c execute.c builtin.c jv.c		\
+  jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_alloc.c	\
+  jq_test.c ${JQ_INCS}
+
+
+### C build options
+
+AM_CFLAGS = -Wextra -Wall -Wno-missing-field-initializers	\
+  -Wno-unused-parameter -Wno-unused-function
+
+
+### Generating the lexer and parser
+
+# While there is some autoconf macro support for lex/flex, it doesn't support
+# header file creation so we'll use good old make
+BUILT_SOURCES = lexer.h lexer.c parser.h parser.c
+lexer.c: lexer.l
+	flex -o lexer.c --header-file=lexer.h $<
+lexer.h: lexer.c
+
+# Tell YACC (bison) autoconf macros that you want a header file created.
+# If the --warnings=all fails, you probably have an old version of bison
+# OSX ships an old bison, so update with homebrew or macports
+AM_YFLAGS = --warnings=all -d
+
+
+### Building the jq binary
+
+main.c: config.h
+
+bin_PROGRAMS = jq
+jq_SOURCES = ${JQ_SRC} main.c
+
+
+### Tests (make check)
+
+TESTS = tests/all.test
+TEST_LOG_COMPILER = ${srcdir}/tests/run
+
+
+### Building the manpage
+
+# If ENABLE_DOCS is not set, just copy jq.1.default to jq.1
+# The real_docs target succeeds (possibly after building jq.1) only if ENABLE_DOCS is set
+# Thus, making "dist" depend on "real_docs" ensures that we never build a tarball with
+# a stub manpage.
+
+man_MANS = jq.1
+.PHONY: real_docs
+if ENABLE_DOCS
+jq.1: $(srcdir)/docs/content/3.manual/manual.yml
+	( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
+jq.1.default: $(srcdir)/docs/default_manpage.md
+	( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
+real_docs: jq.1
+	if cmp jq.1 $(srcdir)/jq.1.default > /dev/null; then\
+	  rm -f jq.1; $(MAKE) $(AM_MAKEFLAGS) jq.1;\
+	fi
+else
+jq.1: $(srcdir)/jq.1.default
+	cp $^ $@
+real_docs:
+	@echo "Ruby dependencies not found, cannot build manpage." > /dev/stderr
+	@echo "Follow the instructions in docs/README.md to install them" > /dev/stderr
+	@echo "and then rerun ./configure" > /dev/stderr
+	false
+endif
+
+
+### Packaging
+
+docs/site.yml: configure.ac
+	sed 's/^jq_version: .*/jq_version: $(VERSION)/' $@ > $@.new
+	mv $@.new $@
+
+install-binaries: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) install-exec
+
+# Ensure "make dist" fails when we can't build the real manpage
+dist-hook: real_docs
+
+DOC_FILES = docs/content docs/public docs/templates docs/site.yml	\
+            docs/Gemfile docs/Gemfile.lock docs/Rakefile docs/README.md \
+            docs/default_manpage.md jq.1.default
+
+# setup is only used by distribution developers, not package developers.
+# Still, as a matter of allowing patching, its not a bad idea to distribute 
+# the developer setup script in the tarball.
+EXTRA_DIST = config.h.in $(BUILT_SOURCES) $(man_MANS) $(TESTS)	\
+	$(TEST_LOG_COMPILER) gen_utf8_tables.py jq.spec \
+	$(DOC_FILES)
+
+# README.md is expected in Github projects, good stuff in it, so we'll 
+# distribute it and install it with the package in the doc directory.
+docdir = ${datadir}/doc/${PACKAGE}
+dist_doc_DATA = README.md COPYING AUTHORS README
+
+releasetag:
+	git tag -s "jq-$$(cat VERSION)" -m "jq release $$(cat VERSION)"
+
+RELEASE ?= 1
+rpm: jq
+	@echo "Packaging jq as an RPM ..."
+	mkdir -p rpm/SOURCES rpm/BUILD rpm/BUILDROOT rpm/RPMS rpm/SPECS
+	cp jq-$$(cat VERSION).tar.gz rpm/SOURCES/
+	rpmbuild -tb --define "_topdir ${PWD}/rpm" --define "_prefix /usr" --define "myver $$(cat VERSION)" --define "myrel ${RELEASE}" rpm/SOURCES/jq-$$(cat VERSION).tar.gz
+	find rpm/RPMS/ -name "*.rpm" -exec mv {} ./ \;
+	rm -rf rpm
+
+dist-clean-local:
+	rm -f ${BUILT_SOURCES}
+
+# Not sure why this doesn't get cleaned up automatically, guess
+# automake used to man pages which are hand coded?
+clean-local:
+	rm -f jq.1

README

@@ -0,0 +1 @@
+README.md

README.md

@@ -8,7 +8,16 @@ If you want to learn to use jq, read the documentation at
 documentation is generated from the docs/ folder of this repository.
 
 If you want to hack on jq, feel free, but be warned that its internals
-are not well-documented at the moment. Bring a hard hat and a shovel.
+are not well-documented at the moment. Bring a hard hat and a
+shovel. Also, read the wiki: http://github.com/stedolan/jq/wiki
 
-You can find some basic build instructions at
-[http://stedolan.github.com/jq/download](http://stedolan.github.com/jq/download)
+To build jq, run
+
+    ./configure
+    make
+    sudo make install (optionally)
+
+If you've just checked out the latest version from git (rather than
+using a released source tarball) then you'll need to run this first:
+
+    autoreconf

VERSION

@@ -1 +0,0 @@
-1.2

build/.gitignore

@@ -0,0 +1 @@
+*

builtin.c

@@ -1,10 +1,11 @@
 #include <string.h>
 #include "builtin.h"
-#include "bytecode.h"
 #include "compile.h"
-#include "parser.h"
+#include "jq_parser.h"
 #include "locfile.h"
 #include "jv_aux.h"
+#include "jv_unicode.h"
+
 
 
 typedef jv (*func_1)(jv);
@@ -43,7 +44,13 @@ static jv type_error2(jv bad1, jv bad2, const char* msg) {
 
 static jv f_plus(jv input, jv a, jv b) {
   jv_free(input);
-  if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
+  if (jv_get_kind(a) == JV_KIND_NULL) {
+    jv_free(a);
+    return b;
+  } else if (jv_get_kind(b) == JV_KIND_NULL) {
+    jv_free(b);
+    return a;
+  } else if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
     return jv_number(jv_number_value(a) + 
                      jv_number_value(b));
   } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
@@ -57,17 +64,25 @@ static jv f_plus(jv input, jv a, jv b) {
   }
 }
 
+static jv f_negate(jv input) {
+  if (jv_get_kind(input) != JV_KIND_NUMBER) {
+    return type_error(input, "cannot be negated");
+  }
+  jv ret = jv_number(-jv_number_value(input));
+  jv_free(input);
+  return ret;
+}
+
 static jv f_minus(jv input, jv a, jv b) {
   jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
     return jv_number(jv_number_value(a) - jv_number_value(b));
   } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
     jv out = jv_array();
-    for (int i=0; i<jv_array_length(jv_copy(a)); i++) {
-      jv x = jv_array_get(jv_copy(a), i);
+    jv_array_foreach(a, i, x) {
       int include = 1;
-      for (int j=0; j<jv_array_length(jv_copy(b)); j++) {
-        if (jv_equal(jv_copy(x), jv_array_get(jv_copy(b), j))) {
+      jv_array_foreach(b, j, y) {
+        if (jv_equal(jv_copy(x), y)) {
           include = 0;
           break;
         }
@@ -102,22 +117,6 @@ static jv f_divide(jv input, jv a, jv b) {
   }  
 }
 
-static jv f_add(jv array) {
-  if (jv_get_kind(array) != JV_KIND_ARRAY) {
-    return type_error(array, "cannot have its elements added");
-  } else if (jv_array_length(jv_copy(array)) == 0) {
-    jv_free(array);
-    return jv_null();
-  } else {
-    jv sum = jv_array_get(jv_copy(array), 0);
-    for (int i = 1; i < jv_array_length(jv_copy(array)); i++) {
-      sum = f_plus(jv_null(), sum, jv_array_get(jv_copy(array), i));
-    }
-    jv_free(array);
-    return sum;
-  }
-}
-
 static jv f_equal(jv input, jv a, jv b) {
   jv_free(input);
   return jv_bool(jv_equal(a, b));
@@ -161,9 +160,7 @@ static jv f_greatereq(jv input, jv a, jv b) {
 }
 
 static jv f_contains(jv a, jv b) {
-  jv_kind akind = jv_get_kind(a);
-
-  if (akind == jv_get_kind(b)) {
+  if (jv_get_kind(a) == jv_get_kind(b)) {
     return jv_bool(jv_contains(a, b));
   } else {
     return type_error2(a, b, "cannot have their containment checked");
@@ -207,6 +204,172 @@ static jv f_tostring(jv input) {
   }
 }
 
+#define CHARS_ALPHANUM "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+
+static jv escape_string(jv input, const char* escapings) {
+
+  assert(jv_get_kind(input) == JV_KIND_STRING);
+  const char* lookup[128] = {0};
+  const char* p = escapings;
+  while (*p) {
+    lookup[(int)*p] = p+1;
+    p++;
+    p += strlen(p);
+    p++;
+  }
+
+  jv ret = jv_string("");
+  const char* i = jv_string_value(input);
+  const char* end = i + jv_string_length(jv_copy(input));
+  const char* cstart;
+  int c = 0;
+  while ((i = jvp_utf8_next((cstart = i), end, &c))) {
+    assert(c != -1);
+    if (c < 128 && lookup[c]) {
+      ret = jv_string_append_str(ret, lookup[c]);
+    } else {
+      ret = jv_string_append_buf(ret, cstart, i - cstart);
+    }
+  }
+  jv_free(input);
+  return ret;
+
+}
+
+static jv f_format(jv input, jv fmt) {
+  if (jv_get_kind(fmt) != JV_KIND_STRING) {
+    jv_free(input);
+    return type_error(fmt, "is not a valid format");
+  }
+  const char* fmt_s = jv_string_value(fmt);
+  if (!strcmp(fmt_s, "json")) {
+    jv_free(fmt);
+    return jv_dump_string(input, 0);
+  } else if (!strcmp(fmt_s, "text")) {
+    jv_free(fmt);
+    return f_tostring(input);
+  } else if (!strcmp(fmt_s, "csv")) {
+    jv_free(fmt);
+    if (jv_get_kind(input) != JV_KIND_ARRAY)
+      return type_error(input, "cannot be csv-formatted, only array");
+    jv line = jv_string("");
+    jv_array_foreach(input, i, x) {
+      if (i) line = jv_string_append_str(line, ",");
+      switch (jv_get_kind(x)) {
+      case JV_KIND_NULL:
+        /* null rendered as empty string */
+        jv_free(x);
+        break;
+      case JV_KIND_TRUE:
+      case JV_KIND_FALSE:
+        line = jv_string_concat(line, jv_dump_string(x, 0));
+        break;
+      case JV_KIND_NUMBER:
+        if (jv_number_value(x) != jv_number_value(x)) {
+          /* NaN, render as empty string */
+          jv_free(x);
+        } else {
+          line = jv_string_concat(line, jv_dump_string(x, 0));
+        }
+        break;
+      case JV_KIND_STRING: {
+        line = jv_string_append_str(line, "\"");
+        line = jv_string_concat(line, escape_string(x, "\"\"\"\0"));
+        line = jv_string_append_str(line, "\"");
+        break;
+      }
+      default:
+        jv_free(input);
+        jv_free(line);
+        return type_error(x, "is not valid in a csv row");
+      }
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "html")) {
+    jv_free(fmt);
+    return escape_string(f_tostring(input), "&&amp;\0<&lt;\0>&gt;\0'&apos;\0\"&quot;\0");
+  } else if (!strcmp(fmt_s, "uri")) {
+    jv_free(fmt);
+    input = f_tostring(input);
+
+    int unreserved[128] = {0};
+    const char* p = CHARS_ALPHANUM "-_.!~*'()";
+    while (*p) unreserved[(int)*p++] = 1;
+
+    jv line = jv_string("");
+    const char* s = jv_string_value(input);
+    for (int i=0; i<jv_string_length(jv_copy(input)); i++) {
+      unsigned ch = (unsigned)(unsigned char)*s;
+      if (ch < 128 && unreserved[ch]) {
+        line = jv_string_append_buf(line, s, 1);
+      } else {
+        line = jv_string_concat(line, jv_string_fmt("%%%02x", ch));
+      }
+      s++;
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "sh")) {
+    jv_free(fmt);
+    if (jv_get_kind(input) != JV_KIND_ARRAY)
+      input = jv_array_set(jv_array(), 0, input);
+    jv line = jv_string("");
+    jv_array_foreach(input, i, x) {
+      if (i) line = jv_string_append_str(line, " ");
+      switch (jv_get_kind(x)) {
+      case JV_KIND_NULL:
+      case JV_KIND_TRUE:
+      case JV_KIND_FALSE:
+      case JV_KIND_NUMBER:
+        line = jv_string_concat(line, jv_dump_string(x, 0));
+        break;
+
+      case JV_KIND_STRING: {
+        line = jv_string_append_str(line, "'");
+        line = jv_string_concat(line, escape_string(x, "''\\''\0"));
+        line = jv_string_append_str(line, "'");
+        break;
+      }
+
+      default:
+        jv_free(input);
+        jv_free(line);
+        return type_error(x, "can not be escaped for shell");
+      }
+    }
+    jv_free(input);
+    return line;
+  } else if (!strcmp(fmt_s, "base64")) {
+    jv_free(fmt);
+    input = f_tostring(input);
+    jv line = jv_string("");
+    const char b64[64 + 1] = CHARS_ALPHANUM "+/";
+    const char* data = jv_string_value(input);
+    int len = jv_string_length(jv_copy(input));
+    for (int i=0; i<len; i+=3) {
+      uint32_t code = 0;
+      int n = len - i >= 3 ? 3 : len-i;
+      for (int j=0; j<3; j++) {
+        code <<= 8;
+        code |= j < n ? (unsigned)data[i+j] : 0;
+      }
+      char buf[4];
+      for (int j=0; j<4; j++) {
+        buf[j] = b64[(code >> (18 - j*6)) & 0x3f];
+      }
+      if (n < 3) buf[3] = '=';
+      if (n < 2) buf[2] = '=';
+      line = jv_string_append_buf(line, buf, sizeof(buf));
+    }
+    jv_free(input);
+    return line;
+  } else {
+    jv_free(input);
+    return jv_invalid_with_msg(jv_string_concat(fmt, jv_string(" is not a valid format")));
+  }
+}
+
 static jv f_keys(jv input) {
   if (jv_get_kind(input) == JV_KIND_OBJECT || jv_get_kind(input) == JV_KIND_ARRAY) {
     return jv_keys(input);
@@ -305,15 +468,19 @@ static jv f_error(jv input, jv msg) {
   return jv_invalid_with_msg(msg);
 }
 
-
-static struct cfunction function_list[] = {
+static const struct cfunction function_list[] = {
   {(cfunction_ptr)f_plus, "_plus", 3},
+  {(cfunction_ptr)f_negate, "_negate", 1},
   {(cfunction_ptr)f_minus, "_minus", 3},
   {(cfunction_ptr)f_multiply, "_multiply", 3},
   {(cfunction_ptr)f_divide, "_divide", 3},
   {(cfunction_ptr)f_tonumber, "tonumber", 1},
   {(cfunction_ptr)f_tostring, "tostring", 1},
   {(cfunction_ptr)f_keys, "keys", 1},
+  {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking
+  {(cfunction_ptr)jv_getpath, "getpath", 2},
+  {(cfunction_ptr)jv_delpaths, "delpaths", 2},
+  {(cfunction_ptr)jv_has, "has", 2},
   {(cfunction_ptr)f_equal, "_equal", 3},
   {(cfunction_ptr)f_notequal, "_notequal", 3},
   {(cfunction_ptr)f_less, "_less", 3},
@@ -323,7 +490,6 @@ static struct cfunction function_list[] = {
   {(cfunction_ptr)f_contains, "contains", 2},
   {(cfunction_ptr)f_length, "length", 1},
   {(cfunction_ptr)f_type, "type", 1},
-  {(cfunction_ptr)f_add, "add", 1},
   {(cfunction_ptr)f_sort, "sort", 1},
   {(cfunction_ptr)f_sort_by_impl, "_sort_by_impl", 2},
   {(cfunction_ptr)f_group_by_impl, "_group_by_impl", 2},
@@ -332,31 +498,43 @@ static struct cfunction function_list[] = {
   {(cfunction_ptr)f_min_by_impl, "_min_by_impl", 2},
   {(cfunction_ptr)f_max_by_impl, "_max_by_impl", 2},
   {(cfunction_ptr)f_error, "error", 2},
+  {(cfunction_ptr)f_format, "format", 2},
 };
 
-static struct symbol_table cbuiltins = 
-  {function_list, sizeof(function_list)/sizeof(function_list[0])};
-
-typedef block (*bytecoded_builtin)();
 struct bytecoded_builtin { const char* name; block code; };
 static block bind_bytecoded_builtins(block b) {
-  struct bytecoded_builtin builtin_defs[] = {
-    {"empty", gen_op_simple(BACKTRACK)},
-    {"false", gen_const(jv_false())},
-    {"true", gen_const(jv_true())},
-    {"null", gen_const(jv_null())},
-    {"not", gen_condbranch(gen_const(jv_false()),
-                           gen_const(jv_true()))}
-  };
   block builtins = gen_noop();
-  for (unsigned i=0; i<sizeof(builtin_defs)/sizeof(builtin_defs[0]); i++) {
-    builtins = BLOCK(builtins, gen_function(builtin_defs[i].name, gen_noop(),
-                                            builtin_defs[i].code));
+  {
+    struct bytecoded_builtin builtin_defs[] = {
+      {"empty", gen_op_simple(BACKTRACK)},
+      {"false", gen_const(jv_false())},
+      {"true", gen_const(jv_true())},
+      {"null", gen_const(jv_null())},
+      {"not", gen_condbranch(gen_const(jv_false()),
+                             gen_const(jv_true()))}
+    };
+    for (unsigned i=0; i<sizeof(builtin_defs)/sizeof(builtin_defs[0]); i++) {
+      builtins = BLOCK(builtins, gen_function(builtin_defs[i].name, gen_noop(),
+                                              builtin_defs[i].code));
+    }
   }
+  {
+    struct bytecoded_builtin builtin_def_1arg[] = {
+      {"path", BLOCK(gen_op_simple(DUP), 
+                     gen_call("arg", gen_noop()),
+                     gen_op_simple(GETPATH))},
+    };
+    for (unsigned i=0; i<sizeof(builtin_def_1arg)/sizeof(builtin_def_1arg[0]); i++) {
+      builtins = BLOCK(builtins, gen_function(builtin_def_1arg[i].name,
+                                              gen_op_block_unbound(CLOSURE_PARAM, "arg"),
+                                              builtin_def_1arg[i].code));
+    }
+  }
+  
   return block_bind(builtins, b, OP_IS_CALL_PSEUDO);
 }
 
-static const char* jq_builtins[] = {
+static const char* const jq_builtins[] = {
   "def map(f): [.[] | f];",
   "def select(f): if f then . else empty end;",
   "def sort_by(f): _sort_by_impl(map([f]));",
@@ -364,6 +542,11 @@ static const char* jq_builtins[] = {
   "def unique: group_by(.) | map(.[0]);",
   "def max_by(f): _max_by_impl(map([f]));",
   "def min_by(f): _min_by_impl(map([f]));",
+  "def add: fold null as $sum (.[] | $sum + .);",
+  "def del(f): delpaths([path(f)]);",
+  "def _assign(paths; value): value as $v | fold . as $obj (path(paths) as $p | $obj | setpath($p; $v));",
+  "def _modify(paths; update): fold . as $obj (path(paths) as $p | $obj | setpath($p; getpath($p) | update));",
+  "def recurse(f): ., (f | select(. != null) | recurse(f));",
 };
 
 
@@ -378,5 +561,5 @@ block builtins_bind(block b) {
     locfile_free(&src);
   }
   b = bind_bytecoded_builtins(b);
-  return gen_cbinding(&cbuiltins, b);
+  return gen_cbinding(function_list, sizeof(function_list)/sizeof(function_list[0]), b);
 }

compile.c

@@ -30,7 +30,7 @@ struct inst {
     uint16_t intval;
     struct inst* target;
     jv constant;
-    struct cfunction* cfunc;
+    const struct cfunction* cfunc;
   } imm;
 
   location source;
@@ -292,11 +292,7 @@ block gen_collect(block expr) {
                                gen_noop(), OP_HAS_VARIABLE);
   block c = BLOCK(gen_op_simple(DUP), gen_const(jv_array()), array_var);
 
-  block tail = BLOCK(gen_op_simple(DUP),
-                     gen_op_var_bound(LOADV, array_var),
-                     gen_op_simple(SWAP),
-                     gen_op_simple(APPEND),
-                     gen_op_var_bound(STOREV, array_var),
+  block tail = BLOCK(gen_op_var_bound(APPEND, array_var),
                      gen_op_simple(BACKTRACK));
 
   return BLOCK(c,
@@ -306,20 +302,15 @@ block gen_collect(block expr) {
                gen_op_var_bound(LOADV, array_var));
 }
 
-block gen_assign(block expr) {
-  block result_var = block_bind(gen_op_var_unbound(STOREV, "result"),
-                                gen_noop(), OP_HAS_VARIABLE);
-
-  block loop = BLOCK(gen_op_simple(DUP),
-                     expr,
-                     gen_op_var_bound(ASSIGN, result_var),
-                     gen_op_simple(BACKTRACK));
-
+block gen_fold(const char* varname, block init, block fold) {
+  block loop = BLOCK(fold, gen_op_var_unbound(STOREV, varname), gen_op_simple(BACKTRACK));
   return BLOCK(gen_op_simple(DUP),
-               result_var,
-               gen_op_target(FORK, loop),
-               loop,
-               gen_op_var_bound(LOADV, result_var));
+               init,
+               block_bind(gen_op_var_unbound(STOREV, varname),
+                          BLOCK(gen_op_target(FORK, loop),
+                                loop,
+                                gen_op_var_unbound(LOADV, varname)),
+                          OP_HAS_VARIABLE));
 }
 
 block gen_definedor(block a, block b) {
@@ -380,16 +371,22 @@ block gen_or(block a, block b) {
                                                    gen_const(jv_false())))));
 }
 
+block gen_var_binding(block var, const char* name, block body) {
+  return BLOCK(gen_op_simple(DUP), var,
+               block_bind(gen_op_var_unbound(STOREV, name),
+                          body, OP_HAS_VARIABLE));
+}
+
 block gen_cond(block cond, block iftrue, block iffalse) {
   return BLOCK(gen_op_simple(DUP), cond, 
                gen_condbranch(BLOCK(gen_op_simple(POP), iftrue),
                               BLOCK(gen_op_simple(POP), iffalse)));
 }
 
-block gen_cbinding(struct symbol_table* t, block code) {
-  for (int cfunc=0; cfunc<t->ncfunctions; cfunc++) {
+block gen_cbinding(const struct cfunction* cfunctions, int ncfunctions, block code) {
+  for (int cfunc=0; cfunc<ncfunctions; cfunc++) {
     inst* i = inst_new(CLOSURE_CREATE_C);
-    i->imm.cfunc = &t->cfunctions[cfunc];
+    i->imm.cfunc = &cfunctions[cfunc];
     i->symbol = strdup(i->imm.cfunc->name);
     code = block_bind(inst_block(i), code, OP_IS_CALL_PSEUDO);
   }

compile.h

@@ -7,6 +7,7 @@
 
 struct bytecode;
 struct symbol_table;
+struct cfunction;
 
 struct inst;
 typedef struct inst inst;
@@ -33,15 +34,17 @@ block gen_call(const char* name, block body);
 block gen_subexp(block a);
 block gen_both(block a, block b);
 block gen_collect(block expr);
-block gen_assign(block expr);
+block gen_fold(const char* varname, block init, block body);
 block gen_definedor(block a, block b);
 block gen_condbranch(block iftrue, block iffalse);
 block gen_and(block a, block b);
 block gen_or(block a, block b);
 
+block gen_var_binding(block var, const char* name, block body);
+
 block gen_cond(block cond, block iftrue, block iffalse);
 
-block gen_cbinding(struct symbol_table* functions, block b);
+block gen_cbinding(const struct cfunction* functions, int nfunctions, block b);
 
 void block_append(block* b, block b2);
 block block_join(block a, block b);

config.h.in

@@ -0,0 +1,32 @@
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Define to 1 if your C compiler doesn't accept -c and -o together. */
+#undef NO_MINUS_C_MINUS_O
+
+/* Name of package */
+#undef PACKAGE
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Version number of package */
+#undef VERSION
+
+/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
+   `char[]'. */
+#undef YYTEXT_POINTER

config/compile

@@ -0,0 +1,343 @@
+#! /bin/sh
+# Wrapper for compilers which do not understand '-c -o'.
+
+scriptversion=2012-03-05.13; # UTC
+
+# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2009, 2010, 2012 Free
+# Software Foundation, Inc.
+# Written by Tom Tromey <tromey@cygnus.com>.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# This file is maintained in Automake, please report
+# bugs to <bug-automake@gnu.org> or send patches to
+# <automake-patches@gnu.org>.
+
+nl='
+'
+
+# We need space, tab and new line, in precisely that order.  Quoting is
+# there to prevent tools from complaining about whitespace usage.
+IFS=" ""	$nl"
+
+file_conv=
+
+# func_file_conv build_file lazy
+# Convert a $build file to $host form and store it in $file
+# Currently only supports Windows hosts. If the determined conversion
+# type is listed in (the comma separated) LAZY, no conversion will
+# take place.
+func_file_conv ()
+{
+  file=$1
+  case $file in
+    / | /[!/]*) # absolute file, and not a UNC file
+      if test -z "$file_conv"; then
+	# lazily determine how to convert abs files
+	case `uname -s` in
+	  MINGW*)
+	    file_conv=mingw
+	    ;;
+	  CYGWIN*)
+	    file_conv=cygwin
+	    ;;
+	  *)
+	    file_conv=wine
+	    ;;
+	esac
+      fi
+      case $file_conv/,$2, in
+	*,$file_conv,*)
+	  ;;
+	mingw/*)
+	  file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'`
+	  ;;
+	cygwin/*)
+	  file=`cygpath -m "$file" || echo "$file"`
+	  ;;
+	wine/*)
+	  file=`winepath -w "$file" || echo "$file"`
+	  ;;
+      esac
+      ;;
+  esac
+}
+
+# func_cl_dashL linkdir
+# Make cl look for libraries in LINKDIR
+func_cl_dashL ()
+{
+  func_file_conv "$1"
+  if test -z "$lib_path"; then
+    lib_path=$file
+  else
+    lib_path="$lib_path;$file"
+  fi
+  linker_opts="$linker_opts -LIBPATH:$file"
+}
+
+# func_cl_dashl library
+# Do a library search-path lookup for cl
+func_cl_dashl ()
+{
+  lib=$1
+  found=no
+  save_IFS=$IFS
+  IFS=';'
+  for dir in $lib_path $LIB
+  do
+    IFS=$save_IFS
+    if $shared && test -f "$dir/$lib.dll.lib"; then
+      found=yes
+      lib=$dir/$lib.dll.lib
+      break
+    fi
+    if test -f "$dir/$lib.lib"; then
+      found=yes
+      lib=$dir/$lib.lib
+      break
+    fi
+  done
+  IFS=$save_IFS
+
+  if test "$found" != yes; then
+    lib=$lib.lib
+  fi
+}
+
+# func_cl_wrapper cl arg...
+# Adjust compile command to suit cl
+func_cl_wrapper ()
+{
+  # Assume a capable shell
+  lib_path=
+  shared=:
+  linker_opts=
+  for arg
+  do
+    if test -n "$eat"; then
+      eat=
+    else
+      case $1 in
+	-o)
+	  # configure might choose to run compile as 'compile cc -o foo foo.c'.
+	  eat=1
+	  case $2 in
+	    *.o | *.[oO][bB][jJ])
+	      func_file_conv "$2"
+	      set x "$@" -Fo"$file"
+	      shift
+	      ;;
+	    *)
+	      func_file_conv "$2"
+	      set x "$@" -Fe"$file"
+	      shift
+	      ;;
+	  esac
+	  ;;
+	-I)
+	  eat=1
+	  func_file_conv "$2" mingw
+	  set x "$@" -I"$file"
+	  shift
+	  ;;
+	-I*)
+	  func_file_conv "${1#-I}" mingw
+	  set x "$@" -I"$file"
+	  shift
+	  ;;
+	-l)
+	  eat=1
+	  func_cl_dashl "$2"
+	  set x "$@" "$lib"
+	  shift
+	  ;;
+	-l*)
+	  func_cl_dashl "${1#-l}"
+	  set x "$@" "$lib"
+	  shift
+	  ;;
+	-L)
+	  eat=1
+	  func_cl_dashL "$2"
+	  ;;
+	-L*)
+	  func_cl_dashL "${1#-L}"
+	  ;;
+	-static)
+	  shared=false
+	  ;;
+	-Wl,*)
+	  arg=${1#-Wl,}
+	  save_ifs="$IFS"; IFS=','
+	  for flag in $arg; do
+	    IFS="$save_ifs"
+	    linker_opts="$linker_opts $flag"
+	  done
+	  IFS="$save_ifs"
+	  ;;
+	-Xlinker)
+	  eat=1
+	  linker_opts="$linker_opts $2"
+	  ;;
+	-*)
+	  set x "$@" "$1"
+	  shift
+	  ;;
+	*.cc | *.CC | *.cxx | *.CXX | *.[cC]++)
+	  func_file_conv "$1"
+	  set x "$@" -Tp"$file"
+	  shift
+	  ;;
+	*.c | *.cpp | *.CPP | *.lib | *.LIB | *.Lib | *.OBJ | *.obj | *.[oO])
+	  func_file_conv "$1" mingw
+	  set x "$@" "$file"
+	  shift
+	  ;;
+	*)
+	  set x "$@" "$1"
+	  shift
+	  ;;
+      esac
+    fi
+    shift
+  done
+  if test -n "$linker_opts"; then
+    linker_opts="-link$linker_opts"
+  fi
+  exec "$@" $linker_opts
+  exit 1
+}
+
+eat=
+
+case $1 in
+  '')
+     echo "$0: No command.  Try '$0 --help' for more information." 1>&2
+     exit 1;
+     ;;
+  -h | --h*)
+    cat <<\EOF
+Usage: compile [--help] [--version] PROGRAM [ARGS]
+
+Wrapper for compilers which do not understand '-c -o'.
+Remove '-o dest.o' from ARGS, run PROGRAM with the remaining
+arguments, and rename the output as expected.
+
+If you are trying to build a whole package this is not the
+right script to run: please start by reading the file 'INSTALL'.
+
+Report bugs to <bug-automake@gnu.org>.
+EOF
+    exit $?
+    ;;
+  -v | --v*)
+    echo "compile $scriptversion"
+    exit $?
+    ;;
+  cl | *[/\\]cl | cl.exe | *[/\\]cl.exe )
+    func_cl_wrapper "$@"      # Doesn't return...
+    ;;
+esac
+
+ofile=
+cfile=
+
+for arg
+do
+  if test -n "$eat"; then
+    eat=
+  else
+    case $1 in
+      -o)
+	# configure might choose to run compile as 'compile cc -o foo foo.c'.
+	# So we strip '-o arg' only if arg is an object.
+	eat=1
+	case $2 in
+	  *.o | *.obj)
+	    ofile=$2
+	    ;;
+	  *)
+	    set x "$@" -o "$2"
+	    shift
+	    ;;
+	esac
+	;;
+      *.c)
+	cfile=$1
+	set x "$@" "$1"
+	shift
+	;;
+      *)
+	set x "$@" "$1"
+	shift
+	;;
+    esac
+  fi
+  shift
+done
+
+if test -z "$ofile" || test -z "$cfile"; then
+  # If no '-o' option was seen then we might have been invoked from a
+  # pattern rule where we don't need one.  That is ok -- this is a
+  # normal compilation that the losing compiler can handle.  If no
+  # '.c' file was seen then we are probably linking.  That is also
+  # ok.
+  exec "$@"
+fi
+
+# Name of file we expect compiler to create.
+cofile=`echo "$cfile" | sed 's|^.*[\\/]||; s|^[a-zA-Z]:||; s/\.c$/.o/'`
+
+# Create the lock directory.
+# Note: use '[/\\:.-]' here to ensure that we don't use the same name
+# that we are using for the .o file.  Also, base the name on the expected
+# object file name, since that is what matters with a parallel build.
+lockdir=`echo "$cofile" | sed -e 's|[/\\:.-]|_|g'`.d
+while true; do
+  if mkdir "$lockdir" >/dev/null 2>&1; then
+    break
+  fi
+  sleep 1
+done
+# FIXME: race condition here if user kills between mkdir and trap.
+trap "rmdir '$lockdir'; exit 1" 1 2 15
+
+# Run the compile.
+"$@"
+ret=$?
+
+if test -f "$cofile"; then
+  test "$cofile" = "$ofile" || mv "$cofile" "$ofile"
+elif test -f "${cofile}bj"; then
+  test "${cofile}bj" = "$ofile" || mv "${cofile}bj" "$ofile"
+fi
+
+rmdir "$lockdir"
+exit $ret
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:

config/depcomp

@@ -0,0 +1,708 @@
+#! /bin/sh
+# depcomp - compile a program generating dependencies as side-effects
+
+scriptversion=2012-03-27.16; # UTC
+
+# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2006, 2007, 2009, 2010,
+# 2011, 2012 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Originally written by Alexandre Oliva <oliva@dcc.unicamp.br>.
+
+case $1 in
+  '')
+     echo "$0: No command.  Try '$0 --help' for more information." 1>&2
+     exit 1;
+     ;;
+  -h | --h*)
+    cat <<\EOF
+Usage: depcomp [--help] [--version] PROGRAM [ARGS]
+
+Run PROGRAMS ARGS to compile a file, generating dependencies
+as side-effects.
+
+Environment variables:
+  depmode     Dependency tracking mode.
+  source      Source file read by 'PROGRAMS ARGS'.
+  object      Object file output by 'PROGRAMS ARGS'.
+  DEPDIR      directory where to store dependencies.
+  depfile     Dependency file to output.
+  tmpdepfile  Temporary file to use when outputting dependencies.
+  libtool     Whether libtool is used (yes/no).
+
+Report bugs to <bug-automake@gnu.org>.
+EOF
+    exit $?
+    ;;
+  -v | --v*)
+    echo "depcomp $scriptversion"
+    exit $?
+    ;;
+esac
+
+# A tabulation character.
+tab='	'
+# A newline character.
+nl='
+'
+
+if test -z "$depmode" || test -z "$source" || test -z "$object"; then
+  echo "depcomp: Variables source, object and depmode must be set" 1>&2
+  exit 1
+fi
+
+# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po.
+depfile=${depfile-`echo "$object" |
+  sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`}
+tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`}
+
+rm -f "$tmpdepfile"
+
+# Some modes work just like other modes, but use different flags.  We
+# parameterize here, but still list the modes in the big case below,
+# to make depend.m4 easier to write.  Note that we *cannot* use a case
+# here, because this file can only contain one case statement.
+if test "$depmode" = hp; then
+  # HP compiler uses -M and no extra arg.
+  gccflag=-M
+  depmode=gcc
+fi
+
+if test "$depmode" = dashXmstdout; then
+   # This is just like dashmstdout with a different argument.
+   dashmflag=-xM
+   depmode=dashmstdout
+fi
+
+cygpath_u="cygpath -u -f -"
+if test "$depmode" = msvcmsys; then
+   # This is just like msvisualcpp but w/o cygpath translation.
+   # Just convert the backslash-escaped backslashes to single forward
+   # slashes to satisfy depend.m4
+   cygpath_u='sed s,\\\\,/,g'
+   depmode=msvisualcpp
+fi
+
+if test "$depmode" = msvc7msys; then
+   # This is just like msvc7 but w/o cygpath translation.
+   # Just convert the backslash-escaped backslashes to single forward
+   # slashes to satisfy depend.m4
+   cygpath_u='sed s,\\\\,/,g'
+   depmode=msvc7
+fi
+
+if test "$depmode" = xlc; then
+   # IBM C/C++ Compilers xlc/xlC can output gcc-like dependency informations.
+   gccflag=-qmakedep=gcc,-MF
+   depmode=gcc
+fi
+
+case "$depmode" in
+gcc3)
+## gcc 3 implements dependency tracking that does exactly what
+## we want.  Yay!  Note: for some reason libtool 1.4 doesn't like
+## it if -MD -MP comes after the -MF stuff.  Hmm.
+## Unfortunately, FreeBSD c89 acceptance of flags depends upon
+## the command line argument order; so add the flags where they
+## appear in depend2.am.  Note that the slowdown incurred here
+## affects only configure: in makefiles, %FASTDEP% shortcuts this.
+  for arg
+  do
+    case $arg in
+    -c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;;
+    *)  set fnord "$@" "$arg" ;;
+    esac
+    shift # fnord
+    shift # $arg
+  done
+  "$@"
+  stat=$?
+  if test $stat -eq 0; then :
+  else
+    rm -f "$tmpdepfile"
+    exit $stat
+  fi
+  mv "$tmpdepfile" "$depfile"
+  ;;
+
+gcc)
+## There are various ways to get dependency output from gcc.  Here's
+## why we pick this rather obscure method:
+## - Don't want to use -MD because we'd like the dependencies to end
+##   up in a subdir.  Having to rename by hand is ugly.
+##   (We might end up doing this anyway to support other compilers.)
+## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like
+##   -MM, not -M (despite what the docs say).
+## - Using -M directly means running the compiler twice (even worse
+##   than renaming).
+  if test -z "$gccflag"; then
+    gccflag=-MD,
+  fi
+  "$@" -Wp,"$gccflag$tmpdepfile"
+  stat=$?
+  if test $stat -eq 0; then :
+  else
+    rm -f "$tmpdepfile"
+    exit $stat
+  fi
+  rm -f "$depfile"
+  echo "$object : \\" > "$depfile"
+  alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
+## The second -e expression handles DOS-style file names with drive letters.
+  sed -e 's/^[^:]*: / /' \
+      -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile"
+## This next piece of magic avoids the "deleted header file" problem.
+## The problem is that when a header file which appears in a .P file
+## is deleted, the dependency causes make to die (because there is
+## typically no way to rebuild the header).  We avoid this by adding
+## dummy dependencies for each header file.  Too bad gcc doesn't do
+## this for us directly.
+  tr ' ' "$nl" < "$tmpdepfile" |
+## Some versions of gcc put a space before the ':'.  On the theory
+## that the space means something, we add a space to the output as
+## well.  hp depmode also adds that space, but also prefixes the VPATH
+## to the object.  Take care to not repeat it in the output.
+## Some versions of the HPUX 10.20 sed can't process this invocation
+## correctly.  Breaking it into two sed invocations is a workaround.
+    sed -e 's/^\\$//' -e '/^$/d' -e "s|.*$object$||" -e '/:$/d' \
+      | sed -e 's/$/ :/' >> "$depfile"
+  rm -f "$tmpdepfile"
+  ;;
+
+hp)
+  # This case exists only to let depend.m4 do its work.  It works by
+  # looking at the text of this script.  This case will never be run,
+  # since it is checked for above.
+  exit 1
+  ;;
+
+sgi)
+  if test "$libtool" = yes; then
+    "$@" "-Wp,-MDupdate,$tmpdepfile"
+  else
+    "$@" -MDupdate "$tmpdepfile"
+  fi
+  stat=$?
+  if test $stat -eq 0; then :
+  else
+    rm -f "$tmpdepfile"
+    exit $stat
+  fi
+  rm -f "$depfile"
+
+  if test -f "$tmpdepfile"; then  # yes, the sourcefile depend on other files
+    echo "$object : \\" > "$depfile"
+
+    # Clip off the initial element (the dependent).  Don't try to be
+    # clever and replace this with sed code, as IRIX sed won't handle
+    # lines with more than a fixed number of characters (4096 in
+    # IRIX 6.2 sed, 8192 in IRIX 6.5).  We also remove comment lines;
+    # the IRIX cc adds comments like '#:fec' to the end of the
+    # dependency line.
+    tr ' ' "$nl" < "$tmpdepfile" \
+    | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \
+    tr "$nl" ' ' >> "$depfile"
+    echo >> "$depfile"
+
+    # The second pass generates a dummy entry for each header file.
+    tr ' ' "$nl" < "$tmpdepfile" \
+   | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \
+   >> "$depfile"
+  else
+    # The sourcefile does not contain any dependencies, so just
+    # store a dummy comment line, to avoid errors with the Makefile
+    # "include basename.Plo" scheme.
+    echo "#dummy" > "$depfile"
+  fi
+  rm -f "$tmpdepfile"
+  ;;
+
+xlc)
+  # This case exists only to let depend.m4 do its work.  It works by
+  # looking at the text of this script.  This case will never be run,
+  # since it is checked for above.
+  exit 1
+  ;;
+
+aix)
+  # The C for AIX Compiler uses -M and outputs the dependencies
+  # in a .u file.  In older versions, this file always lives in the
+  # current directory.  Also, the AIX compiler puts '$object:' at the
+  # start of each line; $object doesn't have directory information.
+  # Version 6 uses the directory in both cases.
+  dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
+  test "x$dir" = "x$object" && dir=
+  base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
+  if test "$libtool" = yes; then
+    tmpdepfile1=$dir$base.u
+    tmpdepfile2=$base.u
+    tmpdepfile3=$dir.libs/$base.u
+    "$@" -Wc,-M
+  else
+    tmpdepfile1=$dir$base.u
+    tmpdepfile2=$dir$base.u
+    tmpdepfile3=$dir$base.u
+    "$@" -M
+  fi
+  stat=$?
+
+  if test $stat -eq 0; then :
+  else
+    rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
+    exit $stat
+  fi
+
+  for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
+  do
+    test -f "$tmpdepfile" && break
+  done
+  if test -f "$tmpdepfile"; then
+    # Each line is of the form 'foo.o: dependent.h'.
+    # Do two passes, one to just change these to
+    # '$object: dependent.h' and one to simply 'dependent.h:'.
+    sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
+    sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
+  else
+    # The sourcefile does not contain any dependencies, so just
+    # store a dummy comment line, to avoid errors with the Makefile
+    # "include basename.Plo" scheme.
+    echo "#dummy" > "$depfile"
+  fi
+  rm -f "$tmpdepfile"
+  ;;
+
+icc)
+  # Intel's C compiler anf tcc (Tiny C Compiler) understand '-MD -MF file'.
+  # However on
+  #    $CC -MD -MF foo.d -c -o sub/foo.o sub/foo.c
+  # ICC 7.0 will fill foo.d with something like
+  #    foo.o: sub/foo.c
+  #    foo.o: sub/foo.h
+  # which is wrong.  We want
+  #    sub/foo.o: sub/foo.c
+  #    sub/foo.o: sub/foo.h
+  #    sub/foo.c:
+  #    sub/foo.h:
+  # ICC 7.1 will output
+  #    foo.o: sub/foo.c sub/foo.h
+  # and will wrap long lines using '\':
+  #    foo.o: sub/foo.c ... \
+  #     sub/foo.h ... \
+  #     ...
+  # tcc 0.9.26 (FIXME still under development at the moment of writing)
+  # will emit a similar output, but also prepend the continuation lines
+  # with horizontal tabulation characters.
+  "$@" -MD -MF "$tmpdepfile"
+  stat=$?
+  if test $stat -eq 0; then :
+  else
+    rm -f "$tmpdepfile"
+    exit $stat
+  fi
+  rm -f "$depfile"
+  # Each line is of the form 'foo.o: dependent.h',
+  # or 'foo.o: dep1.h dep2.h \', or ' dep3.h dep4.h \'.
+  # Do two passes, one to just change these to
+  # '$object: dependent.h' and one to simply 'dependent.h:'.
+  sed -e "s/^[ $tab][ $tab]*/  /" -e "s,^[^:]*:,$object :," \
+    < "$tmpdepfile" > "$depfile"
+  sed '
+    s/[ '"$tab"'][ '"$tab"']*/ /g
+    s/^ *//
+    s/ *\\*$//
+    s/^[^:]*: *//
+    /^$/d
+    /:$/d
+    s/$/ :/
+  ' < "$tmpdepfile" >> "$depfile"
+  rm -f "$tmpdepfile"
+  ;;
+
+hp2)
+  # The "hp" stanza above does not work with aCC (C++) and HP's ia64
+  # compilers, which have integrated preprocessors.  The correct option
+  # to use with these is +Maked; it writes dependencies to a file named
+  # 'foo.d', which lands next to the object file, wherever that
+  # happens to be.
+  # Much of this is similar to the tru64 case; see comments there.
+  dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
+  test "x$dir" = "x$object" && dir=
+  base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
+  if test "$libtool" = yes; then
+    tmpdepfile1=$dir$base.d
+    tmpdepfile2=$dir.libs/$base.d
+    "$@" -Wc,+Maked
+  else
+    tmpdepfile1=$dir$base.d
+    tmpdepfile2=$dir$base.d
+    "$@" +Maked
+  fi
+  stat=$?
+  if test $stat -eq 0; then :
+  else
+     rm -f "$tmpdepfile1" "$tmpdepfile2"
+     exit $stat
+  fi
+
+  for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2"
+  do
+    test -f "$tmpdepfile" && break
+  done
+  if test -f "$tmpdepfile"; then
+    sed -e "s,^.*\.[a-z]*:,$object:," "$tmpdepfile" > "$depfile"
+    # Add 'dependent.h:' lines.
+    sed -ne '2,${
+	       s/^ *//
+	       s/ \\*$//
+	       s/$/:/
+	       p
+	     }' "$tmpdepfile" >> "$depfile"
+  else
+    echo "#dummy" > "$depfile"
+  fi
+  rm -f "$tmpdepfile" "$tmpdepfile2"
+  ;;
+
+tru64)
+   # The Tru64 compiler uses -MD to generate dependencies as a side
+   # effect.  'cc -MD -o foo.o ...' puts the dependencies into 'foo.o.d'.
+   # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put
+   # dependencies in 'foo.d' instead, so we check for that too.
+   # Subdirectories are respected.
+   dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
+   test "x$dir" = "x$object" && dir=
+   base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
+
+   if test "$libtool" = yes; then
+      # With Tru64 cc, shared objects can also be used to make a
+      # static library.  This mechanism is used in libtool 1.4 series to
+      # handle both shared and static libraries in a single compilation.
+      # With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d.
+      #
+      # With libtool 1.5 this exception was removed, and libtool now
+      # generates 2 separate objects for the 2 libraries.  These two
+      # compilations output dependencies in $dir.libs/$base.o.d and
+      # in $dir$base.o.d.  We have to check for both files, because
+      # one of the two compilations can be disabled.  We should prefer
+      # $dir$base.o.d over $dir.libs/$base.o.d because the latter is
+      # automatically cleaned when .libs/ is deleted, while ignoring
+      # the former would cause a distcleancheck panic.
+      tmpdepfile1=$dir.libs/$base.lo.d   # libtool 1.4
+      tmpdepfile2=$dir$base.o.d          # libtool 1.5
+      tmpdepfile3=$dir.libs/$base.o.d    # libtool 1.5
+      tmpdepfile4=$dir.libs/$base.d      # Compaq CCC V6.2-504
+      "$@" -Wc,-MD
+   else
+      tmpdepfile1=$dir$base.o.d
+      tmpdepfile2=$dir$base.d
+      tmpdepfile3=$dir$base.d
+      tmpdepfile4=$dir$base.d
+      "$@" -MD
+   fi
+
+   stat=$?
+   if test $stat -eq 0; then :
+   else
+      rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
+      exit $stat
+   fi
+
+   for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
+   do
+     test -f "$tmpdepfile" && break
+   done
+   if test -f "$tmpdepfile"; then
+      sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
+      sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
+   else
+      echo "#dummy" > "$depfile"
+   fi
+   rm -f "$tmpdepfile"
+   ;;
+
+msvc7)
+  if test "$libtool" = yes; then
+    showIncludes=-Wc,-showIncludes
+  else
+    showIncludes=-showIncludes
+  fi
+  "$@" $showIncludes > "$tmpdepfile"
+  stat=$?
+  grep -v '^Note: including file: ' "$tmpdepfile"
+  if test "$stat" = 0; then :
+  else
+    rm -f "$tmpdepfile"
+    exit $stat
+  fi
+  rm -f "$depfile"
+  echo "$object : \\" > "$depfile"
+  # The first sed program below extracts the file names and escapes
+  # backslashes for cygpath.  The second sed program outputs the file
+  # name when reading, but also accumulates all include files in the
+  # hold buffer in order to output them again at the end.  This only
+  # works with sed implementations that can handle large buffers.
+  sed < "$tmpdepfile" -n '
+/^Note: including file:  *\(.*\)/ {
+  s//\1/
+  s/\\/\\\\/g
+  p
+}' | $cygpath_u | sort -u | sed -n '
+s/ /\\ /g
+s/\(.*\)/'"$tab"'\1 \\/p
+s/.\(.*\) \\/\1:/
+H
+$ {
+  s/.*/'"$tab"'/
+  G
+  p
+}' >> "$depfile"
+  rm -f "$tmpdepfile"
+  ;;
+
+msvc7msys)
+  # This case exists only to let depend.m4 do its work.  It works by
+  # looking at the text of this script.  This case will never be run,
+  # since it is checked for above.
+  exit 1
+  ;;
+
+#nosideeffect)
+  # This comment above is used by automake to tell side-effect
+  # dependency tracking mechanisms from slower ones.
+
+dashmstdout)
+  # Important note: in order to support this mode, a compiler *must*
+  # always write the preprocessed file to stdout, regardless of -o.
+  "$@" || exit $?
+
+  # Remove the call to Libtool.
+  if test "$libtool" = yes; then
+    while test "X$1" != 'X--mode=compile'; do
+      shift
+    done
+    shift
+  fi
+
+  # Remove '-o $object'.
+  IFS=" "
+  for arg
+  do
+    case $arg in
+    -o)
+      shift
+      ;;
+    $object)
+      shift
+      ;;
+    *)
+      set fnord "$@" "$arg"
+      shift # fnord
+      shift # $arg
+      ;;
+    esac
+  done
+
+  test -z "$dashmflag" && dashmflag=-M
+  # Require at least two characters before searching for ':'
+  # in the target name.  This is to cope with DOS-style filenames:
+  # a dependency such as 'c:/foo/bar' could be seen as target 'c' otherwise.
+  "$@" $dashmflag |
+    sed 's:^['"$tab"' ]*[^:'"$tab"' ][^:][^:]*\:['"$tab"' ]*:'"$object"'\: :' > "$tmpdepfile"
+  rm -f "$depfile"
+  cat < "$tmpdepfile" > "$depfile"
+  tr ' ' "$nl" < "$tmpdepfile" | \
+## Some versions of the HPUX 10.20 sed can't process this invocation
+## correctly.  Breaking it into two sed invocations is a workaround.
+    sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
+  rm -f "$tmpdepfile"
+  ;;
+
+dashXmstdout)
+  # This case only exists to satisfy depend.m4.  It is never actually
+  # run, as this mode is specially recognized in the preamble.
+  exit 1
+  ;;
+
+makedepend)
+  "$@" || exit $?
+  # Remove any Libtool call
+  if test "$libtool" = yes; then
+    while test "X$1" != 'X--mode=compile'; do
+      shift
+    done
+    shift
+  fi
+  # X makedepend
+  shift
+  cleared=no eat=no
+  for arg
+  do
+    case $cleared in
+    no)
+      set ""; shift
+      cleared=yes ;;
+    esac
+    if test $eat = yes; then
+      eat=no
+      continue
+    fi
+    case "$arg" in
+    -D*|-I*)
+      set fnord "$@" "$arg"; shift ;;
+    # Strip any option that makedepend may not understand.  Remove
+    # the object too, otherwise makedepend will parse it as a source file.
+    -arch)
+      eat=yes ;;
+    -*|$object)
+      ;;
+    *)
+      set fnord "$@" "$arg"; shift ;;
+    esac
+  done
+  obj_suffix=`echo "$object" | sed 's/^.*\././'`
+  touch "$tmpdepfile"
+  ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@"
+  rm -f "$depfile"
+  # makedepend may prepend the VPATH from the source file name to the object.
+  # No need to regex-escape $object, excess matching of '.' is harmless.
+  sed "s|^.*\($object *:\)|\1|" "$tmpdepfile" > "$depfile"
+  sed '1,2d' "$tmpdepfile" | tr ' ' "$nl" | \
+## Some versions of the HPUX 10.20 sed can't process this invocation
+## correctly.  Breaking it into two sed invocations is a workaround.
+    sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
+  rm -f "$tmpdepfile" "$tmpdepfile".bak
+  ;;
+
+cpp)
+  # Important note: in order to support this mode, a compiler *must*
+  # always write the preprocessed file to stdout.
+  "$@" || exit $?
+
+  # Remove the call to Libtool.
+  if test "$libtool" = yes; then
+    while test "X$1" != 'X--mode=compile'; do
+      shift
+    done
+    shift
+  fi
+
+  # Remove '-o $object'.
+  IFS=" "
+  for arg
+  do
+    case $arg in
+    -o)
+      shift
+      ;;
+    $object)
+      shift
+      ;;
+    *)
+      set fnord "$@" "$arg"
+      shift # fnord
+      shift # $arg
+      ;;
+    esac
+  done
+
+  "$@" -E |
+    sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \
+       -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' |
+    sed '$ s: \\$::' > "$tmpdepfile"
+  rm -f "$depfile"
+  echo "$object : \\" > "$depfile"
+  cat < "$tmpdepfile" >> "$depfile"
+  sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile"
+  rm -f "$tmpdepfile"
+  ;;
+
+msvisualcpp)
+  # Important note: in order to support this mode, a compiler *must*
+  # always write the preprocessed file to stdout.
+  "$@" || exit $?
+
+  # Remove the call to Libtool.
+  if test "$libtool" = yes; then
+    while test "X$1" != 'X--mode=compile'; do
+      shift
+    done
+    shift
+  fi
+
+  IFS=" "
+  for arg
+  do
+    case "$arg" in
+    -o)
+      shift
+      ;;
+    $object)
+      shift
+      ;;
+    "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI")
+	set fnord "$@"
+	shift
+	shift
+	;;
+    *)
+	set fnord "$@" "$arg"
+	shift
+	shift
+	;;
+    esac
+  done
+  "$@" -E 2>/dev/null |
+  sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile"
+  rm -f "$depfile"
+  echo "$object : \\" > "$depfile"
+  sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::'"$tab"'\1 \\:p' >> "$depfile"
+  echo "$tab" >> "$depfile"
+  sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile"
+  rm -f "$tmpdepfile"
+  ;;
+
+msvcmsys)
+  # This case exists only to let depend.m4 do its work.  It works by
+  # looking at the text of this script.  This case will never be run,
+  # since it is checked for above.
+  exit 1
+  ;;
+
+none)
+  exec "$@"
+  ;;
+
+*)
+  echo "Unknown depmode $depmode" 1>&2
+  exit 1
+  ;;
+esac
+
+exit 0
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:

config/install-sh

@@ -0,0 +1,527 @@
+#!/bin/sh
+# install - install a program, script, or datafile
+
+scriptversion=2011-01-19.21; # UTC
+
+# This originates from X11R5 (mit/util/scripts/install.sh), which was
+# later released in X11R6 (xc/config/util/install.sh) with the
+# following copyright and license.
+#
+# Copyright (C) 1994 X Consortium
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
+# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# Except as contained in this notice, the name of the X Consortium shall not
+# be used in advertising or otherwise to promote the sale, use or other deal-
+# ings in this Software without prior written authorization from the X Consor-
+# tium.
+#
+#
+# FSF changes to this file are in the public domain.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+
+nl='
+'
+IFS=" ""	$nl"
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit=${DOITPROG-}
+if test -z "$doit"; then
+  doit_exec=exec
+else
+  doit_exec=$doit
+fi
+
+# Put in absolute file names if you don't have them in your path;
+# or use environment vars.
+
+chgrpprog=${CHGRPPROG-chgrp}
+chmodprog=${CHMODPROG-chmod}
+chownprog=${CHOWNPROG-chown}
+cmpprog=${CMPPROG-cmp}
+cpprog=${CPPROG-cp}
+mkdirprog=${MKDIRPROG-mkdir}
+mvprog=${MVPROG-mv}
+rmprog=${RMPROG-rm}
+stripprog=${STRIPPROG-strip}
+
+posix_glob='?'
+initialize_posix_glob='
+  test "$posix_glob" != "?" || {
+    if (set -f) 2>/dev/null; then
+      posix_glob=
+    else
+      posix_glob=:
+    fi
+  }
+'
+
+posix_mkdir=
+
+# Desired mode of installed file.
+mode=0755
+
+chgrpcmd=
+chmodcmd=$chmodprog
+chowncmd=
+mvcmd=$mvprog
+rmcmd="$rmprog -f"
+stripcmd=
+
+src=
+dst=
+dir_arg=
+dst_arg=
+
+copy_on_change=false
+no_target_directory=
+
+usage="\
+Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
+   or: $0 [OPTION]... SRCFILES... DIRECTORY
+   or: $0 [OPTION]... -t DIRECTORY SRCFILES...
+   or: $0 [OPTION]... -d DIRECTORIES...
+
+In the 1st form, copy SRCFILE to DSTFILE.
+In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
+In the 4th, create DIRECTORIES.
+
+Options:
+     --help     display this help and exit.
+     --version  display version info and exit.
+
+  -c            (ignored)
+  -C            install only if different (preserve the last data modification time)
+  -d            create directories instead of installing files.
+  -g GROUP      $chgrpprog installed files to GROUP.
+  -m MODE       $chmodprog installed files to MODE.
+  -o USER       $chownprog installed files to USER.
+  -s            $stripprog installed files.
+  -t DIRECTORY  install into DIRECTORY.
+  -T            report an error if DSTFILE is a directory.
+
+Environment variables override the default commands:
+  CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
+  RMPROG STRIPPROG
+"
+
+while test $# -ne 0; do
+  case $1 in
+    -c) ;;
+
+    -C) copy_on_change=true;;
+
+    -d) dir_arg=true;;
+
+    -g) chgrpcmd="$chgrpprog $2"
+	shift;;
+
+    --help) echo "$usage"; exit $?;;
+
+    -m) mode=$2
+	case $mode in
+	  *' '* | *'	'* | *'
+'*	  | *'*'* | *'?'* | *'['*)
+	    echo "$0: invalid mode: $mode" >&2
+	    exit 1;;
+	esac
+	shift;;
+
+    -o) chowncmd="$chownprog $2"
+	shift;;
+
+    -s) stripcmd=$stripprog;;
+
+    -t) dst_arg=$2
+	# Protect names problematic for `test' and other utilities.
+	case $dst_arg in
+	  -* | [=\(\)!]) dst_arg=./$dst_arg;;
+	esac
+	shift;;
+
+    -T) no_target_directory=true;;
+
+    --version) echo "$0 $scriptversion"; exit $?;;
+
+    --)	shift
+	break;;
+
+    -*)	echo "$0: invalid option: $1" >&2
+	exit 1;;
+
+    *)  break;;
+  esac
+  shift
+done
+
+if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
+  # When -d is used, all remaining arguments are directories to create.
+  # When -t is used, the destination is already specified.
+  # Otherwise, the last argument is the destination.  Remove it from $@.
+  for arg
+  do
+    if test -n "$dst_arg"; then
+      # $@ is not empty: it contains at least $arg.
+      set fnord "$@" "$dst_arg"
+      shift # fnord
+    fi
+    shift # arg
+    dst_arg=$arg
+    # Protect names problematic for `test' and other utilities.
+    case $dst_arg in
+      -* | [=\(\)!]) dst_arg=./$dst_arg;;
+    esac
+  done
+fi
+
+if test $# -eq 0; then
+  if test -z "$dir_arg"; then
+    echo "$0: no input file specified." >&2
+    exit 1
+  fi
+  # It's OK to call `install-sh -d' without argument.
+  # This can happen when creating conditional directories.
+  exit 0
+fi
+
+if test -z "$dir_arg"; then
+  do_exit='(exit $ret); exit $ret'
+  trap "ret=129; $do_exit" 1
+  trap "ret=130; $do_exit" 2
+  trap "ret=141; $do_exit" 13
+  trap "ret=143; $do_exit" 15
+
+  # Set umask so as not to create temps with too-generous modes.
+  # However, 'strip' requires both read and write access to temps.
+  case $mode in
+    # Optimize common cases.
+    *644) cp_umask=133;;
+    *755) cp_umask=22;;
+
+    *[0-7])
+      if test -z "$stripcmd"; then
+	u_plus_rw=
+      else
+	u_plus_rw='% 200'
+      fi
+      cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
+    *)
+      if test -z "$stripcmd"; then
+	u_plus_rw=
+      else
+	u_plus_rw=,u+rw
+      fi
+      cp_umask=$mode$u_plus_rw;;
+  esac
+fi
+
+for src
+do
+  # Protect names problematic for `test' and other utilities.
+  case $src in
+    -* | [=\(\)!]) src=./$src;;
+  esac
+
+  if test -n "$dir_arg"; then
+    dst=$src
+    dstdir=$dst
+    test -d "$dstdir"
+    dstdir_status=$?
+  else
+
+    # Waiting for this to be detected by the "$cpprog $src $dsttmp" command
+    # might cause directories to be created, which would be especially bad
+    # if $src (and thus $dsttmp) contains '*'.
+    if test ! -f "$src" && test ! -d "$src"; then
+      echo "$0: $src does not exist." >&2
+      exit 1
+    fi
+
+    if test -z "$dst_arg"; then
+      echo "$0: no destination specified." >&2
+      exit 1
+    fi
+    dst=$dst_arg
+
+    # If destination is a directory, append the input filename; won't work
+    # if double slashes aren't ignored.
+    if test -d "$dst"; then
+      if test -n "$no_target_directory"; then
+	echo "$0: $dst_arg: Is a directory" >&2
+	exit 1
+      fi
+      dstdir=$dst
+      dst=$dstdir/`basename "$src"`
+      dstdir_status=0
+    else
+      # Prefer dirname, but fall back on a substitute if dirname fails.
+      dstdir=`
+	(dirname "$dst") 2>/dev/null ||
+	expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	     X"$dst" : 'X\(//\)[^/]' \| \
+	     X"$dst" : 'X\(//\)$' \| \
+	     X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
+	echo X"$dst" |
+	    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+		   s//\1/
+		   q
+		 }
+		 /^X\(\/\/\)[^/].*/{
+		   s//\1/
+		   q
+		 }
+		 /^X\(\/\/\)$/{
+		   s//\1/
+		   q
+		 }
+		 /^X\(\/\).*/{
+		   s//\1/
+		   q
+		 }
+		 s/.*/./; q'
+      `
+
+      test -d "$dstdir"
+      dstdir_status=$?
+    fi
+  fi
+
+  obsolete_mkdir_used=false
+
+  if test $dstdir_status != 0; then
+    case $posix_mkdir in
+      '')
+	# Create intermediate dirs using mode 755 as modified by the umask.
+	# This is like FreeBSD 'install' as of 1997-10-28.
+	umask=`umask`
+	case $stripcmd.$umask in
+	  # Optimize common cases.
+	  *[2367][2367]) mkdir_umask=$umask;;
+	  .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
+
+	  *[0-7])
+	    mkdir_umask=`expr $umask + 22 \
+	      - $umask % 100 % 40 + $umask % 20 \
+	      - $umask % 10 % 4 + $umask % 2
+	    `;;
+	  *) mkdir_umask=$umask,go-w;;
+	esac
+
+	# With -d, create the new directory with the user-specified mode.
+	# Otherwise, rely on $mkdir_umask.
+	if test -n "$dir_arg"; then
+	  mkdir_mode=-m$mode
+	else
+	  mkdir_mode=
+	fi
+
+	posix_mkdir=false
+	case $umask in
+	  *[123567][0-7][0-7])
+	    # POSIX mkdir -p sets u+wx bits regardless of umask, which
+	    # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
+	    ;;
+	  *)
+	    tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
+	    trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
+
+	    if (umask $mkdir_umask &&
+		exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
+	    then
+	      if test -z "$dir_arg" || {
+		   # Check for POSIX incompatibilities with -m.
+		   # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
+		   # other-writeable bit of parent directory when it shouldn't.
+		   # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
+		   ls_ld_tmpdir=`ls -ld "$tmpdir"`
+		   case $ls_ld_tmpdir in
+		     d????-?r-*) different_mode=700;;
+		     d????-?--*) different_mode=755;;
+		     *) false;;
+		   esac &&
+		   $mkdirprog -m$different_mode -p -- "$tmpdir" && {
+		     ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
+		     test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
+		   }
+		 }
+	      then posix_mkdir=:
+	      fi
+	      rmdir "$tmpdir/d" "$tmpdir"
+	    else
+	      # Remove any dirs left behind by ancient mkdir implementations.
+	      rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
+	    fi
+	    trap '' 0;;
+	esac;;
+    esac
+
+    if
+      $posix_mkdir && (
+	umask $mkdir_umask &&
+	$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
+      )
+    then :
+    else
+
+      # The umask is ridiculous, or mkdir does not conform to POSIX,
+      # or it failed possibly due to a race condition.  Create the
+      # directory the slow way, step by step, checking for races as we go.
+
+      case $dstdir in
+	/*) prefix='/';;
+	[-=\(\)!]*) prefix='./';;
+	*)  prefix='';;
+      esac
+
+      eval "$initialize_posix_glob"
+
+      oIFS=$IFS
+      IFS=/
+      $posix_glob set -f
+      set fnord $dstdir
+      shift
+      $posix_glob set +f
+      IFS=$oIFS
+
+      prefixes=
+
+      for d
+      do
+	test X"$d" = X && continue
+
+	prefix=$prefix$d
+	if test -d "$prefix"; then
+	  prefixes=
+	else
+	  if $posix_mkdir; then
+	    (umask=$mkdir_umask &&
+	     $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
+	    # Don't fail if two instances are running concurrently.
+	    test -d "$prefix" || exit 1
+	  else
+	    case $prefix in
+	      *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
+	      *) qprefix=$prefix;;
+	    esac
+	    prefixes="$prefixes '$qprefix'"
+	  fi
+	fi
+	prefix=$prefix/
+      done
+
+      if test -n "$prefixes"; then
+	# Don't fail if two instances are running concurrently.
+	(umask $mkdir_umask &&
+	 eval "\$doit_exec \$mkdirprog $prefixes") ||
+	  test -d "$dstdir" || exit 1
+	obsolete_mkdir_used=true
+      fi
+    fi
+  fi
+
+  if test -n "$dir_arg"; then
+    { test -z "$chowncmd" || $doit $chowncmd "$dst"; } &&
+    { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } &&
+    { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false ||
+      test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1
+  else
+
+    # Make a couple of temp file names in the proper directory.
+    dsttmp=$dstdir/_inst.$$_
+    rmtmp=$dstdir/_rm.$$_
+
+    # Trap to clean up those temp files at exit.
+    trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
+
+    # Copy the file name to the temp name.
+    (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
+
+    # and set any options; do chmod last to preserve setuid bits.
+    #
+    # If any of these fail, we abort the whole thing.  If we want to
+    # ignore errors from any of these, just make sure not to ignore
+    # errors from the above "$doit $cpprog $src $dsttmp" command.
+    #
+    { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
+    { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
+    { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
+    { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
+
+    # If -C, don't bother to copy if it wouldn't change the file.
+    if $copy_on_change &&
+       old=`LC_ALL=C ls -dlL "$dst"	2>/dev/null` &&
+       new=`LC_ALL=C ls -dlL "$dsttmp"	2>/dev/null` &&
+
+       eval "$initialize_posix_glob" &&
+       $posix_glob set -f &&
+       set X $old && old=:$2:$4:$5:$6 &&
+       set X $new && new=:$2:$4:$5:$6 &&
+       $posix_glob set +f &&
+
+       test "$old" = "$new" &&
+       $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
+    then
+      rm -f "$dsttmp"
+    else
+      # Rename the file to the real destination.
+      $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
+
+      # The rename failed, perhaps because mv can't rename something else
+      # to itself, or perhaps because mv is so ancient that it does not
+      # support -f.
+      {
+	# Now remove or move aside any old file at destination location.
+	# We try this two ways since rm can't unlink itself on some
+	# systems and the destination file might be busy for other
+	# reasons.  In this case, the final cleanup might fail but the new
+	# file should still install successfully.
+	{
+	  test ! -f "$dst" ||
+	  $doit $rmcmd -f "$dst" 2>/dev/null ||
+	  { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
+	    { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
+	  } ||
+	  { echo "$0: cannot unlink or rename $dst" >&2
+	    (exit 1); exit 1
+	  }
+	} &&
+
+	# Now rename the file to the real destination.
+	$doit $mvcmd "$dsttmp" "$dst"
+      }
+    fi || exit 1
+
+    trap '' 0
+  fi
+done
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:

config/missing

@@ -0,0 +1,331 @@
+#! /bin/sh
+# Common stub for a few missing GNU programs while installing.
+
+scriptversion=2012-01-06.13; # UTC
+
+# Copyright (C) 1996, 1997, 1999, 2000, 2002, 2003, 2004, 2005, 2006,
+# 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+# Originally by Fran,cois Pinard <pinard@iro.umontreal.ca>, 1996.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+if test $# -eq 0; then
+  echo 1>&2 "Try \`$0 --help' for more information"
+  exit 1
+fi
+
+run=:
+sed_output='s/.* --output[ =]\([^ ]*\).*/\1/p'
+sed_minuso='s/.* -o \([^ ]*\).*/\1/p'
+
+# In the cases where this matters, `missing' is being run in the
+# srcdir already.
+if test -f configure.ac; then
+  configure_ac=configure.ac
+else
+  configure_ac=configure.in
+fi
+
+msg="missing on your system"
+
+case $1 in
+--run)
+  # Try to run requested program, and just exit if it succeeds.
+  run=
+  shift
+  "$@" && exit 0
+  # Exit code 63 means version mismatch.  This often happens
+  # when the user try to use an ancient version of a tool on
+  # a file that requires a minimum version.  In this case we
+  # we should proceed has if the program had been absent, or
+  # if --run hadn't been passed.
+  if test $? = 63; then
+    run=:
+    msg="probably too old"
+  fi
+  ;;
+
+  -h|--h|--he|--hel|--help)
+    echo "\
+$0 [OPTION]... PROGRAM [ARGUMENT]...
+
+Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an
+error status if there is no known handling for PROGRAM.
+
+Options:
+  -h, --help      display this help and exit
+  -v, --version   output version information and exit
+  --run           try to run the given command, and emulate it if it fails
+
+Supported PROGRAM values:
+  aclocal      touch file \`aclocal.m4'
+  autoconf     touch file \`configure'
+  autoheader   touch file \`config.h.in'
+  autom4te     touch the output file, or create a stub one
+  automake     touch all \`Makefile.in' files
+  bison        create \`y.tab.[ch]', if possible, from existing .[ch]
+  flex         create \`lex.yy.c', if possible, from existing .c
+  help2man     touch the output file
+  lex          create \`lex.yy.c', if possible, from existing .c
+  makeinfo     touch the output file
+  yacc         create \`y.tab.[ch]', if possible, from existing .[ch]
+
+Version suffixes to PROGRAM as well as the prefixes \`gnu-', \`gnu', and
+\`g' are ignored when checking the name.
+
+Send bug reports to <bug-automake@gnu.org>."
+    exit $?
+    ;;
+
+  -v|--v|--ve|--ver|--vers|--versi|--versio|--version)
+    echo "missing $scriptversion (GNU Automake)"
+    exit $?
+    ;;
+
+  -*)
+    echo 1>&2 "$0: Unknown \`$1' option"
+    echo 1>&2 "Try \`$0 --help' for more information"
+    exit 1
+    ;;
+
+esac
+
+# normalize program name to check for.
+program=`echo "$1" | sed '
+  s/^gnu-//; t
+  s/^gnu//; t
+  s/^g//; t'`
+
+# Now exit if we have it, but it failed.  Also exit now if we
+# don't have it and --version was passed (most likely to detect
+# the program).  This is about non-GNU programs, so use $1 not
+# $program.
+case $1 in
+  lex*|yacc*)
+    # Not GNU programs, they don't have --version.
+    ;;
+
+  *)
+    if test -z "$run" && ($1 --version) > /dev/null 2>&1; then
+       # We have it, but it failed.
+       exit 1
+    elif test "x$2" = "x--version" || test "x$2" = "x--help"; then
+       # Could not run --version or --help.  This is probably someone
+       # running `$TOOL --version' or `$TOOL --help' to check whether
+       # $TOOL exists and not knowing $TOOL uses missing.
+       exit 1
+    fi
+    ;;
+esac
+
+# If it does not exist, or fails to run (possibly an outdated version),
+# try to emulate it.
+case $program in
+  aclocal*)
+    echo 1>&2 "\
+WARNING: \`$1' is $msg.  You should only need it if
+         you modified \`acinclude.m4' or \`${configure_ac}'.  You might want
+         to install the \`Automake' and \`Perl' packages.  Grab them from
+         any GNU archive site."
+    touch aclocal.m4
+    ;;
+
+  autoconf*)
+    echo 1>&2 "\
+WARNING: \`$1' is $msg.  You should only need it if
+         you modified \`${configure_ac}'.  You might want to install the
+         \`Autoconf' and \`GNU m4' packages.  Grab them from any GNU
+         archive site."
+    touch configure
+    ;;
+
+  autoheader*)
+    echo 1>&2 "\
+WARNING: \`$1' is $msg.  You should only need it if
+         you modified \`acconfig.h' or \`${configure_ac}'.  You might want
+         to install the \`Autoconf' and \`GNU m4' packages.  Grab them
+         from any GNU archive site."
+    files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' ${configure_ac}`
+    test -z "$files" && files="config.h"
+    touch_files=
+    for f in $files; do
+      case $f in
+      *:*) touch_files="$touch_files "`echo "$f" |
+				       sed -e 's/^[^:]*://' -e 's/:.*//'`;;
+      *) touch_files="$touch_files $f.in";;
+      esac
+    done
+    touch $touch_files
+    ;;
+
+  automake*)
+    echo 1>&2 "\
+WARNING: \`$1' is $msg.  You should only need it if
+         you modified \`Makefile.am', \`acinclude.m4' or \`${configure_ac}'.
+         You might want to install the \`Automake' and \`Perl' packages.
+         Grab them from any GNU archive site."
+    find . -type f -name Makefile.am -print |
+	   sed 's/\.am$/.in/' |
+	   while read f; do touch "$f"; done
+    ;;
+
+  autom4te*)
+    echo 1>&2 "\
+WARNING: \`$1' is needed, but is $msg.
+         You might have modified some files without having the
+         proper tools for further handling them.
+         You can get \`$1' as part of \`Autoconf' from any GNU
+         archive site."
+
+    file=`echo "$*" | sed -n "$sed_output"`
+    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
+    if test -f "$file"; then
+	touch $file
+    else
+	test -z "$file" || exec >$file
+	echo "#! /bin/sh"
+	echo "# Created by GNU Automake missing as a replacement of"
+	echo "#  $ $@"
+	echo "exit 0"
+	chmod +x $file
+	exit 1
+    fi
+    ;;
+
+  bison*|yacc*)
+    echo 1>&2 "\
+WARNING: \`$1' $msg.  You should only need it if
+         you modified a \`.y' file.  You may need the \`Bison' package
+         in order for those modifications to take effect.  You can get
+         \`Bison' from any GNU archive site."
+    rm -f y.tab.c y.tab.h
+    if test $# -ne 1; then
+        eval LASTARG=\${$#}
+	case $LASTARG in
+	*.y)
+	    SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'`
+	    if test -f "$SRCFILE"; then
+	         cp "$SRCFILE" y.tab.c
+	    fi
+	    SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'`
+	    if test -f "$SRCFILE"; then
+	         cp "$SRCFILE" y.tab.h
+	    fi
+	  ;;
+	esac
+    fi
+    if test ! -f y.tab.h; then
+	echo >y.tab.h
+    fi
+    if test ! -f y.tab.c; then
+	echo 'main() { return 0; }' >y.tab.c
+    fi
+    ;;
+
+  lex*|flex*)
+    echo 1>&2 "\
+WARNING: \`$1' is $msg.  You should only need it if
+         you modified a \`.l' file.  You may need the \`Flex' package
+         in order for those modifications to take effect.  You can get
+         \`Flex' from any GNU archive site."
+    rm -f lex.yy.c
+    if test $# -ne 1; then
+        eval LASTARG=\${$#}
+	case $LASTARG in
+	*.l)
+	    SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'`
+	    if test -f "$SRCFILE"; then
+	         cp "$SRCFILE" lex.yy.c
+	    fi
+	  ;;
+	esac
+    fi
+    if test ! -f lex.yy.c; then
+	echo 'main() { return 0; }' >lex.yy.c
+    fi
+    ;;
+
+  help2man*)
+    echo 1>&2 "\
+WARNING: \`$1' is $msg.  You should only need it if
+	 you modified a dependency of a manual page.  You may need the
+	 \`Help2man' package in order for those modifications to take
+	 effect.  You can get \`Help2man' from any GNU archive site."
+
+    file=`echo "$*" | sed -n "$sed_output"`
+    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
+    if test -f "$file"; then
+	touch $file
+    else
+	test -z "$file" || exec >$file
+	echo ".ab help2man is required to generate this page"
+	exit $?
+    fi
+    ;;
+
+  makeinfo*)
+    echo 1>&2 "\
+WARNING: \`$1' is $msg.  You should only need it if
+         you modified a \`.texi' or \`.texinfo' file, or any other file
+         indirectly affecting the aspect of the manual.  The spurious
+         call might also be the consequence of using a buggy \`make' (AIX,
+         DU, IRIX).  You might want to install the \`Texinfo' package or
+         the \`GNU make' package.  Grab either from any GNU archive site."
+    # The file to touch is that specified with -o ...
+    file=`echo "$*" | sed -n "$sed_output"`
+    test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
+    if test -z "$file"; then
+      # ... or it is the one specified with @setfilename ...
+      infile=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'`
+      file=`sed -n '
+	/^@setfilename/{
+	  s/.* \([^ ]*\) *$/\1/
+	  p
+	  q
+	}' $infile`
+      # ... or it is derived from the source name (dir/f.texi becomes f.info)
+      test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info
+    fi
+    # If the file does not exist, the user really needs makeinfo;
+    # let's fail without touching anything.
+    test -f $file || exit 1
+    touch $file
+    ;;
+
+  *)
+    echo 1>&2 "\
+WARNING: \`$1' is needed, and is $msg.
+         You might have modified some files without having the
+         proper tools for further handling them.  Check the \`README' file,
+         it often tells you about the needed prerequisites for installing
+         this package.  You may also peek at any GNU archive site, in case
+         some other package would contain this missing \`$1' program."
+    exit 1
+    ;;
+esac
+
+exit 0
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:

config/ylwrap

@@ -0,0 +1,226 @@
+#! /bin/sh
+# ylwrap - wrapper for lex/yacc invocations.
+
+scriptversion=2011-08-25.18; # UTC
+
+# Copyright (C) 1996, 1997, 1998, 1999, 2001, 2002, 2003, 2004, 2005,
+# 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
+#
+# Written by Tom Tromey <tromey@cygnus.com>.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# This file is maintained in Automake, please report
+# bugs to <bug-automake@gnu.org> or send patches to
+# <automake-patches@gnu.org>.
+
+case "$1" in
+  '')
+    echo "$0: No files given.  Try \`$0 --help' for more information." 1>&2
+    exit 1
+    ;;
+  --basedir)
+    basedir=$2
+    shift 2
+    ;;
+  -h|--h*)
+    cat <<\EOF
+Usage: ylwrap [--help|--version] INPUT [OUTPUT DESIRED]... -- PROGRAM [ARGS]...
+
+Wrapper for lex/yacc invocations, renaming files as desired.
+
+  INPUT is the input file
+  OUTPUT is one file PROG generates
+  DESIRED is the file we actually want instead of OUTPUT
+  PROGRAM is program to run
+  ARGS are passed to PROG
+
+Any number of OUTPUT,DESIRED pairs may be used.
+
+Report bugs to <bug-automake@gnu.org>.
+EOF
+    exit $?
+    ;;
+  -v|--v*)
+    echo "ylwrap $scriptversion"
+    exit $?
+    ;;
+esac
+
+
+# The input.
+input="$1"
+shift
+case "$input" in
+  [\\/]* | ?:[\\/]*)
+    # Absolute path; do nothing.
+    ;;
+  *)
+    # Relative path.  Make it absolute.
+    input="`pwd`/$input"
+    ;;
+esac
+
+pairlist=
+while test "$#" -ne 0; do
+  if test "$1" = "--"; then
+    shift
+    break
+  fi
+  pairlist="$pairlist $1"
+  shift
+done
+
+# The program to run.
+prog="$1"
+shift
+# Make any relative path in $prog absolute.
+case "$prog" in
+  [\\/]* | ?:[\\/]*) ;;
+  *[\\/]*) prog="`pwd`/$prog" ;;
+esac
+
+# FIXME: add hostname here for parallel makes that run commands on
+# other machines.  But that might take us over the 14-char limit.
+dirname=ylwrap$$
+do_exit="cd '`pwd`' && rm -rf $dirname > /dev/null 2>&1;"' (exit $ret); exit $ret'
+trap "ret=129; $do_exit" 1
+trap "ret=130; $do_exit" 2
+trap "ret=141; $do_exit" 13
+trap "ret=143; $do_exit" 15
+mkdir $dirname || exit 1
+
+cd $dirname
+
+case $# in
+  0) "$prog" "$input" ;;
+  *) "$prog" "$@" "$input" ;;
+esac
+ret=$?
+
+if test $ret -eq 0; then
+  set X $pairlist
+  shift
+  first=yes
+  # Since DOS filename conventions don't allow two dots,
+  # the DOS version of Bison writes out y_tab.c instead of y.tab.c
+  # and y_tab.h instead of y.tab.h. Test to see if this is the case.
+  y_tab_nodot="no"
+  if test -f y_tab.c || test -f y_tab.h; then
+    y_tab_nodot="yes"
+  fi
+
+  # The directory holding the input.
+  input_dir=`echo "$input" | sed -e 's,\([\\/]\)[^\\/]*$,\1,'`
+  # Quote $INPUT_DIR so we can use it in a regexp.
+  # FIXME: really we should care about more than `.' and `\'.
+  input_rx=`echo "$input_dir" | sed 's,\\\\,\\\\\\\\,g;s,\\.,\\\\.,g'`
+
+  while test "$#" -ne 0; do
+    from="$1"
+    # Handle y_tab.c and y_tab.h output by DOS
+    if test $y_tab_nodot = "yes"; then
+      if test $from = "y.tab.c"; then
+        from="y_tab.c"
+      else
+        if test $from = "y.tab.h"; then
+          from="y_tab.h"
+        fi
+      fi
+    fi
+    if test -f "$from"; then
+      # If $2 is an absolute path name, then just use that,
+      # otherwise prepend `../'.
+      case "$2" in
+        [\\/]* | ?:[\\/]*) target="$2";;
+        *) target="../$2";;
+      esac
+
+      # We do not want to overwrite a header file if it hasn't
+      # changed.  This avoid useless recompilations.  However the
+      # parser itself (the first file) should always be updated,
+      # because it is the destination of the .y.c rule in the
+      # Makefile.  Divert the output of all other files to a temporary
+      # file so we can compare them to existing versions.
+      if test $first = no; then
+        realtarget="$target"
+        target="tmp-`echo $target | sed s/.*[\\/]//g`"
+      fi
+      # Edit out `#line' or `#' directives.
+      #
+      # We don't want the resulting debug information to point at
+      # an absolute srcdir; it is better for it to just mention the
+      # .y file with no path.
+      #
+      # We want to use the real output file name, not yy.lex.c for
+      # instance.
+      #
+      # We want the include guards to be adjusted too.
+      FROM=`echo "$from" | sed \
+            -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'\
+            -e 's/[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]/_/g'`
+      TARGET=`echo "$2" | sed \
+            -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'\
+            -e 's/[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]/_/g'`
+
+      sed -e "/^#/!b" -e "s,$input_rx,," -e "s,$from,$2," \
+          -e "s,$FROM,$TARGET," "$from" >"$target" || ret=$?
+
+      # Check whether header files must be updated.
+      if test $first = no; then
+        if test -f "$realtarget" && cmp -s "$realtarget" "$target"; then
+          echo "$2" is unchanged
+          rm -f "$target"
+        else
+          echo updating "$2"
+          mv -f "$target" "$realtarget"
+        fi
+      fi
+    else
+      # A missing file is only an error for the first file.  This
+      # is a blatant hack to let us support using "yacc -d".  If -d
+      # is not specified, we don't want an error when the header
+      # file is "missing".
+      if test $first = yes; then
+        ret=1
+      fi
+    fi
+    shift
+    shift
+    first=no
+  done
+else
+  ret=$?
+fi
+
+# Remove the directory.
+cd ..
+rm -rf $dirname
+
+exit $ret
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:

configure.ac

@@ -0,0 +1,66 @@
+AC_INIT([jq], [1.2], [mu@netsoc.tcd.ie],
+             [jq], [http://stedolan.github.com/jq/])
+
+dnl Created autoconf implementation thompson@dtosolutions, 26NOV12
+AC_PREREQ([2.61])
+AC_CONFIG_AUX_DIR([config])
+AM_INIT_AUTOMAKE([parallel-tests foreign])
+AM_SILENT_RULES([yes])
+AC_PROG_CC
+AC_PROG_CC_STDC
+AC_PROG_CPP_WERROR
+AC_PROG_YACC
+
+AM_PROG_CC_C_O
+
+dnl couldn't use AM_PROG_LEX as it doesn't support header files like the
+dnl AC_PROG_YACC macros...
+
+dnl
+dnl these program checks should probably be deleted
+dnl
+
+AC_PROG_LEX
+if test "x$LEX" != xflex; then
+  LEX="$SHELL $missing_dir/missing flex"
+fi
+
+
+dnl Check for valgrind
+AC_CHECK_PROGS(valgrind_cmd, valgrind)
+if test "x$valgrind_cmd" = "x" ; then
+    AC_MSG_WARN([valgrind is required to test jq.])
+fi
+
+
+dnl Don't attempt to build docs if there's no Ruby lying around
+AC_ARG_ENABLE([docs],
+   AC_HELP_STRING([--disable-docs], [don't build docs]))
+
+AS_IF([test "x$enable_docs" != "xno"],[
+   AC_CHECK_PROGS(bundle_cmd, bundle)
+   AS_IF([test "x$bundle_cmd" = "x" || \
+      ! ( cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null )],[
+         AC_MSG_WARN([no bundler])
+         cat <<EOF
+*****************************************************************
+*  Ruby dependencies for building jq documentation not found.   *
+*  You can still build, install and hack on jq, but the manpage *
+*  will not be rebuilt and some of the tests won't run.         *
+*  See docs/README.md for how to install the docs dependencies. *
+*****************************************************************
+EOF
+         enable_docs=no
+   ])
+])
+
+AM_CONDITIONAL([ENABLE_DOCS], [test "x$enable_docs" != xno])
+AC_SUBST([BUNDLER], ["$bundle_cmd"])
+
+dnl AC_CONFIG_MACRO_DIR([m4])
+AC_CONFIG_HEADERS(config.h)
+AM_INIT_AUTOMAKE([-Wall])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+
+

docs/Gemfile

@@ -1,5 +1,6 @@
-source :rubygems
+source "https://rubygems.org"
 
+gem "rake"
 gem "bonsai"
 gem "maruku"
 gem "ronn"

docs/Gemfile.lock

@@ -1,11 +1,15 @@
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
-    activesupport (3.2.9)
+    POpen4 (0.1.4)
+      Platform (>= 0.4.0)
+      open4
+    Platform (0.4.0)
+    activesupport (3.2.12)
       i18n (~> 0.6)
       multi_json (~> 1.0)
-    addressable (2.3.2)
-    bonsai (1.4.7)
+    addressable (2.3.4)
+    bonsai (1.4.8)
       activesupport (>= 3.0.3)
       builder (>= 3.0.0)
       i18n (>= 0.5.0)
@@ -17,32 +21,37 @@ GEM
       sinatra (>= 1.0)
       tilt (>= 1.3)
       watch (>= 0.1.0)
-    builder (3.1.4)
+      yui-compressor
+    builder (3.2.0)
     hpricot (0.8.6)
-    i18n (0.6.1)
-    launchy (2.1.2)
+    i18n (0.6.4)
+    launchy (2.3.0)
       addressable (~> 2.3)
-    liquid (2.4.1)
+    liquid (2.5.0)
     maruku (0.6.1)
       syntax (>= 1.0.0)
-    multi_json (1.5.0)
+    multi_json (1.7.3)
     mustache (0.99.4)
-    rack (1.4.1)
-    rack-protection (1.3.2)
+    open4 (1.3.0)
+    rack (1.5.2)
+    rack-protection (1.5.0)
       rack
-    rdiscount (1.6.8)
+    rake (10.0.3)
+    rdiscount (2.0.7.2)
     ronn (0.7.3)
       hpricot (>= 0.8.2)
       mustache (>= 0.7.0)
       rdiscount (>= 1.5.8)
-    sass (3.2.3)
-    sinatra (1.3.3)
-      rack (~> 1.3, >= 1.3.6)
-      rack-protection (~> 1.2)
-      tilt (~> 1.3, >= 1.3.3)
+    sass (3.2.8)
+    sinatra (1.4.2)
+      rack (~> 1.5, >= 1.5.2)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
     syntax (1.0.0)
-    tilt (1.3.3)
+    tilt (1.4.0)
     watch (0.1.0)
+    yui-compressor (0.9.6)
+      POpen4 (>= 0.1.4)
 
 PLATFORMS
   ruby
@@ -50,4 +59,5 @@ PLATFORMS
 DEPENDENCIES
   bonsai
   maruku
+  rake
   ronn

docs/README.md

@@ -0,0 +1,25 @@
+Documentation
+=============
+
+The jq website, manpages and some of the tests are generated from this
+directory. The directory holds a [Bonsai](http://tinytree.info)
+website, and the manual is a YAML file in `content/3.manual`.
+
+To build the documentation (including building the jq manpage), you'll
+need a working Ruby setup. The easiest way to get one is to install
+RVM and Ruby 1.9.3 like so:
+
+    \curl -L https://get.rvm.io | bash -s stable --ruby=1.9.3
+
+After that finishes installing, you'll need to make sure RVM is on
+your path by doing `source $HOME/.rvm/scripts/rvm`, or just opening a
+new shell. See <http://rvm.io> for more info on RVM.
+
+Once RVM is installed, you can install all the dependencies for jq's
+documentation build by running this from the `docs` directory:
+
+    bundle install
+
+When bundle manages to install the dependencies, rerun `./configure`
+in the jq root directory and then the Makefile will be able to
+generate the jq manpage.

docs/Rakefile

@@ -1,9 +1,11 @@
+require 'yaml'
 require 'bonsai'
 require 'liquid'
 require 'maruku'
 require 'json'
 require 'ronn'
 require 'tempfile'
+require 'yaml'
 
 module ExtraFilters
   def markdownify(input)
@@ -46,7 +48,7 @@ task :serve do
         run Bonsai::DevelopmentServer
       }
       Rack::Handler.default.run(app, :Port => 5000) do
-        Launchy.open("http://localhost:5000/jq")
+        Launchy.open("http://localhost:5000/jq/")
       end
     }    
     Watch.new("{content,templates,public}/**/*") { Bonsai::Exporter.process! }
@@ -62,6 +64,28 @@ task :build do
   Bonsai::Exporter.publish!
 end
 
+$BINARIES = {
+  :osx32 => "--host=i686-apple-darwin10 CFLAGS='-m32 -g -O'",
+  :osx64 => "--host=i686-apple-darwin10 CFLAGS='-m64 -g -O'",
+  :win32 => "--host=i686-w64-mingw32 CFLAGS='-g -O'",
+  :win64 => "--host=x86_64-w64-mingw32 CFLAGS='-g -O'",
+  :linux32 => "--host=x86_64-linux-gnu CFLAGS='-m32 -g -O'",
+  :linux64 => "--host=x86_64-linux-gnu CFLAGS='-m64 -g -O'"
+}
+
+$BINARIES.each do |name, args|
+  file "../build/#{name}" do |t|
+    sh "MAKEFLAGS=-j4 ../scripts/crosscompile #{name} #{args}"
+  end
+  task :binaries => ["../build/#{name}"]
+end
+
+task :binaries => ["output/download"] do
+  $BINARIES.each do |name, args|
+    FileUtils.cp_r "../build/#{name}", "output/download/"
+  end
+end
+
 def load_manual
   YAML::ENGINE.yamler = 'syck'
   YAML::load(File.open("content/3.manual/manual.yml"))
@@ -96,6 +120,10 @@ task :manpage do
   end  
 end
 
+task :manpage_default => ["default_manpage.md"] do
+  puts Ronn::Document.new("default_manpage.md").convert('roff').gsub(/<\/?code>/,"")
+end
+
 task :mantests do
   load_manual['sections'].each do |section|
     (section['entries'] || []).each do |entry|
@@ -108,3 +136,11 @@ task :mantests do
     end
   end
 end
+
+directory "output/download/source"
+task :tarball => ["output/download/source"] do
+  sh "cd ..; ./configure && make dist && make distclean"
+  sh "cp ../jq-*.tar.gz output/download/source"
+end
+
+task :dist => [:build, :binaries, :tarball]

docs/content/1.tutorial/default.yml

@@ -237,5 +237,5 @@ body:
       - - -
 
       Here endeth the tutorial! There's lots more to play with, go
-      read [the manual](../manual) if you're interested, and [download
-      jq](../download) if you haven't already.
+      read [the manual](../manual/) if you're interested, and [download
+      jq](../download/) if you haven't already.

docs/content/2.download/default.yml

@@ -38,7 +38,7 @@ body:
 
       ### Windows
 
-       * Executables for [64-bit](win64/jq) or [32-bit](win32/jq)
+       * Executables for [64-bit](win64/jq.exe) or [32-bit](win32/jq.exe)
 
       ### From source on Linux or OS X
 

docs/content/3.manual/manual.yml

@@ -75,51 +75,57 @@ sections:
       You can affect how jq reads and writes its input and output
       using some command-line options:
 
-        * `--slurp`/`-s`:
+      * `--slurp`/`-s`:
   
-          Instead of running the filter for each JSON object in the
-          input, read the entire input stream into a large array and run
-          the filter just once.
-        
-        * `--raw-input`/`-R`:
-        
-          Don't parse the input as JSON. Instead, each line of text is
-          passed to the filter as a string. If combined with `--slurp`,
-          then the entire input is passed to the filter as a single long
-          string.
+        Instead of running the filter for each JSON object in the
+        input, read the entire input stream into a large array and run
+        the filter just once.
+      
+      * `--raw-input`/`-R`:
+      
+        Don't parse the input as JSON. Instead, each line of text is
+        passed to the filter as a string. If combined with `--slurp`,
+        then the entire input is passed to the filter as a single long
+        string.
   
-        * `--null-input`/`-n`:
+      * `--null-input`/`-n`:
   
-          Don't read any input at all! Instead, the filter is run once
-          using `null` as the input. This is useful when using jq as a
-          simple calculator or to construct JSON data from scratch.
+        Don't read any input at all! Instead, the filter is run once
+        using `null` as the input. This is useful when using jq as a
+        simple calculator or to construct JSON data from scratch.
   
-        * `--compact-output` / `-c`:
+      * `--compact-output` / `-c`:
   
-          By default, jq pretty-prints JSON output. Using this option
-          will result in more compact output by instead putting each
-          JSON object on a single line.
+        By default, jq pretty-prints JSON output. Using this option
+        will result in more compact output by instead putting each
+        JSON object on a single line.
   
-        * `--colour-output` / `-C` and `--monochrome-output` / `-M`:
-        
-          By default, jq outputs colored JSON if writing to a
-          terminal. You can force it to produce color even if writing to
-          a pipe or a file using `-C`, and disable color with `-M`.
+      * `--colour-output` / `-C` and `--monochrome-output` / `-M`:
+      
+        By default, jq outputs colored JSON if writing to a
+        terminal. You can force it to produce color even if writing to
+        a pipe or a file using `-C`, and disable color with `-M`.
   
-        * `--ascii-output` / `-a`:
+      * `--ascii-output` / `-a`:
   
-          jq usually outputs non-ASCII Unicode codepoints as UTF-8, even
-          if the input specified them as escape sequences (like
-          "\u03bc"). Using this option, you can force jq to produce pure
-          ASCII output with every non-ASCII character replaced with the
-          equivalent escape sequence.
+        jq usually outputs non-ASCII Unicode codepoints as UTF-8, even
+        if the input specified them as escape sequences (like
+        "\u03bc"). Using this option, you can force jq to produce pure
+        ASCII output with every non-ASCII character replaced with the
+        equivalent escape sequence.
   
-        * `--raw-output` / `-r`:
+      * `--raw-output` / `-r`:
   
-          With this option, if the filter's result is a string then it
-          will be written directly to standard output rather than being
-          formatted as a JSON string with quotes. This can be useful for
-          making jq filters talk to non-JSON-based systems.
+        With this option, if the filter's result is a string then it
+        will be written directly to standard output rather than being
+        formatted as a JSON string with quotes. This can be useful for
+        making jq filters talk to non-JSON-based systems.
+
+      * `--arg name value`:
+
+        This option passes a value to the jq program as a predefined
+        variable. If you run jq with `--arg foo bar`, then `$foo` is
+        available in the program and has the value `"bar"`.
 
   - title: Basic filters
     entries:
@@ -144,7 +150,7 @@ sections:
           
           The simplest *useful* filter is .foo. When given a
           JSON object (aka dictionary or hash) as input, it produces
-          the value at the key "foo", or null if there\'s none present.
+          the value at the key "foo", or null if there's none present.
 
         examples:
           - program: '.foo'
@@ -160,7 +166,7 @@ sections:
           You can also look up fields of an object using syntax like
           `.["foo"]` (.foo above is a shorthand version of this). This
           one works for arrays as well, if the key is an
-          integer. Arrays are zero-based (like javascript), so .[2]
+          integer. Arrays are zero-based (like javascript), so `.[2]`
           returns the third element of the array.
 
         examples:
@@ -178,9 +184,12 @@ sections:
           If you use the `.[foo]` syntax, but omit the index
           entirely, it will return *all* of the elements of an
           array. Running `.[]` with the input `[1,2,3]` will produce the
-          numbers as three seperate results, rather than as a single
+          numbers as three separate results, rather than as a single
           array.
 
+          You can also use this on an object, and it will return all
+          the values of the object.
+
         examples:
           - program: '.[]'
             input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
@@ -192,6 +201,10 @@ sections:
             input: '[]'
             output: []
 
+          - program: '.[]'
+            input: '{"a": 1, "b": 2}'
+            output: ['[1, 2]']
+
       - title: "`,`"
         body: |
           
@@ -218,8 +231,8 @@ sections:
       - title: "`|`"
         body: |
           The | operator combines two filters by feeding the output(s) of
-          the one on the left into the input of the one on the right. It\'s
-          pretty much the same as the Unix shell\'s pipe, if you\'re used to
+          the one on the left into the input of the one on the right. It's
+          pretty much the same as the Unix shell's pipe, if you're used to
           that. 
 
           If the one on the left produces multiple results, the one on
@@ -257,8 +270,8 @@ sections:
           in `[.foo, .bar, .baz]`) or to "collect" all the results of a
           filter into an array (as in `[.items[].name]`)
         
-          Once you understand the "," operator, you can look at jq\'s array
-          syntax in a different light: the expression [1,2,3] is not using a
+          Once you understand the "," operator, you can look at jq's array
+          syntax in a different light: the expression `[1,2,3]` is not using a
           built-in syntax for comma-separated arrays, but is instead applying
           the `[]` operator (collect results) to the expression 1,2,3 (which
           produces three different results).
@@ -279,8 +292,8 @@ sections:
           
           If the keys are "sensible" (all alphabetic characters), then
           the quotes can be left off. The value can be any expression
-          (although you may need to wrap it in parentheses if it\'s a
-          complicated one), which gets applied to the {} expression\'s
+          (although you may need to wrap it in parentheses if it's a
+          complicated one), which gets applied to the {} expression's
           input (remember, all filters have an input and an
           output).
           
@@ -294,10 +307,10 @@ sections:
           
               {user: .user, title: .title}
           
-          Because that\'s so common, there\'s a shortcut syntax: `{user, title}`.
+          Because that's so common, there's a shortcut syntax: `{user, title}`.
           
           If one of the expressions produces multiple results,
-          multiple dictionaries will be produced. If the input\'s
+          multiple dictionaries will be produced. If the input's
           
               {"user":"stedolan","titles":["JQ Primer", "More JQ"]}
           
@@ -356,6 +369,9 @@ sections:
               the key-value pairs from both objects into a single
               combined object. If both objects contain a value for the
               same key, the object on the right of the `+` wins.
+          
+          `null` can be added to any value, and returns the other
+          value unchanged.
 
         examples:
           - program: '.a + 1'
@@ -364,6 +380,12 @@ sections:
           - program: '.a + .b'
             input: '{"a": [1,2], "b": [3,4]}'
             output: ['[1,2,3,4]']
+          - program: '.a + null'
+            input: '{"a": 1}'
+            output: ['1']
+          - program: '.a + 1'
+            input: '{}'
+            output: ['1']
           - program: '{a: 1} + {b: 2} + {c: 3} + {a: 42}'
             input: 'null'
             output: ['{"a": 42, "b": 2, "c": 3}']
@@ -437,6 +459,25 @@ sections:
             input: '[42,3,35]'
             output: ['[0,1,2]']
 
+      - title: `has`
+        body: |
+          
+          The builtin function `has` returns whether the input object
+          has the given key, or the input array has an element at the
+          given index.
+
+          `has($key)` has the same effect as checking whether `$key`
+          is a member of the array returned by `keys`, although `has`
+          will be faster.
+
+        examples:
+          - program: 'map(has("foo"))'
+            input: '[{"foo": 42}, {}]'
+            output: ['[true, false]']
+          - program: 'map(has(2))'
+            input: '[[0,1], ["a","b","c"]]'
+            output: ['[false, true]']
+
       - title: `select`
         body: |
           
@@ -528,6 +569,18 @@ sections:
             input: '[1, "1", [1]]'
             output: ['"1"', '"1"', '"[1]"']
 
+      - title: `type`
+        body: |
+          
+          The `type` function returns the type of its argument as a
+          string, which is one of null, boolean, number, string, array
+          or object.
+
+        examples:
+          - program: 'map(type)'
+            input: '[0, false, [], {}, null, "hello"]'
+            output: ['["number", "boolean", "array", "object", "null", "string"]']
+
       - title: `sort, sort_by`
         body: |
           
@@ -634,6 +687,38 @@ sections:
             input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
             output: ['false']
 
+      - title: `recurse`
+        body: |
+          
+          The `recurse` function allows you to search through a
+          recursive structure, and extract interesting data from all
+          levels. Suppose your input represents a filesystem:
+
+              {"name": "/", "children": [
+                {"name": "/bin", "children": [
+                  {"name": "/bin/ls", "children": []},
+                  {"name": "/bin/sh", "children": []}]},
+                {"name": "/home", "children": [
+                  {"name": "/home/stephen", "children": [
+                    {"name": "/home/stephen/jq", "children": []}]}]}]}
+          
+          Now suppose you want to extract all of the filenames
+          present. You need to retrieve `.name`, `.children[].name`,
+          `.children[].children[].name`, and so on. You can do this
+          with:
+
+              recurse(.children[]) | .name
+
+        examples:
+          - program: 'recurse(.foo[])'
+            input: '{"foo":[{"foo": []}, {"foo":[{"foo":[]}]}]}'
+            output: 
+              - '{"foo":[{"foo":[]},{"foo":[{"foo":[]}]}]}'
+              - '{"foo":[]}'
+              - '{"foo":[{"foo":[]}]}'
+              - '{"foo":[]}'
+
+
       - title: "String interpolation - `\(foo)`"
         body: |
 
@@ -646,10 +731,77 @@ sections:
             input: '42'
             output: ['"The input was 42, which is one less than 43"']
           
-              
+      - title: "Format strings and escaping"
+        body: |
 
-              
+          The `@foo` syntax is used to format and escape strings,
+          which is useful for building URLs, documents in a language
+          like HTML or XML, and so forth. `@foo` can be used as a
+          filter on its own, the possible escapings are:
 
+          * `@text`:
+
+            Calls `tostring`, see that function for details.
+
+          * `@json`:
+
+            Serialises the input as JSON.
+
+          * `@html`:
+
+            Applies HTML/XML escaping, by mapping the characters
+            `<>&'"` to their entity equivalents `&lt;`, `&gt;`,
+            `&amp;`, `&apos;`, `&quot;`.
+
+          * `@uri`:
+
+            Applies percent-encoding, by mapping all reserved URI
+            characters to a `%xx` sequence.
+
+          * `@csv`:
+            
+            The input must be an array, and it is rendered as CSV
+            with double quotes for strings, and quotes escaped by
+            repetition.
+
+          * `@sh`:
+            
+            The input is escaped suitable for use in a command-line
+            for a POSIX shell. If the input is an array, the output
+            will be a series of space-separated strings.
+            
+          * `@base64`:
+
+            The input is converted to base64 as specified by RFC 4648.
+
+          This syntax can be combined with string interpolation in a
+          useful way. You can follow a `@foo` token with a string
+          literal. The contents of the string literal will *not* be
+          escaped. However, all interpolations made inside that string
+          literal will be escaped. For instance,
+
+              @uri "http://www.google.com/search?q=\(.search)"
+
+          will produce the following output for the input
+          `{"search":"jq!"}`:
+
+              http://www.google.com/search?q=jq%21
+
+          Note that the slashes, question mark, etc. in the URL are
+          not escaped, as they were part of the string literal.
+
+        examples:
+          - program: '@html'
+            input: '"This works if x < y"'
+            output: ['"This works if x &lt; y"']
+
+#          - program: '@html "<span>Anonymous said: \(.)</span>"'
+#            input: '"<script>alert(\"lol hax\");</script>"'
+#            output: ["<span>Anonymous said: &lt;script&gt;alert(&quot;lol hax&quot;);&lt;/script&gt;</span>"]
+
+          - program: '@sh "echo \(.)"'
+            input: "\"O'Hara's Ale\""
+            output: ["\"echo 'O'\\''Hara'\\''s Ale\""]
         
   - title: Conditionals and Comparisons
     entries:
@@ -681,8 +833,8 @@ sections:
           means that you'll sometimes have to be more explicit about
           the condition you want: you can't test whether, e.g. a
           string is empty using `if .name then A else B end`, you'll
-          need something more like 'if (.name | count) > 0 then A else
-          B end' instead.
+          need something more like `if (.name | count) > 0 then A else
+          B end` instead.
           
           If the condition A produces multiple results, it is
           considered "true" if any of those results is not false or
@@ -774,7 +926,7 @@ sections:
             input: '{}'
             output: [42]
           
-  - title: Variables and Functions
+  - title: Advanced features
     body: |
       Variables are an absolute necessity in most programming languages, but
       they're relegated to an "advanced feature" in jq.
@@ -791,6 +943,10 @@ sections:
       (many jq functions such as `map` and `find` are in fact written
       in jq).
 
+      Finally, jq has a `fold` operation, which is very powerful but a
+      bit tricky. Again, it's mostly used internally, to define some
+      useful bits of jq's standard library.
+
     entries:
       - title: Variables
         body: |
@@ -805,8 +961,8 @@ sections:
           For instance, calculating the average value of an array of numbers
           requires a few variables in most languages - at least one to hold the
           array, perhaps one for each element or for a loop counter. In jq, it's
-          simply `add / length` - the `sum` expression is given the array and
-          produces its sum, and the `count` expression is given the array and
+          simply `add / length` - the `add` expression is given the array and
+          produces its sum, and the `length` expression is given the array and
           produces its length.
           
           So, there's generally a cleaner way to solve most problems in jq that
@@ -841,10 +997,11 @@ sections:
           
               .realnames as $names | .posts[] | {title, author: $names[.author]}
           
-          The expression "foo as $x" runs foo, puts the result in $x,
-          and returns the original input. Apart from the side-effect
-          of binding the variable, it has the same effect as ".".
-          
+          The expression `exp as $x | ...` means: for each value of expression
+          `exp`, run the rest of the pipeline with the entire original input, and
+          with `$x` set to that value.  Thus `as` functions as something of a 
+          foreach loop.
+
           Variables are scoped over the rest of the expression that defines
           them, so 
           
@@ -895,6 +1052,39 @@ sections:
           - program: 'def addvalue(f): f as $x | map(. + $x); addvalue(.[0])'
             input: '[[1,2],[10,20]]'
             output: ['[[1,2,1,2], [10,20,1,2]]']
+
+      - title: Fold
+        body: |
+        
+          The `fold` syntax in jq allows you to combine all of the
+          results of an expression by accumulating them into a single
+          answer. As an example, we'll pass `[3,2,1]` to this expression:
+
+              fold 0 as $sum (.[] | $sum + .)
+          
+          The variable `$sum` is first given the value `0`. The body
+          of the fold (i.e. `.[] | $sum + .`) is evaluated. `.[]`
+          produces three results, `3`, `2`, and `1`. For the first
+          one, `$sum + .` gives `3`.
+
+          Having produced this answer, jq backtracks to find the next
+          result as per usual. However, this time, `$sum` is set to
+          the previous value of the body, so `$sum + .` gives
+          `5`. After the final backtracking, `$sum + .` gives
+          `6`. This final value is used as the value of the entire
+          `fold` expression, so the above filter returns `6`.
+
+          More formally, in order to evaluate `fold INIT as $VAR
+          (BODY)`, jq first sets `$VAR` to the value of `INIT`. It
+          then runs through `BODY`. Each time `BODY` produces a value,
+          `$VAR` is set to that value and jq backtracks to find the
+          next one. When `BODY` stops producing values, the final
+          value of `$VAR` is the result of the entire expression.
+
+        examples:
+          - program: 'fold 0 as $sum (.[] | $sum + .)'
+            input: '[10,2,5,3]'
+            output: ['20']
       
 
   - title: Assignment

docs/content/index/index.yml

@@ -23,5 +23,5 @@ body3: |
   shorter and simpler than you'd expect.
   
 tail: |
-  Go read the [tutorial](tutorial) for more, or the [manual](manual)
+  Go read the [tutorial](tutorial/) for more, or the [manual](manual/)
   for *way* more.

docs/default_manpage.md

@@ -0,0 +1,22 @@
+jq(1) -- Command-line JSON processor
+====================================
+
+## DESCRIPTION
+
+`jq` can transform JSON in various ways, by selecting, iterating,
+reducing and otherwise mangling JSON documents. 
+
+This version of `jq` was built without a manual, so this manpage is a
+stub. For full documentation of the `jq` language, see:
+
+    http://stedolan.github.com/jq
+    
+## BUGS
+
+Presumably. Report them or discuss them at:
+
+    https://github.com/stedolan/jq/issues
+
+## AUTHOR
+
+Stephen Dolan `<mu@netsoc.tcd.ie>`

docs/site.yml

@@ -1,7 +1,11 @@
 # The key value pairs found below are available within the templates.
 
+:url: http://stedolan.github.io/jq
 
+# This line is modified by the Makefile. To change the version number,
+# edit the Autoconf version number at the top of configure.ac
 jq_version: 1.2
+
 root: '/jq'
 footer: |
   
@@ -11,4 +15,4 @@ footer: |
 
   jq is licensed under the MIT license (code) and the
   [CC-BY-3.0](http://creativecommons.org/licenses/by/3.0/) license
-  (docs).
+  (docs).

docs/templates/index.liquid

@@ -18,7 +18,7 @@
               <ul class="dropdown-menu">
                 <li><a href="{{root}}/download/linux64/jq">Linux (64-bit)</a></li>
                 <li><a href="{{root}}/download/osx64/jq">OS X (64-bit)</a></li>
-                <li><a href="{{root}}/download">Other platforms and source</a></li>
+                <li><a href="{{root}}/download/">Other platforms and source</a></li>
               </ul>
             </div>
 

execute.c

@@ -12,6 +12,7 @@
 #include "frame_layout.h"
 
 #include "jv_alloc.h"
+#include "jq_parser.h"
 #include "locfile.h"
 #include "jv.h"
 #include "jv_aux.h"
@@ -23,24 +24,29 @@ typedef struct {
   int pathidx;
 } stackval;
 
+struct jq_state {
+  struct forkable_stack data_stk;
+  struct forkable_stack frame_stk;
+  struct forkable_stack fork_stk;
+  jv* pathbuf;
+  int pathsize; // number of allocated elements
+  int debug_trace_enabled;
+};
 
-jv* pathbuf;
-int pathsize; // number of allocated elements
-
-int path_push(stackval sv, jv val) {
+int path_push(jq_state *jq, stackval sv, jv val) {
   int pos = sv.pathidx;
-  assert(pos <= pathsize);
+  assert(pos <= jq->pathsize);
   assert(pos >= 0);
-  if (pos == pathsize) {
-    int oldpathsize = pathsize;
-    pathsize = oldpathsize ? oldpathsize * 2 : 100;
-    pathbuf = jv_mem_realloc(pathbuf, sizeof(pathbuf[0]) * pathsize);
-    for (int i=oldpathsize; i<pathsize; i++) {
-      pathbuf[i] = jv_invalid();
+  if (pos == jq->pathsize) {
+    int oldpathsize = jq->pathsize;
+    jq->pathsize = oldpathsize ? oldpathsize * 2 : 100;
+    jq->pathbuf = jv_mem_realloc(jq->pathbuf, sizeof(jq->pathbuf[0]) * jq->pathsize);
+    for (int i=oldpathsize; i<jq->pathsize; i++) {
+      jq->pathbuf[i] = jv_invalid();
     }
   }
-  jv_free(pathbuf[pos]);
-  pathbuf[pos] = val;
+  jv_free(jq->pathbuf[pos]);
+  jq->pathbuf[pos] = val;
   return pos + 1;
 }
 
@@ -57,30 +63,28 @@ stackval stackval_root(jv v) {
   return s;
 }
 
-struct forkable_stack data_stk;
 typedef struct {
   FORKABLE_STACK_HEADER;
   stackval sv;
 } data_stk_elem;
 
-void stack_push(stackval val) {
+void stack_push(jq_state *jq, stackval val) {
   assert(jv_is_valid(val.value));
-  data_stk_elem* s = forkable_stack_push(&data_stk, sizeof(data_stk_elem));
+  data_stk_elem* s = forkable_stack_push(&jq->data_stk, sizeof(data_stk_elem));
   s->sv = val;
 }
 
-stackval stack_pop() {
-  data_stk_elem* s = forkable_stack_peek(&data_stk);
+stackval stack_pop(jq_state *jq) {
+  data_stk_elem* s = forkable_stack_peek(&jq->data_stk);
   stackval sv = s->sv;
-  if (!forkable_stack_pop_will_free(&data_stk)) {
+  if (!forkable_stack_pop_will_free(&jq->data_stk)) {
     sv.value = jv_copy(sv.value);
   }
-  forkable_stack_pop(&data_stk);
+  forkable_stack_pop(&jq->data_stk);
   assert(jv_is_valid(sv.value));
   return sv;
 }
 
-struct forkable_stack frame_stk;
 
 
 struct forkpoint {
@@ -89,36 +93,35 @@ struct forkpoint {
   struct forkable_stack_state saved_call_stack;
 };
 
-struct forkable_stack fork_stk;
 
-void stack_save(){
-  struct forkpoint* fork = forkable_stack_push(&fork_stk, sizeof(struct forkpoint));
-  forkable_stack_save(&data_stk, &fork->saved_data_stack);
-  forkable_stack_save(&frame_stk, &fork->saved_call_stack);
+void stack_save(jq_state *jq){
+  struct forkpoint* fork = forkable_stack_push(&jq->fork_stk, sizeof(struct forkpoint));
+  forkable_stack_save(&jq->data_stk, &fork->saved_data_stack);
+  forkable_stack_save(&jq->frame_stk, &fork->saved_call_stack);
 }
 
-void stack_switch() {
-  struct forkpoint* fork = forkable_stack_peek(&fork_stk);
-  forkable_stack_switch(&data_stk, &fork->saved_data_stack);
-  forkable_stack_switch(&frame_stk, &fork->saved_call_stack);
+void stack_switch(jq_state *jq) {
+  struct forkpoint* fork = forkable_stack_peek(&jq->fork_stk);
+  forkable_stack_switch(&jq->data_stk, &fork->saved_data_stack);
+  forkable_stack_switch(&jq->frame_stk, &fork->saved_call_stack);
 }
 
-int stack_restore(){
-  while (!forkable_stack_empty(&data_stk) && 
-         forkable_stack_pop_will_free(&data_stk)) {
-    jv_free(stack_pop().value);
+int stack_restore(jq_state *jq){
+  while (!forkable_stack_empty(&jq->data_stk) && 
+         forkable_stack_pop_will_free(&jq->data_stk)) {
+    jv_free(stack_pop(jq).value);
   }
-  while (!forkable_stack_empty(&frame_stk) && 
-         forkable_stack_pop_will_free(&frame_stk)) {
-    frame_pop(&frame_stk);
+  while (!forkable_stack_empty(&jq->frame_stk) && 
+         forkable_stack_pop_will_free(&jq->frame_stk)) {
+    frame_pop(&jq->frame_stk);
   }
-  if (forkable_stack_empty(&fork_stk)) {
+  if (forkable_stack_empty(&jq->fork_stk)) {
     return 0;
   } else {
-    struct forkpoint* fork = forkable_stack_peek(&fork_stk);
-    forkable_stack_restore(&data_stk, &fork->saved_data_stack);
-    forkable_stack_restore(&frame_stk, &fork->saved_call_stack);
-    forkable_stack_pop(&fork_stk);
+    struct forkpoint* fork = forkable_stack_peek(&jq->fork_stk);
+    forkable_stack_restore(&jq->data_stk, &fork->saved_data_stack);
+    forkable_stack_restore(&jq->frame_stk, &fork->saved_call_stack);
+    forkable_stack_pop(&jq->fork_stk);
     return 1;
   }
 }
@@ -148,42 +151,42 @@ void print_error(jv value) {
 }
 #define ON_BACKTRACK(op) ((op)+NUM_OPCODES)
 
-jv jq_next() {
+jv jq_next(jq_state *jq) {
   jv cfunc_input[MAX_CFUNCTION_ARGS];
 
-  assert(!forkable_stack_empty(&frame_stk));
-  uint16_t* pc = *frame_current_retaddr(&frame_stk);
-  frame_pop(&frame_stk);
+  assert(!forkable_stack_empty(&jq->frame_stk));
+  uint16_t* pc = *frame_current_retaddr(&jq->frame_stk);
+  frame_pop(&jq->frame_stk);
   
-  assert(!forkable_stack_empty(&frame_stk));
+  assert(!forkable_stack_empty(&jq->frame_stk));
 
   int backtracking = 0;
   while (1) {
     uint16_t opcode = *pc;
 
-#if JQ_DEBUG
-    dump_operation(frame_current_bytecode(&frame_stk), pc);
-    printf("\t");
-    const struct opcode_description* opdesc = opcode_describe(opcode);
-    data_stk_elem* param;
-    int stack_in = opdesc->stack_in;
-    if (stack_in == -1) stack_in = pc[1];
-    for (int i=0; i<stack_in; i++) {
-      if (i == 0) {
-        param = forkable_stack_peek(&data_stk);
-      } else {
-        printf(" | ");
-        param = forkable_stack_peek_next(&data_stk, param);
+    if (jq->debug_trace_enabled) {
+      dump_operation(frame_current_bytecode(&jq->frame_stk), pc);
+      printf("\t");
+      const struct opcode_description* opdesc = opcode_describe(opcode);
+      data_stk_elem* param = 0;
+      int stack_in = opdesc->stack_in;
+      if (stack_in == -1) stack_in = pc[1];
+      for (int i=0; i<stack_in; i++) {
+        if (i == 0) {
+          param = forkable_stack_peek(&jq->data_stk);
+        } else {
+          printf(" | ");
+          param = forkable_stack_peek_next(&jq->data_stk, param);
+        }
+        if (!param) break;
+        jv_dump(jv_copy(param->sv.value), 0);
+        printf("<%d>", jv_get_refcnt(param->sv.value));
       }
-      if (!param) break;
-      jv_dump(jv_copy(param->sv.value), 0);
-      printf("<%d>", jv_get_refcnt(param->sv.value));
+
+      if (backtracking) printf("\t<backtracking>");
+
+      printf("\n");
     }
-
-    if (backtracking) printf("\t<backtracking>");
-
-    printf("\n");
-#endif
     if (backtracking) {
       opcode = ON_BACKTRACK(opcode);
       backtracking = 0;
@@ -194,64 +197,67 @@ jv jq_next() {
     default: assert(0 && "invalid instruction");
 
     case LOADK: {
-      jv v = jv_array_get(jv_copy(frame_current_bytecode(&frame_stk)->constants), *pc++);
+      jv v = jv_array_get(jv_copy(frame_current_bytecode(&jq->frame_stk)->constants), *pc++);
       assert(jv_is_valid(v));
-      stack_push(stackval_replace(stack_pop(), v));
+      stack_push(jq, stackval_replace(stack_pop(jq), v));
       break;
     }
 
     case DUP: {
-      stackval v = stack_pop();
+      stackval v = stack_pop(jq);
       stackval v2 = v;
       v2.value = jv_copy(v.value);
-      stack_push(v);
-      stack_push(v2);
+      stack_push(jq, v);
+      stack_push(jq, v2);
       break;
     }
 
     case DUP2: {
-      stackval keep = stack_pop();
-      stackval v = stack_pop();
+      stackval keep = stack_pop(jq);
+      stackval v = stack_pop(jq);
       stackval v2 = v;
       v2.value = jv_copy(v.value);
-      stack_push(v);
-      stack_push(keep);
-      stack_push(v2);
+      stack_push(jq, v);
+      stack_push(jq, keep);
+      stack_push(jq, v2);
       break;
     }
 
     case SWAP: {
-      stackval a = stack_pop();
-      stackval b = stack_pop();
-      stack_push(a);
-      stack_push(b);
+      stackval a = stack_pop(jq);
+      stackval b = stack_pop(jq);
+      stack_push(jq, a);
+      stack_push(jq, b);
       break;
     }
       
     case POP: {
-      jv_free(stack_pop().value);
+      jv_free(stack_pop(jq).value);
       break;
     }
 
     case APPEND: {
       // FIXME paths
-      jv v = stack_pop().value;
-      jv array = stack_pop().value;
-      array = jv_array_append(array, v);
-      stack_push(stackval_root(array));
+      jv v = stack_pop(jq).value;
+      uint16_t level = *pc++;
+      uint16_t vidx = *pc++;
+      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
+      jv* var = frame_local_var(fp, vidx);
+      assert(jv_get_kind(*var) == JV_KIND_ARRAY);
+      *var = jv_array_append(*var, v);
       break;
     }
 
     case INSERT: {
-      stackval stktop = stack_pop();
-      jv v = stack_pop().value;
-      jv k = stack_pop().value;
-      stackval objv = stack_pop();
+      stackval stktop = stack_pop(jq);
+      jv v = stack_pop(jq).value;
+      jv k = stack_pop(jq).value;
+      stackval objv = stack_pop(jq);
       assert(jv_get_kind(k) == JV_KIND_STRING);
       assert(jv_get_kind(objv.value) == JV_KIND_OBJECT);
       objv.value = jv_object_set(objv.value, k, v);
-      stack_push(objv);
-      stack_push(stktop);
+      stack_push(jq, objv);
+      stack_push(jq, stktop);
       break;
     }
 
@@ -259,64 +265,55 @@ jv jq_next() {
     case LOADV: {
       uint16_t level = *pc++;
       uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&frame_stk, frame_current(&frame_stk), level);
+      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
       jv* var = frame_local_var(fp, v);
-      #if JQ_DEBUG
-      printf("V%d = ", v);
-      jv_dump(jv_copy(*var), 0);
-      printf("\n");
-      #endif
-      stack_push(stackval_replace(stack_pop(), jv_copy(*var)));
+      if (jq->debug_trace_enabled) {
+        printf("V%d = ", v);
+        jv_dump(jv_copy(*var), 0);
+        printf("\n");
+      }
+      stack_push(jq, stackval_replace(stack_pop(jq), jv_copy(*var)));
       break;
     }
 
     case STOREV: {
       uint16_t level = *pc++;
       uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&frame_stk, frame_current(&frame_stk), level);
+      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
       jv* var = frame_local_var(fp, v);
-      stackval val = stack_pop();
-      #if JQ_DEBUG
-      printf("V%d = ", v);
-      jv_dump(jv_copy(val.value), 0);
-      printf("\n");
-      #endif
+      stackval val = stack_pop(jq);
+      if (jq->debug_trace_enabled) {
+        printf("V%d = ", v);
+        jv_dump(jv_copy(val.value), 0);
+        printf("\n");
+      }
       jv_free(*var);
       *var = val.value;
       break;
     }
 
-    case ASSIGN: {
-      stackval replacement = stack_pop();
-      stackval path_end = stack_pop();
-      stackval path_start = stack_pop();
+    case GETPATH: {
+      stackval path_end = stack_pop(jq);
+      stackval path_start = stack_pop(jq);
       jv_free(path_end.value);
-      jv_free(path_start.value);
-
-      uint16_t level = *pc++;
-      uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&frame_stk, frame_current(&frame_stk), level);
-      jv* var = frame_local_var(fp, v);
-      jv result = jv_insert(*var, replacement.value, pathbuf + path_start.pathidx, path_end.pathidx - path_start.pathidx);
-      if (jv_is_valid(result)) {
-        *var = result;
-      } else {
-        print_error(result);
-        *var = jv_null();
+      jv path = jv_array();
+      for (int i=path_start.pathidx; i<path_end.pathidx; i++) {
+        path = jv_array_set(path, i, jv_copy(jq->pathbuf[i]));
       }
+      stack_push(jq, stackval_replace(path_start, path));
       break;
     }
 
     case INDEX: {
-      stackval t = stack_pop();
-      jv k = stack_pop().value;
-      int pathidx = path_push(t, jv_copy(k));
-      jv v = jv_lookup(t.value, k);
+      stackval t = stack_pop(jq);
+      jv k = stack_pop(jq).value;
+      int pathidx = path_push(jq, t, jv_copy(k));
+      jv v = jv_get(t.value, k);
       if (jv_is_valid(v)) {
         stackval sv;
         sv.value = v;
         sv.pathidx = pathidx;
-        stack_push(sv);
+        stack_push(jq, sv);
       } else {
         print_error(v);
         goto do_backtrack;
@@ -333,21 +330,21 @@ jv jq_next() {
 
     case JUMP_F: {
       uint16_t offset = *pc++;
-      stackval t = stack_pop();
+      stackval t = stack_pop(jq);
       jv_kind kind = jv_get_kind(t.value);
       if (kind == JV_KIND_FALSE || kind == JV_KIND_NULL) {
         pc += offset;
       }
-      stack_push(t); // FIXME do this better
+      stack_push(jq, t); // FIXME do this better
       break;
     }
 
     case EACH: 
-      stack_push(stackval_root(jv_number(-1)));
+      stack_push(jq, stackval_root(jv_number(-1)));
       // fallthrough
     case ON_BACKTRACK(EACH): {
-      int idx = jv_number_value(stack_pop().value);
-      stackval container = stack_pop();
+      int idx = jv_number_value(stack_pop(jq).value);
+      stackval container = stack_pop(jq);
 
       int keep_going;
       jv key, value;
@@ -378,34 +375,34 @@ jv jq_next() {
         jv_free(container.value);
         goto do_backtrack;
       } else {
-        stack_save();
-        stack_push(container);
-        stack_push(stackval_root(jv_number(idx)));
-        frame_push_backtrack(&frame_stk, pc - 1);
-        stack_switch();
+        stack_save(jq);
+        stack_push(jq, container);
+        stack_push(jq, stackval_root(jv_number(idx)));
+        frame_push_backtrack(&jq->frame_stk, pc - 1);
+        stack_switch(jq);
         
         stackval sv = {value,
-                       path_push(container, key)};
-        stack_push(sv);
+                       path_push(jq, container, key)};
+        stack_push(jq, sv);
       }
       break;
     }
 
     do_backtrack:
     case BACKTRACK: {
-      if (!stack_restore()) {
+      if (!stack_restore(jq)) {
         return jv_invalid();
       }
-      pc = *frame_current_retaddr(&frame_stk);
-      frame_pop(&frame_stk);
+      pc = *frame_current_retaddr(&jq->frame_stk);
+      frame_pop(&jq->frame_stk);
       backtracking = 1;
       break;
     }
 
     case FORK: {
-      stack_save();
-      frame_push_backtrack(&frame_stk, pc - 1);
-      stack_switch();
+      stack_save(jq);
+      frame_push_backtrack(&jq->frame_stk, pc - 1);
+      stack_switch(jq);
       pc++; // skip offset this time
       break;
     }
@@ -417,22 +414,22 @@ jv jq_next() {
     }
 
     case YIELD: {
-      jv value = stack_pop().value;
-      frame_push_backtrack(&frame_stk, pc);
+      jv value = stack_pop(jq).value;
+      frame_push_backtrack(&jq->frame_stk, pc);
       return value;
     }
       
     case CALL_BUILTIN: {
       int nargs = *pc++;
-      stackval top = stack_pop();
+      stackval top = stack_pop(jq);
       cfunc_input[0] = top.value;
       for (int i = 1; i < nargs; i++) {
-        cfunc_input[i] = stack_pop().value;
+        cfunc_input[i] = stack_pop(jq).value;
       }
-      struct cfunction* func = &frame_current_bytecode(&frame_stk)->globals->cfunctions[*pc++];
+      struct cfunction* func = &frame_current_bytecode(&jq->frame_stk)->globals->cfunctions[*pc++];
       top.value = cfunction_invoke(func, cfunc_input);
       if (jv_is_valid(top.value)) {
-        stack_push(top);
+        stack_push(jq, top);
       } else {
         print_error(top.value);
         goto do_backtrack;
@@ -443,24 +440,24 @@ jv jq_next() {
     case CALL_JQ: {
       uint16_t nclosures = *pc++;
       uint16_t* retaddr = pc + 2 + nclosures*2;
-      frame_ptr new_frame = frame_push(&frame_stk, 
-                                       make_closure(&frame_stk, frame_current(&frame_stk), pc),
+      frame_ptr new_frame = frame_push(&jq->frame_stk, 
+                                       make_closure(&jq->frame_stk, frame_current(&jq->frame_stk), pc),
                                        retaddr);
       pc += 2;
-      frame_ptr old_frame = forkable_stack_peek_next(&frame_stk, new_frame);
+      frame_ptr old_frame = forkable_stack_peek_next(&jq->frame_stk, new_frame);
       assert(nclosures == frame_self(new_frame)->bc->nclosures);
       for (int i=0; i<nclosures; i++) {
-        *frame_closure_arg(new_frame, i) = make_closure(&frame_stk, old_frame, pc);
+        *frame_closure_arg(new_frame, i) = make_closure(&jq->frame_stk, old_frame, pc);
         pc += 2;
       }
 
-      pc = frame_current_bytecode(&frame_stk)->code;
+      pc = frame_current_bytecode(&jq->frame_stk)->code;
       break;
     }
 
     case RET: {
-      pc = *frame_current_retaddr(&frame_stk);
-      frame_pop(&frame_stk);
+      pc = *frame_current_retaddr(&jq->frame_stk);
+      frame_pop(&jq->frame_stk);
       break;
     }
     }
@@ -468,42 +465,64 @@ jv jq_next() {
 }
 
 
-void jq_init(struct bytecode* bc, jv input) {
-  forkable_stack_init(&data_stk, sizeof(stackval) * 100);
-  forkable_stack_init(&frame_stk, 1024);
-  forkable_stack_init(&fork_stk, 1024);
+void jq_init(struct bytecode* bc, jv input, jq_state **jq, int flags) {
+  jq_state *new_jq;
+  new_jq = jv_mem_alloc(sizeof(*new_jq));
+  memset(new_jq, 0, sizeof(*new_jq));
+  forkable_stack_init(&new_jq->data_stk, sizeof(stackval) * 100);
+  forkable_stack_init(&new_jq->frame_stk, 1024);
+  forkable_stack_init(&new_jq->fork_stk, 1024);
   
-  stack_push(stackval_root(input));
+  stack_push(new_jq, stackval_root(input));
   struct closure top = {bc, -1};
-  frame_push(&frame_stk, top, 0);
-  frame_push_backtrack(&frame_stk, bc->code);
-}
-
-void jq_teardown() {
-  while (stack_restore()) {}
-
-  assert(forkable_stack_empty(&fork_stk));
-  assert(forkable_stack_empty(&data_stk));
-  assert(forkable_stack_empty(&frame_stk));
-  forkable_stack_free(&fork_stk);
-  forkable_stack_free(&data_stk);
-  forkable_stack_free(&frame_stk);
-
-  for (int i=0; i<pathsize; i++) {
-    jv_free(pathbuf[i]);
+  frame_push(&new_jq->frame_stk, top, 0);
+  frame_push_backtrack(&new_jq->frame_stk, bc->code);
+  if (flags & JQ_DEBUG_TRACE) {
+    new_jq->debug_trace_enabled = 1;
+  } else {
+    new_jq->debug_trace_enabled = 0;
   }
-  jv_mem_free(pathbuf);
-  pathbuf = 0;
-  pathsize = 0;
+  *jq = new_jq;
 }
 
-struct bytecode* jq_compile(const char* str) {
+void jq_teardown(jq_state **jq) {
+  jq_state *old_jq = *jq;
+  if (old_jq == NULL)
+    return;
+  *jq = NULL;
+
+  while (stack_restore(old_jq)) {}
+
+  assert(forkable_stack_empty(&old_jq->fork_stk));
+  assert(forkable_stack_empty(&old_jq->data_stk));
+  assert(forkable_stack_empty(&old_jq->frame_stk));
+  forkable_stack_free(&old_jq->fork_stk);
+  forkable_stack_free(&old_jq->data_stk);
+  forkable_stack_free(&old_jq->frame_stk);
+
+  for (int i=0; i<old_jq->pathsize; i++) {
+    jv_free(old_jq->pathbuf[i]);
+  }
+  jv_mem_free(old_jq->pathbuf);
+  jv_mem_free(old_jq);
+}
+
+struct bytecode* jq_compile_args(const char* str, jv args) {
+  assert(jv_get_kind(args) == JV_KIND_ARRAY);
   struct locfile locations;
   locfile_init(&locations, str, strlen(str));
   block program;
   struct bytecode* bc = 0;
   int nerrors = jq_parse(&locations, &program);
   if (nerrors == 0) {
+    for (int i=0; i<jv_array_length(jv_copy(args)); i++) {
+      jv arg = jv_array_get(jv_copy(args), i);
+      jv name = jv_object_get(jv_copy(arg), jv_string("name"));
+      jv value = jv_object_get(arg, jv_string("value"));
+      program = gen_var_binding(gen_const(value), jv_string_value(name), program);
+      jv_free(name);
+    }
+    jv_free(args);
     program = builtins_bind(program);
     nerrors = block_compile(program, &locations, &bc);
   }
@@ -513,3 +532,7 @@ struct bytecode* jq_compile(const char* str) {
   locfile_free(&locations);
   return bc;
 }
+
+struct bytecode* jq_compile(const char* str) {
+  return jq_compile_args(str, jv_array());
+}

execute.h

@@ -3,11 +3,18 @@
 #include "bytecode.h"
 
 
+
 struct bytecode* jq_compile(const char* str);
 
-void jq_init(struct bytecode* bc, jv value);
-jv jq_next();
-void jq_teardown();
+/* args must be an array of the form [{name:"foo", value:"thing"}, {name:"bar",value:3}] */
+struct bytecode* jq_compile_args(const char* str, jv args);
+
+typedef struct jq_state jq_state;
+enum {JQ_DEBUG_TRACE = 1};
+
+void jq_init(struct bytecode* bc, jv value, jq_state **, int flags);
+jv jq_next(jq_state *);
+void jq_teardown(jq_state **);
 
 
 #endif

forkable_stack.h

@@ -100,13 +100,22 @@ static void* forkable_stack_peek_next(struct forkable_stack* s, void* top) {
 // Returns 1 if the next forkable_stack_pop will permanently remove an
 // object from the stack (i.e. the top object was not saved with a fork)
 static int forkable_stack_pop_will_free(struct forkable_stack* s) {
-  return s->pos < s->savedlimit ? 1 : 0;
+  return s->pos < s->savedlimit;
 }
 
 static void forkable_stack_pop(struct forkable_stack* s) {
   forkable_stack_check(s);
   struct forkable_stack_header* elem = forkable_stack_peek(s);
-  s->pos += elem->next_delta;
+  int reclaim_upto = s->pos + elem->next_delta < s->savedlimit ? 
+    s->pos + elem->next_delta : s->savedlimit;
+  assert(reclaim_upto <= s->length);
+  int reclaimed = reclaim_upto > s->pos ? reclaim_upto - s->pos : 0;
+  assert(0 <= reclaimed && s->pos + reclaimed <= s->length);
+  // s->pos <= i < reclaim_upto means that position i is to be freed
+  assert((reclaimed > 0) == forkable_stack_pop_will_free(s));
+  int new_pos = s->pos + elem->next_delta;
+  jv_mem_invalidate(elem, reclaimed);
+  s->pos = new_pos;
 }
 
 

gen_utf8_tables.py

@@ -1,5 +1,7 @@
 #!/usr/bin/python
 
+# This program was used to generate jv_utf8_tables.gen.h
+
 mask = lambda n: (1 << n) - 1
 
 

jq.1.default

@@ -0,0 +1,39 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "JQ" "1" "May 2013" "" ""
+.
+.SH "NAME"
+\fBjq\fR \- Command\-line JSON processor
+.
+.SH "DESCRIPTION"
+\fBjq\fR can transform JSON in various ways, by selecting, iterating, reducing and otherwise mangling JSON documents\.
+.
+.P
+This version of \fBjq\fR was built without a manual, so this manpage is a stub\. For full documentation of the \fBjq\fR language, see:
+.
+.IP "" 4
+.
+.nf
+
+http://stedolan\.github\.com/jq
+.
+.fi
+.
+.IP "" 0
+.
+.SH "BUGS"
+Presumably\. Report them or discuss them at:
+.
+.IP "" 4
+.
+.nf
+
+https://github\.com/stedolan/jq/issues
+.
+.fi
+.
+.IP "" 0
+.
+.SH "AUTHOR"
+Stephen Dolan \fB<mu@netsoc\.tcd\.ie>\fR

jq.spec

@@ -0,0 +1,65 @@
+# This is spec file maintained by developers of JQ, not by a OS distro.
+# Your OS of choice will likely ignore this RPM spec file.
+Summary: Command-line JSON processor
+Name: jq
+Version: %{myver}
+Release: %{myrel}%{?dist}
+Source0: jq-%{myver}.tar.gz
+URL: https://github.com/stedolan/jq
+License: BSD
+AutoReqProv: no
+#BuildPrereq: autoconf, libtool, automake, flex, bison, python
+
+Group: Applications/System
+# Requires:
+
+# Disables debug packages and stripping of binaries:
+%global _enable_debug_package 0
+%global debug_package %{nil}
+%global __os_install_post %{nil}
+
+# Crank up the compression
+%define _binary_payload w7.lzdio
+
+%description
+jq is a command-line JSON processor
+
+%prep
+%setup
+
+%build
+echo "Building in: \"$(pwd)\""
+%if "%{devbuild}" == "yes"
+./configure --prefix=%{_prefix} --enable-devel
+%else
+./configure --prefix=%{_prefix}
+%endif
+make
+
+%install
+echo "Installing to: \"%{buildroot}\""
+make install DESTDIR=%{buildroot}
+
+%clean
+rm -rf %{buildroot}
+
+%files
+%defattr(-,root,root)
+%{_bindir}/jq
+%if "%{devbuild}" == "yes"
+%{_libexecdir}/%{name}/jq_test
+%{_libexecdir}/%{name}/testdata
+%endif
+%{_datadir}/doc/%{name}/AUTHORS
+%{_datadir}/doc/%{name}/COPYING
+%{_datadir}/doc/%{name}/INSTALL
+%{_datadir}/doc/%{name}/NEWS
+%{_datadir}/doc/%{name}/README
+%{_datadir}/doc/%{name}/README.md
+%{_datadir}/man/man1/jq.1
+
+%changelog
+
+%pre
+
+%post

jq_parser.h

@@ -1,5 +1,5 @@
-#ifndef PARSER_H
-#define PARSER_H
+#ifndef JQ_PARSER_H
+#define JQ_PARSER_H
 
 int jq_parse(struct locfile* source, block* answer);
 int jq_parse_library(struct locfile* locations, block* answer);

jq_test.c

@@ -8,29 +8,13 @@
 static void jv_test();
 static void run_jq_tests();
 
-FILE* testdata;
 
-int main(int argc, char* argv[]) {
+int jq_testsuite(int argc, char* argv[]) {
   jv_test();
-  if (argc == 1) {
-    testdata = fopen("testdata", "r");
-  } else if (argc == 2) {
-    if (!strcmp(argv[1], "-")) {
-      testdata = stdin;
-    } else {
-      testdata = fopen(argv[1], "r");
-    }
-  } else {
-    printf("usage: %s OR cat testdata | %s - OR %s testdata\n", argv[0], argv[0], argv[0]);
-    return 127;
-  }
-  run_jq_tests();
-  if (testdata != stdin) fclose(testdata);
+  run_jq_tests(stdin);
+  return 0;
 }
 
-
-
-
 static int skipline(const char* buf) {
   int p = 0;
   while (buf[p] == ' ' || buf[p] == '\t') p++;
@@ -38,9 +22,10 @@ static int skipline(const char* buf) {
   return 0;
 }
 
-static void run_jq_tests() {
+static void run_jq_tests(FILE *testdata) {
   char buf[4096];
   int tests = 0, passed = 0, invalid = 0;
+  jq_state *jq = NULL;
 
   while (1) {
     if (!fgets(buf, sizeof(buf), testdata)) break;
@@ -51,21 +36,19 @@ static void run_jq_tests() {
     tests++;
     struct bytecode* bc = jq_compile(buf);
     if (!bc) {invalid++; continue;}
-#if JQ_DEBUG
     printf("Disassembly:\n");
     dump_disassembly(2, bc);
     printf("\n");
-#endif
     fgets(buf, sizeof(buf), testdata);
     jv input = jv_parse(buf);
     if (!jv_is_valid(input)){ invalid++; continue; }
-    jq_init(bc, input);
+    jq_init(bc, input, &jq, JQ_DEBUG_TRACE);
 
     while (fgets(buf, sizeof(buf), testdata)) {
       if (skipline(buf)) break;
       jv expected = jv_parse(buf);
       if (!jv_is_valid(expected)){ invalid++; continue; }
-      jv actual = jq_next();
+      jv actual = jq_next(jq);
       if (!jv_is_valid(actual)) {
         jv_free(actual);
         printf("*** Insufficient results\n");
@@ -88,7 +71,7 @@ static void run_jq_tests() {
       jv_free(actual);
     }
     if (pass) {
-      jv extra = jq_next();
+      jv extra = jq_next(jq);
       if (jv_is_valid(extra)) {
         printf("*** Superfluous result: ");
         jv_dump(extra, 0);
@@ -98,7 +81,7 @@ static void run_jq_tests() {
         jv_free(extra);
       }
     }
-    jq_teardown();
+    jq_teardown(&jq);
     bytecode_free(bc);
     passed+=pass;
   }

jv.c

@@ -254,8 +254,7 @@ static int jvp_array_equal(jv_complex* a, jv_complex* b) {
 static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
   // FIXME: maybe slice should reallocate if the slice is small enough
   assert(start <= end);
-  jvp_array* array = jvp_array_ptr(a);
-  assert(a->i[1] + end < array->length);
+  assert(a->i[0] + end <= a->i[1]);
   jv_complex slice = *a;
   slice.i[0] += start;
   slice.i[1] = slice.i[0] + (end - start);
@@ -316,8 +315,8 @@ jv jv_array_concat(jv a, jv b) {
   assert(jv_get_kind(b) == JV_KIND_ARRAY);
 
   // FIXME: could be much faster
-  for (int i=0; i<jv_array_length(jv_copy(b)); i++) {
-    a = jv_array_append(a, jv_array_get(jv_copy(b), i));
+  jv_array_foreach(b, i, elem) {
+    a = jv_array_append(a, elem);
   }
   jv_free(b);
   return a;
@@ -332,16 +331,15 @@ jv jv_array_slice(jv a, int start, int end) {
 
 int jv_array_contains(jv a, jv b) {
   int r = 1;
-  int a_length = jv_array_length(jv_copy(a));
-  int b_length = jv_array_length(jv_copy(b));
-  for (int bi = 0; bi < b_length; bi++) {
+  jv_array_foreach(b, bi, belem) {
     int ri = 0;
-    for (int ai = 0; ai < a_length; ai++) {
-      if (jv_contains(jv_array_get(jv_copy(a), ai), jv_array_get(jv_copy(b), bi))) {
+    jv_array_foreach(a, ai, aelem) {
+      if (jv_contains(aelem, jv_copy(belem))) {
         ri = 1;
         break;
       }
     }
+    jv_free(belem);
     if (!ri) {
       r = 0;
       break;
@@ -877,10 +875,8 @@ int jv_object_length(jv object) {
 
 jv jv_object_merge(jv a, jv b) {
   assert(jv_get_kind(a) == JV_KIND_OBJECT);
-  jv_object_foreach(i, b) {
-    a = jv_object_set(a, 
-                      jv_object_iter_key(b, i),
-                      jv_object_iter_value(b, i));
+  jv_object_foreach(b, k, v) {
+    a = jv_object_set(a, k, v);
   }
   jv_free(b);
   return a;
@@ -891,10 +887,8 @@ int jv_object_contains(jv a, jv b) {
   assert(jv_get_kind(b) == JV_KIND_OBJECT);
   int r = 1;
 
-  jv_object_foreach(i, b) {
-    jv key = jv_object_iter_key(b, i);
+  jv_object_foreach(b, key, b_val) {
     jv a_val = jv_object_get(jv_copy(a), jv_copy(key));
-    jv b_val = jv_object_get(jv_copy(b), jv_copy(key));
 
     r = jv_contains(a_val, b_val);
     jv_free(key);

jv.h

@@ -72,7 +72,13 @@ jv jv_array_set(jv, int, jv);
 jv jv_array_append(jv, jv);
 jv jv_array_concat(jv, jv);
 jv jv_array_slice(jv, int, int);
-
+#define jv_array_foreach(a, i, x) \
+  for (int jv_len__ = jv_array_length(jv_copy(a)), i=0, jv_j__ = 1;     \
+       jv_j__; jv_j__ = 0)                                              \
+    for (jv x;                                                          \
+         i < jv_len__ ?                                                 \
+           (x = jv_array_get(jv_copy(a), i), 1) : 0;                    \
+         i++)
 
 jv jv_string(const char*);
 jv jv_string_sized(const char*, int);
@@ -96,10 +102,15 @@ int jv_object_iter_next(jv, int);
 int jv_object_iter_valid(jv, int);
 jv jv_object_iter_key(jv, int);
 jv jv_object_iter_value(jv, int);
-#define jv_object_foreach(i,t) \
-  for (int i = jv_object_iter(t);               \
-       jv_object_iter_valid(t, i);              \
-       i = jv_object_iter_next(t, i))           \
+#define jv_object_foreach(t, k, v)                                      \
+  for (int jv_i__ = jv_object_iter(t), jv_j__ = 1; jv_j__; jv_j__ = 0)  \
+    for (jv k, v;                                                       \
+         jv_object_iter_valid((t), jv_i__) ?                            \
+           (k = jv_object_iter_key(t, jv_i__),                          \
+            v = jv_object_iter_value(t, jv_i__),                        \
+            1)                                                          \
+           : 0;                                                         \
+         jv_i__ = jv_object_iter_next(t, jv_i__))                       \
  
 
 

jv_alloc.c

@@ -26,3 +26,12 @@ void* jv_mem_realloc(void* p, size_t sz) {
   }
   return p;
 }
+
+#ifndef NDEBUG
+volatile char jv_mem_uninitialised;
+__attribute__((constructor)) void jv_mem_uninit_setup(){
+  char* p = malloc(1);
+  jv_mem_uninitialised = *p;
+  free(p);
+}
+#endif

jv_alloc.h

@@ -3,6 +3,17 @@
 
 #include <stddef.h>
 
+#ifndef NDEBUG
+extern volatile char jv_mem_uninitialised;
+#endif
+
+static void jv_mem_invalidate(void* mem, size_t n) {
+#ifndef NDEBUG
+  char* m = mem;
+  while (n--) *m++ ^= jv_mem_uninitialised ^ jv_mem_uninitialised;
+#endif
+}
+
 void* jv_mem_alloc(size_t);
 void jv_mem_free(void*);
 __attribute__((warn_unused_result)) void* jv_mem_realloc(void*, size_t);

jv_aux.c

@@ -3,7 +3,7 @@
 #include <stdlib.h>
 #include "jv_alloc.h"
 
-jv jv_lookup(jv t, jv k) {
+jv jv_get(jv t, jv k) {
   jv v;
   if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) {
     v = jv_object_get(t, k);
@@ -33,7 +33,12 @@ jv jv_lookup(jv t, jv k) {
   return v;
 }
 
-jv jv_modify(jv t, jv k, jv v) {
+jv jv_set(jv t, jv k, jv v) {
+  if (!jv_is_valid(v)) {
+    jv_free(t);
+    jv_free(k);
+    return v;
+  }
   int isnull = jv_get_kind(t) == JV_KIND_NULL;
   if (jv_get_kind(k) == JV_KIND_STRING && 
       (jv_get_kind(t) == JV_KIND_OBJECT || isnull)) {
@@ -55,13 +60,211 @@ jv jv_modify(jv t, jv k, jv v) {
   return t;
 }
 
-jv jv_insert(jv root, jv value, jv* path, int pathlen) {
-  if (pathlen == 0) {
+jv jv_has(jv t, jv k) {
+  assert(jv_is_valid(t));
+  assert(jv_is_valid(k));
+  jv ret;
+  if (jv_get_kind(t) == JV_KIND_NULL) {
+    jv_free(t);
+    jv_free(k);
+    ret = jv_false();
+  } else if (jv_get_kind(t) == JV_KIND_OBJECT &&
+             jv_get_kind(k) == JV_KIND_STRING) {
+    jv elem = jv_object_get(t, k);
+    ret = jv_bool(jv_is_valid(elem));
+    jv_free(elem);
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY &&
+             jv_get_kind(k) == JV_KIND_NUMBER) {
+    jv elem = jv_array_get(t, (int)jv_number_value(k));
+    ret = jv_bool(jv_is_valid(elem));
+    jv_free(elem);
+  } else {
+    ret = jv_invalid_with_msg(jv_string_fmt("Cannot check whether %s has a %s key",
+                                            jv_kind_name(jv_get_kind(t)),
+                                            jv_kind_name(jv_get_kind(k))));
+    jv_free(t);
+    jv_free(k);
+  }
+  return ret;
+}
+
+// assumes keys is a sorted array
+jv jv_dels(jv t, jv keys) {
+  assert(jv_get_kind(keys) == JV_KIND_ARRAY);
+  assert(jv_is_valid(t));
+  
+  if (jv_get_kind(t) == JV_KIND_NULL || jv_array_length(jv_copy(keys)) == 0) {
+    // no change
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY) {
+    jv new_array = jv_array();
+    int kidx = 0;
+    jv_array_foreach(t, i, elem) {
+      int del = 0;
+      while (kidx < jv_array_length(jv_copy(keys))) {
+        jv nextdel = jv_array_get(jv_copy(keys), kidx);
+        if (jv_get_kind(nextdel) != JV_KIND_NUMBER) {
+          jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array",
+                                                     jv_kind_name(jv_get_kind(nextdel))));
+          jv_free(nextdel);
+          jv_free(new_array);
+          jv_free(elem);
+          new_array = err;
+          goto arr_out; // break twice
+        }
+        int delidx = (int)jv_number_value(nextdel);
+        jv_free(nextdel);
+        if (i == delidx) {
+          del = 1;
+        }
+        if (i < delidx) {
+          break;
+        }
+        kidx++;
+      }
+      if (!del)
+        new_array = jv_array_append(new_array, elem);
+      else
+        jv_free(elem);
+    }
+  arr_out:
+    jv_free(t);
+    t = new_array;
+  } else if (jv_get_kind(t) == JV_KIND_OBJECT) {
+    jv_array_foreach(keys, i, k) {
+      if (jv_get_kind(k) != JV_KIND_STRING) {
+        jv_free(t);
+        t = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s field of object",
+                                              jv_kind_name(jv_get_kind(k))));
+        jv_free(k);
+        break;
+      }
+      t = jv_object_delete(t, k);      
+    }
+  } else {
+    jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete fields from %s",
+                                               jv_kind_name(jv_get_kind(t))));
+    jv_free(t);
+    t = err;
+  }
+  jv_free(keys);
+  return t;
+}
+
+jv jv_setpath(jv root, jv path, jv value) {
+  if (jv_get_kind(path) != JV_KIND_ARRAY) {
+    jv_free(value);
+    jv_free(root);
+    jv_free(path);
+    return jv_invalid_with_msg(jv_string("Path must be specified as an array"));
+  }
+  if (!jv_is_valid(root)){
+    jv_free(value);
+    jv_free(path);
+    return root;
+  }
+  if (jv_array_length(jv_copy(path)) == 0) {
+    jv_free(path);
     jv_free(root);
     return value;
   }
-  return jv_modify(root, jv_copy(*path), 
-                   jv_insert(jv_lookup(jv_copy(root), jv_copy(*path)), value, path+1, pathlen-1));
+  jv pathcurr = jv_array_get(jv_copy(path), 0);
+  jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path)));
+  return jv_set(root, pathcurr, 
+                jv_setpath(jv_get(jv_copy(root), jv_copy(pathcurr)), pathrest, value));
+}
+
+jv jv_getpath(jv root, jv path) {
+  if (jv_get_kind(path) != JV_KIND_ARRAY) {
+    jv_free(root);
+    jv_free(path);
+    return jv_invalid_with_msg(jv_string("Path must be specified as an array"));
+  }
+  if (!jv_is_valid(root)) {
+    jv_free(path);
+    return root;
+  }
+  if (jv_array_length(jv_copy(path)) == 0) {
+    jv_free(path);
+    return root;
+  }
+  jv pathcurr = jv_array_get(jv_copy(path), 0);
+  jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path)));
+  return jv_getpath(jv_get(root, pathcurr), pathrest);
+}
+
+// assumes paths is a sorted array of arrays
+static jv delpaths_sorted(jv object, jv paths, int start) {
+  jv delkeys = jv_array();
+  for (int i=0; i<jv_array_length(jv_copy(paths));) {
+    int j = i;
+    assert(jv_array_length(jv_array_get(jv_copy(paths), i)) > start);
+    int delkey = jv_array_length(jv_array_get(jv_copy(paths), i)) == start + 1;
+    jv key = jv_array_get(jv_array_get(jv_copy(paths), i), start);
+    while (j < jv_array_length(jv_copy(paths)) &&
+           jv_equal(jv_copy(key), jv_array_get(jv_array_get(jv_copy(paths), j), start)))
+      j++;
+    // if i <= entry < j, then entry starts with key
+    if (delkey) {
+      // deleting this entire key, we don't care about any more specific deletions
+      delkeys = jv_array_append(delkeys, key);
+    } else {
+      // deleting certain sub-parts of this key
+      jv subobject = jv_get(jv_copy(object), jv_copy(key));
+      if (!jv_is_valid(subobject)) {
+        jv_free(key);
+        jv_free(object);
+        object = subobject;
+        break;
+      } else if (jv_get_kind(subobject) == JV_KIND_NULL) {
+        jv_free(key);
+        jv_free(subobject);
+      } else {
+        jv newsubobject = delpaths_sorted(subobject, jv_array_slice(jv_copy(paths), i, j), start+1);
+        if (!jv_is_valid(newsubobject)) {
+          jv_free(key);
+          jv_free(object);
+          object = newsubobject;
+          break;
+        }
+        object = jv_set(object, key, newsubobject);
+      }
+      if (!jv_is_valid(object)) break;
+    }
+    i = j;
+  }
+  jv_free(paths);
+  if (jv_is_valid(object))
+    object = jv_dels(object, delkeys);
+  else 
+    jv_free(delkeys);
+  return object;
+}
+
+jv jv_delpaths(jv object, jv paths) {
+  paths = jv_sort(paths, jv_copy(paths));
+  jv_array_foreach(paths, i, elem) {
+    if (jv_get_kind(elem) != JV_KIND_ARRAY) {
+      jv_free(object);
+      jv_free(paths);
+      jv err = jv_invalid_with_msg(jv_string_fmt("Path must be specified as array, not %s",
+                                                 jv_kind_name(jv_get_kind(elem))));
+      jv_free(elem);
+      return err;
+    }
+    jv_free(elem);
+  }
+  if (jv_array_length(jv_copy(paths)) == 0) {
+    // nothing is being deleted
+    jv_free(paths);
+    return object;
+  }
+  if (jv_array_length(jv_array_get(jv_copy(paths), 0)) == 0) {
+    // everything is being deleted
+    jv_free(paths);
+    jv_free(object);
+    return jv_null();
+  }
+  return delpaths_sorted(object, paths, 0);
 }
 
 
@@ -81,8 +284,9 @@ jv jv_keys(jv x) {
     int nkeys = jv_object_length(jv_copy(x));
     jv* keys = jv_mem_alloc(sizeof(jv) * nkeys);
     int kidx = 0;
-    jv_object_foreach(i, x) {
-      keys[kidx++] = jv_object_iter_key(x, i);
+    jv_object_foreach(x, key, value) {
+      keys[kidx++] = key;
+      jv_free(value);
     }
     qsort(keys, nkeys, sizeof(jv), string_cmp);
     jv answer = jv_array_sized(nkeys);
@@ -163,8 +367,7 @@ int jv_cmp(jv a, jv b) {
     jv keys_b = jv_keys(jv_copy(b));
     r = jv_cmp(jv_copy(keys_a), keys_b);
     if (r == 0) {
-      for (int i=0; i<jv_array_length(jv_copy(keys_a)); i++) {
-        jv key = jv_array_get(jv_copy(keys_a), i);
+      jv_array_foreach(keys_a, i, key) {
         jv xa = jv_object_get(jv_copy(a), jv_copy(key));
         jv xb = jv_object_get(jv_copy(b), key);
         r = jv_cmp(xa, xb);

jv_aux.h

@@ -3,9 +3,12 @@
 
 #include "jv.h"
 
-jv jv_lookup(jv t, jv k);
-jv jv_modify(jv t, jv k, jv v);
-jv jv_insert(jv root, jv value, jv* path, int pathlen);
+jv jv_get(jv t, jv k);
+jv jv_set(jv t, jv k, jv v);
+jv jv_has(jv t, jv k);
+jv jv_setpath(jv root, jv path, jv value);
+jv jv_getpath(jv root, jv path);
+jv jv_delpaths(jv root, jv paths);
 
 jv jv_keys(jv /*object or array*/);
 int jv_cmp(jv, jv);

jv_parse.c

@@ -26,6 +26,8 @@ void jv_parser_init(struct jv_parser* p) {
   p->curr_buf = 0;
   p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0;
   p->bom_strip_position = 0;
+  p->line = 1;
+  p->column = 0;
   jvp_dtoa_context_init(&p->dtoa);
 }
 
@@ -281,7 +283,7 @@ static chclass classify(char c) {
 }
 
 
-static presult OK = "output produced";
+static const presult OK = "output produced";
 
 static int check_done(struct jv_parser* p, jv* out) {
   if (p->stackpos == 0 && jv_is_valid(p->next)) {
@@ -294,6 +296,11 @@ static int check_done(struct jv_parser* p, jv* out) {
 }
 
 static pfunc scan(struct jv_parser* p, char ch, jv* out) {
+  p->column++;
+  if (ch == '\n') {
+    p->line++;
+    p->column = 0;
+  }
   presult answer = 0;
   if (p->st == JV_PARSER_NORMAL) {
     chclass cls = classify(ch);
@@ -334,7 +341,7 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
   return answer;
 }
 
-static unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};
+static const unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};
 
 void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) {
   assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length)
@@ -373,7 +380,7 @@ jv jv_parser_next(struct jv_parser* p) {
   if (msg == OK) {
     return value;
   } else if (msg) {
-    return jv_invalid_with_msg(jv_string(msg));
+    return jv_invalid_with_msg(jv_string_fmt("%s at line %d, column %d", msg, p->line, p->column));
   } else if (p->curr_buf_is_partial) {
     assert(p->curr_buf_pos == p->curr_buf_length);
     // need another buffer

jv_parse.h

@@ -16,6 +16,8 @@ struct jv_parser {
   char* tokenbuf;
   int tokenpos;
   int tokenlen;
+
+  int line, column;
   
   struct dtoa_context dtoa;
 

jv_print.c

@@ -12,10 +12,10 @@
 
 // Colour table. See http://en.wikipedia.org/wiki/ANSI_escape_code#Colors
 // for how to choose these.
-static jv_kind colour_kinds[] = 
+static const jv_kind colour_kinds[] = 
   {JV_KIND_NULL,   JV_KIND_FALSE, JV_KIND_TRUE, JV_KIND_NUMBER,
    JV_KIND_STRING, JV_KIND_ARRAY, JV_KIND_OBJECT};
-static const char* colours[] =
+static const char* const colours[] =
   {COL("30;1"),    COL("0"),      COL("0"),     COL("0"),
    COL("32"),      COL("37"),     COL("37")};
 #define FIELD_COLOUR COL("34;1")
@@ -162,7 +162,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       put_char('\n', F, S);
       put_space(indent+INDENT, F, S);
     }
-    for (int i=0; i<jv_array_length(jv_copy(x)); i++) {
+    jv_array_foreach(x, i, elem) {
       if (i!=0) {
         if (flags & JV_PRINT_PRETTY) {
           put_str(",\n", F, S);
@@ -171,7 +171,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
           put_str(",", F, S);
         }
       }
-      jv_dump_term(C, jv_array_get(jv_copy(x), i), flags, indent + INDENT, F, S);
+      jv_dump_term(C, elem, flags, indent + INDENT, F, S);
       if (colour) put_str(colour, F, S);
     }
     if (flags & JV_PRINT_PRETTY) {
@@ -193,7 +193,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       put_space(indent+INDENT, F, S);
     }
     int first = 1;
-    jv_object_foreach(i, x) {
+    jv_object_foreach(x, key, value) {
       if (!first) {
         if (flags & JV_PRINT_PRETTY){
           put_str(",\n", F, S);
@@ -205,7 +205,6 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       if (colour) put_str(COLRESET, F, S);
 
       first = 0;
-      jv key = jv_object_iter_key(x, i);
       if (colour) put_str(FIELD_COLOUR, F, S);
       jvp_dump_string(key, flags & JV_PRINT_ASCII, F, S);
       jv_free(key);
@@ -215,7 +214,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       put_str((flags & JV_PRINT_PRETTY) ? ": " : ":", F, S);
       if (colour) put_str(COLRESET, F, S);
       
-      jv_dump_term(C, jv_object_iter_value(x, i), flags, indent + INDENT, F, S);
+      jv_dump_term(C, value, flags, indent + INDENT, F, S);
       if (colour) put_str(colour, F, S);
     }
     if (flags & JV_PRINT_PRETTY) {

jv_unicode.c

@@ -1,7 +1,7 @@
 #include <stdio.h>
 #include <assert.h>
 #include "jv_unicode.h"
-#include "jv_utf8_tables.gen.h"
+#include "jv_utf8_tables.h"
 
 const char* jvp_utf8_next(const char* in, const char* end, int* codepoint) {
   if (in == end) {

jv_utf8_tables.h

@@ -0,0 +1,35 @@
+#define UTF8_CONTINUATION_BYTE ((unsigned char)255)
+static const unsigned char utf8_coding_length[] =
+ {0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+  0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+  0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+  0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
+  0x04, 0x04, 0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+static const unsigned char utf8_coding_bits[] =
+ {0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+  0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+  0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+  0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+  0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f,
+  0x00, 0x00, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f,
+  0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f,
+  0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
+  0x07, 0x07, 0x07, 0x07, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};

lexer.l

@@ -4,7 +4,7 @@
 
 struct lexer_param;
 
-#include "parser.gen.h"  /* Generated by bison. */
+#include "parser.h"  /* Generated by bison. */
 
 #define YY_USER_ACTION                           \
   do {                                           \
@@ -49,6 +49,7 @@ struct lexer_param;
 "and" { return AND; }
 "or" { return OR; }
 "end" { return END; }
+"fold" { return FOLD; }
 "//" { return DEFINEDOR; }
 "|=" { return SETPIPE; }
 "+=" { return SETPLUS; }
@@ -68,8 +69,11 @@ struct lexer_param;
   return try_exit(yytext[0], YY_START, yyscanner);
 }
 
+"@"[a-zA-Z0-9_]+ {
+  yylval->literal = jv_string_sized(yytext + 1, yyleng - 1); return FORMAT;
+}
 
--?[0-9.]+([eE][+-]?[0-9]+)? { 
+[0-9.]+([eE][+-]?[0-9]+)? { 
    yylval->literal = jv_parse_sized(yytext, yyleng); return LITERAL; 
 }
 

main.c

@@ -5,19 +5,18 @@
 #include <ctype.h>
 #include <unistd.h>
 #include "compile.h"
-#include "builtin.h"
 #include "jv.h"
 #include "jv_parse.h"
-#include "locfile.h"
-#include "parser.h"
 #include "execute.h"
+#include "config.h"  /* Autoconf generated header file */
 #include "jv_alloc.h"
-#include "version.gen.h"
+
+int jq_testsuite(int argc, char* argv[]);
 
 static const char* progname;
 
 static void usage() {
-  fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", JQ_VERSION);
+  fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", PACKAGE_VERSION);
   fprintf(stderr, "Usage: %s [options] <jq filter> [file...]\n\n", progname);
   fprintf(stderr, "For a description of the command line options and\n");
   fprintf(stderr, "how to write jq filters (and why you might want to)\n");
@@ -58,20 +57,27 @@ enum {
   NO_COLOUR_OUTPUT = 128,
 
   FROM_FILE = 256,
+
+  /* debugging only */
+  DUMP_DISASM = 2048,
 };
 static int options = 0;
 static struct bytecode* bc;
 
-static void process(jv value) {
-  jq_init(bc, value);
+static void process(jv value, int flags) {
+  jq_state *jq = NULL;
+  jq_init(bc, value, &jq, flags);
   jv result;
-  while (jv_is_valid(result = jq_next())) {
+  while (jv_is_valid(result = jq_next(jq))) {
     if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
       fwrite(jv_string_value(result), 1, jv_string_length(jv_copy(result)), stdout);
+      jv_free(result);
     } else {
       int dumpopts;
-#ifdef JQ_DEFAULT_ENABLE_COLOR
-      dumpopts = JQ_DEFAULT_ENABLE_COLOR ? JV_PRINT_COLOUR : 0;
+      /* Disable colour by default on Windows builds as Windows
+         terminals tend not to display it correctly */
+#ifdef WIN32
+      dumpopts = 0;
 #else
       dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0;
 #endif
@@ -84,7 +90,7 @@ static void process(jv value) {
     printf("\n");
   }
   jv_free(result);
-  jq_teardown();
+  jq_teardown(&jq);
 }
 
 static jv slurp_file(const char* filename) {
@@ -134,17 +140,24 @@ static int read_more(char* buf, size_t size) {
     next_input_idx++;
   }
 
-  if (!fgets(buf, sizeof(buf), current_input)) buf[0] = 0;
+  if (!fgets(buf, size, current_input)) buf[0] = 0;
   return 1;
 }
 
 int main(int argc, char* argv[]) {
+  int ret = 0;
   if (argc) progname = argv[0];
 
+  if (argc > 1 && !strcmp(argv[1], "--run-tests")) {
+    return jq_testsuite(argc - 1, argv + 1);
+  }
+
   const char* program = 0;
   input_filenames = jv_mem_alloc(sizeof(const char*) * argc);
   ninput_files = 0;
   int further_args_are_files = 0;
+  int jq_flags = 0;
+  jv program_arguments = jv_array();
   for (int i=1; i<argc; i++) {
     if (further_args_are_files) {
       input_filenames[ninput_files++] = argv[i];
@@ -175,10 +188,24 @@ int main(int argc, char* argv[]) {
       options |= PROVIDE_NULL;
     } else if (isoption(argv[i], 'f', "from-file")) {
       options |= FROM_FILE;
+    } else if (isoption(argv[i], 0, "arg")) {
+      if (i >= argc - 2) {
+        fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname);
+        die();
+      }
+      jv arg = jv_object();
+      arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
+      arg = jv_object_set(arg, jv_string("value"), jv_string(argv[i+2]));
+      program_arguments = jv_array_append(program_arguments, arg);
+      i += 2; // skip the next two arguments
+    } else if (isoption(argv[i],  0,  "debug-dump-disasm")) {
+      options |= DUMP_DISASM;
+    } else if (isoption(argv[i],  0,  "debug-trace")) {
+      jq_flags |= JQ_DEBUG_TRACE;
     } else if (isoption(argv[i], 'h', "help")) {
       usage();
     } else if (isoption(argv[i], 'V', "version")) {
-      fprintf(stderr, "jq version %s\n", JQ_VERSION);
+      fprintf(stderr, "jq version %s\n", PACKAGE_VERSION);
       return 0;
     } else {
       fprintf(stderr, "%s: Unknown option %s\n", progname, argv[i]);
@@ -201,20 +228,20 @@ int main(int argc, char* argv[]) {
       jv_free(data);
       return 1;
     }
-    bc = jq_compile(jv_string_value(data));
+    bc = jq_compile_args(jv_string_value(data), program_arguments);
     jv_free(data);
   } else {
-    bc = jq_compile(program);
+    bc = jq_compile_args(program, program_arguments);
   }
   if (!bc) return 1;
 
-#if JQ_DEBUG
-  dump_disassembly(0, bc);
-  printf("\n");
-#endif
+  if (options & DUMP_DISASM) {
+    dump_disassembly(0, bc);
+    printf("\n");
+  }
 
   if (options & PROVIDE_NULL) {
-    process(jv_null());
+    process(jv_null(), jq_flags);
   } else {
     jv slurped;
     if (options & SLURP) {
@@ -235,7 +262,7 @@ int main(int argc, char* argv[]) {
             slurped = jv_string_concat(slurped, jv_string(buf));
           } else {
             if (buf[len-1] == '\n') buf[len-1] = 0;
-            process(jv_string(buf));
+            process(jv_string(buf), jq_flags);
           }
         }
       } else {
@@ -245,13 +272,14 @@ int main(int argc, char* argv[]) {
           if (options & SLURP) {
             slurped = jv_array_append(slurped, value);
           } else {
-            process(value);
+            process(value, jq_flags);
           }
         }
         if (jv_invalid_has_msg(jv_copy(value))) {
           jv msg = jv_invalid_get_msg(value);
           fprintf(stderr, "parse error: %s\n", jv_string_value(msg));
           jv_free(msg);
+          ret = 1;
           break;
         } else {
           jv_free(value);
@@ -259,11 +287,14 @@ int main(int argc, char* argv[]) {
       }
     }
     jv_parser_free(&parser);
+    if (ret != 0)
+      goto out;
     if (options & SLURP) {
-      process(slurped);
+      process(slurped, jq_flags);
     }
   }
+out:
   jv_mem_free(input_filenames);
   bytecode_free(bc);
-  return 0;
+  return ret;
 }

opcode_list.h

@@ -12,10 +12,10 @@ OP(FORK,  BRANCH,   0, 0)
 OP(JUMP,  BRANCH,   0, 0)
 OP(JUMP_F,BRANCH,   1, 0)
 OP(BACKTRACK, NONE, 0, 0)
-OP(APPEND, NONE,    2, 1)
+OP(APPEND, VARIABLE,1, 0)
 OP(INSERT, NONE,    4, 2)
 
-OP(ASSIGN, VARIABLE, 3, 0)
+OP(GETPATH, NONE,   2, 1)
 
 OP(CALL_BUILTIN, CFUNC, -1, 1)
 

parser.y

@@ -48,6 +48,7 @@ struct lexer_param;
 %token INVALID_CHARACTER
 %token <literal> IDENT
 %token <literal> LITERAL
+%token <literal> FORMAT
 %token EQ "=="
 %token NEQ "!="
 %token DEFINEDOR "//"
@@ -57,6 +58,7 @@ struct lexer_param;
 %token THEN "then"
 %token ELSE "else"
 %token ELSE_IF "elif"
+%token FOLD "fold"
 %token END "end"
 %token AND "and"
 %token OR "or"
@@ -90,7 +92,7 @@ struct lexer_param;
 
 %type <blk> Exp Term MkDict MkDictPair ExpD ElseBody QQString FuncDef FuncDefs String
 %{
-#include "lexer.gen.h"
+#include "lexer.h"
 struct lexer_param {
   yyscan_t lexer;
 };
@@ -158,15 +160,20 @@ static block gen_binop(block a, block b, int op) {
   return gen_call(funcname, BLOCK(gen_lambda(a), gen_lambda(b)));
 }
 
-static block gen_format(block a) {
-  return BLOCK(a, gen_call("tostring", gen_noop()));
+static block gen_format(block a, jv fmt) {
+  return BLOCK(a, gen_call("format", BLOCK(gen_lambda(gen_const(fmt)))));
 }
  
-static block gen_update(block a, block op, int optype) {
-  if (optype) {
-    op = gen_binop(gen_noop(), op, optype);
-  }
-  return gen_assign(BLOCK(a, gen_op_simple(DUP), op));
+static block gen_update(block object, block val, int optype) {
+  block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
+                         gen_noop(), OP_HAS_VARIABLE);
+  return BLOCK(gen_op_simple(DUP),
+               val,
+               tmp,
+               gen_call("_modify", BLOCK(gen_lambda(object), 
+                                         gen_lambda(gen_binop(gen_noop(),
+                                                              gen_op_var_bound(LOADV, tmp),
+                                                              optype)))));
 }
 
 %}
@@ -194,12 +201,15 @@ FuncDef Exp %prec ';' {
 } |
 
 Term "as" '$' IDENT '|' Exp {
-  $$ = BLOCK(gen_op_simple(DUP), $1, 
-             block_bind(gen_op_var_unbound(STOREV, jv_string_value($4)), 
-                        $6, OP_HAS_VARIABLE));
+  $$ = gen_var_binding($1, jv_string_value($4), $6);
   jv_free($4);
 } |
 
+"fold" Term "as" '$' IDENT '(' Exp ')' {
+  $$ = gen_fold(jv_string_value($5), $2, $7);
+  jv_free($5);
+} |
+
 "if" Exp "then" Exp ElseBody {
   $$ = gen_cond($2, $4, $5);
 } |
@@ -209,7 +219,7 @@ Term "as" '$' IDENT '|' Exp {
 } |
 
 Exp '=' Exp {
-  $$ = gen_assign(BLOCK(gen_op_simple(DUP), $3, gen_op_simple(SWAP), $1, gen_op_simple(SWAP)));
+  $$ = gen_call("_assign", BLOCK(gen_lambda($1), gen_lambda($3)));
 } |
 
 Exp "or" Exp {
@@ -229,7 +239,7 @@ Exp "//=" Exp {
 } |
 
 Exp "|=" Exp {
-  $$ = gen_update($1, $3, 0);
+  $$ = gen_call("_modify", BLOCK(gen_lambda($1), gen_lambda($3)));
 } |
 
 Exp '|' Exp { 
@@ -248,6 +258,10 @@ Exp "+=" Exp {
   $$ = gen_update($1, $3, '+');
 } |
 
+'-' Exp {
+  $$ = BLOCK($2, gen_call("_negate", gen_noop()));
+} |
+
 Exp '-' Exp {
   $$ = gen_binop($1, $3, '-');
 } |
@@ -312,14 +326,31 @@ FuncDef:
                     $7);
   jv_free($2);
   jv_free($4);
+} |
+
+"def" IDENT '(' IDENT ';' IDENT ')' ':' Exp ';' {
+  $$ = gen_function(jv_string_value($2), 
+                    BLOCK(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), 
+                          gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($6))),
+                    $9);
+  jv_free($2);
+  jv_free($4);
+  jv_free($6);
 }
 
 
+
 String:
-QQSTRING_START QQString QQSTRING_END {
-  $$ = $2;
+QQSTRING_START { $<literal>$ = jv_string("text"); } QQString QQSTRING_END {
+  $$ = $3;
+  jv_free($<literal>2);
+} |
+FORMAT QQSTRING_START { $<literal>$ = $1; } QQString QQSTRING_END {
+  $$ = $4;
+  jv_free($<literal>3);
 }
 
+
 QQString:
 /* empty */ {
   $$ = gen_const(jv_string(""));
@@ -328,7 +359,7 @@ QQString QQSTRING_TEXT {
   $$ = gen_binop($1, gen_const($2), '+');
 } |
 QQString QQSTRING_INTERP_START Exp QQSTRING_INTERP_END {
-  $$ = gen_binop($1, gen_format($3), '+');
+  $$ = gen_binop($1, gen_format($3, jv_copy($<literal>0)), '+');
 }
 
 
@@ -373,6 +404,9 @@ LITERAL {
 String {
   $$ = $1;
 } |
+FORMAT {
+  $$ = gen_format(gen_noop(), $1);
+} |
 '(' Exp ')' { 
   $$ = $2; 
 } | 
@@ -398,6 +432,11 @@ IDENT '(' Exp ')' {
   $$ = gen_location(@1, $$);
   jv_free($1);
 } |
+IDENT '(' Exp ';' Exp ')' {
+  $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5)));
+  $$ = gen_location(@1, $$);
+  jv_free($1);
+} |
 '(' error ')' { $$ = gen_noop(); } |
 '[' error ']' { $$ = gen_noop(); } |
 Term '[' error ']' { $$ = $1; } |

scripts/crosscompile

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# This script is used to cross-compile binaries for 
+# platforms other than the current one
+
+# Usage: $0 <platformname> <configure options>
+# <platformname> is arbitrary, it is the name
+# of the directory which will be created to contain
+# the output binaries.
+
+# e.g. $0 win32 --host=i686-w64-mingw32
+
+set -e
+cd `dirname "$0"`/../build
+
+plat="$1"
+[ -z "$plat" ] && exit 1
+shift
+
+[ -d "$plat" ] || mkdir "$plat"
+rm -rf "$plat/tmp"
+mkdir "$plat/tmp"
+cd "$plat/tmp"
+mkdir install_other
+../../../configure \
+    --prefix="`pwd`/install_other" \
+    --bindir="`pwd`/.." \
+    "$@"
+make install-binaries
+cd ..
+rm -rf tmp

setup.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# RUN ME after pulling the code from git!
+#
+# This script is handy after pulling the code out of git without having
+# to remember all the typing needed to build up and tear down all the
+# crap that autotools puts in your source repo.  The intent is to keep
+# all generated files out of git.
+
+if [ "clean" == "$1" ]; then
+  make distclean || echo "continuing..."
+  rm -rf tmp autom4te.cache config
+  rm -f INSTALL Makefile.in aclocal.m4 configure config.h.in ChangeLog
+  rm -f jv_utf8_tables.gen.h lexer.c lexer.h parser.c parser.h
+elif [ "superclean" == "$1" ]; then
+  # if autoconf errors during distcheck, it leaves files that need chmod'ing
+  ver=$(tr -d '\n' <VERSION)
+  if [ "x${ver}" != "x" ]; then
+    if [ -d jq-${ver} ]; then
+      chmod -R u+w jq-${ver}
+      rm -rf jq-${ver}
+    fi
+    rm -f jq-${ver}*
+  fi
+else
+  autoreconf --install
+  ./configure --prefix=/opt/junk
+  make check
+  mkdir tmp
+  make DESTDIR=./tmp install
+  make distcheck
+fi

tests/all.test

@@ -21,6 +21,11 @@ null
 null
 1
 
+
+-1
+null
+-1
+
 # FIXME: much more number testing needed
 
 {}
@@ -52,6 +57,24 @@ null
 null
 "interpolation"
 
+@text,@json,([1,.] | @csv),@html,@uri,@sh,@base64
+"<>&'\""
+"<>&'\""
+"\"<>&'\\\"\""
+"1,\"<>&'\"\"\""
+"&lt;&gt;&amp;&apos;&quot;"
+"%3c%3e%26'%22"
+"'<>&'\\''\"'"
+"PD4mJyI="
+
+@uri
+"\u03bc"
+"%ce%bc"
+
+@html "<b>\(.)</b>"
+"<script>hax</script>"
+"<b>&lt;script&gt;hax&lt;/script&gt;</b>"
+
 #
 # Dictionary construction syntax
 #
@@ -179,6 +202,14 @@ null
 "wtasdf"
 2.0
 
+2-1
+null
+1
+
+2-(-1)
+null
+3
+
 1e+0+0.001e3
 "I wonder what this will be?"
 20e-1
@@ -187,6 +218,18 @@ null
 15
 19.0
 
+.+null
+{"a":42}
+{"a":42}
+
+null+.
+null
+null
+
+.a+.b
+{"a":42}
+42
+
 [1,2,3] + [.]
 null
 [1,2,3,null]
@@ -317,6 +360,60 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
 # []
 # 1001
 
+fold 0 as $s (.[] | $s + .)
+[1,2,4]
+7
+
+#
+# Paths
+#
+
+path(.foo[0,1])
+null
+["foo", 0]
+["foo", 1]
+
+path(.[] | select(.>3))
+[1,5,3]
+[1]
+
+path(.)
+42
+[]
+
+["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
+{"bar": 42, "foo": ["a", "b", "c", "d"]}
+"b"
+{"bar": 42, "foo": ["a", 20, "c", "d"]}
+{"bar": 42, "foo": ["a", "c", "d"]}
+
+map(getpath([2])), map(setpath([2]; 42)), map(delpaths([[2]]))
+[[0], [0,1], [0,1,2]]
+[null, null, 2]
+[[0,null,42], [0,1,42], [0,1,42]]
+[[0], [0,1], [0,1]]
+
+map(delpaths([[0,"foo"]]))
+[[{"foo":2, "x":1}], [{"bar":2}]]
+[[{"x":1}], [{"bar":2}]]
+
+["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
+{"bar":false}
+null
+{"bar":false, "foo": [null, 20]}
+{"bar":false}
+
+delpaths([[-200]])
+[1,2,3]
+[1,2,3]
+
+del(.), del(empty), del((.foo,.bar,.baz) | .[2,3,0]), del(.foo[0], .bar[0], .foo, .baz.bar[0].x)
+{"foo": [0,1,2,3,4], "bar": [0,1]}
+null
+{"foo": [0,1,2,3,4], "bar": [0,1]}
+{"foo": [1,4], "bar": [1]}
+{"bar": [1]}
+
 #
 # Assignment
 #
@@ -339,6 +436,10 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
 [-1,1,3]
 [0.5, 1.5, 2.5]
 
+.foo += .foo
+{"foo":2}
+{"foo":4}
+
 .[0].a |= {"old":., "new":(.+1)}
 [{"a":1,"b":2}]
 [{"a":{"old":1, "new":2},"b":2}]

tests/run

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cat $@ | valgrind --error-exitcode=1 -q --leak-check=full ./jq --run-tests