Move cfunction invocation code to the interpreter loop.

Closes #144

.gitignore

@@ -1,5 +1,7 @@
 *.o
 *~
+.*.sw[a-p]
+tags
 
 jq
 jq.1
@@ -26,3 +28,6 @@ jq-*.tar.gz
 configure
 aclocal.m4
 Makefile.in
+version.h
+.remake-version-h
+config.cache

Makefile.am

@@ -2,12 +2,12 @@
 
 JQ_INCS = jq_parser.h builtin.h bytecode.h compile.h execute.h		\
   forkable_stack.h frame_layout.h jv.h jv_alloc.h jv_aux.h jv_dtoa.h	\
-  jv_parse.h jv_unicode.h locfile.h opcode.h opcode_list.h parser.y	\
+  jv_file.h jv_parse.h jv_unicode.h locfile.h opcode_list.h parser.y	\
   jv_utf8_tables.h lexer.l
 
-JQ_SRC = opcode.c bytecode.c compile.c execute.c builtin.c jv.c		\
-  jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_alloc.c	\
-  jq_test.c ${JQ_INCS}
+JQ_SRC = locfile.c bytecode.c compile.c execute.c builtin.c jv.c	\
+  jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_file.c	\
+  jv_alloc.c jq_test.c ${JQ_INCS}
 
 
 ### C build options
@@ -20,9 +20,9 @@ AM_CFLAGS = -Wextra -Wall -Wno-missing-field-initializers	\
 
 # While there is some autoconf macro support for lex/flex, it doesn't support
 # header file creation so we'll use good old make
-BUILT_SOURCES = lexer.h lexer.c parser.h parser.c
+BUILT_SOURCES = lexer.h lexer.c parser.h parser.c version.h
 lexer.c: lexer.l
-	flex -o lexer.c --header-file=lexer.h $<
+	$(AM_V_LEX) flex -o lexer.c --header-file=lexer.h $<
 lexer.h: lexer.c
 
 # Tell YACC (bison) autoconf macros that you want a header file created.
@@ -33,12 +33,18 @@ AM_YFLAGS = --warnings=all -d
 
 ### Building the jq binary
 
-main.c: config.h
+# Remake the version.h header file if, and only if, the git ID has changed
+.PHONY: .FORCE
+.FORCE:
+generate_ver = ver="`{ $(srcdir)/scripts/version || echo '$(VERSION)' ; } | sed 's/.*/\#define JQ_VERSION \"&\"/'`"
+.remake-version-h: .FORCE
+	@ $(generate_ver); test "x`cat version.h 2>/dev/null`" = "x$$ver" || touch .remake-version-h
+version.h: .remake-version-h
+	$(AM_V_GEN) $(generate_ver); echo "$$ver" > $@
 
 bin_PROGRAMS = jq
 jq_SOURCES = ${JQ_SRC} main.c
 
-
 ### Tests (make check)
 
 TESTS = tests/all.test
@@ -56,16 +62,16 @@ man_MANS = jq.1
 .PHONY: real_docs
 if ENABLE_DOCS
 jq.1: $(srcdir)/docs/content/3.manual/manual.yml
-	( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
 jq.1.default: $(srcdir)/docs/default_manpage.md
-	( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
+	$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
 real_docs: jq.1
 	if cmp jq.1 $(srcdir)/jq.1.default > /dev/null; then\
 	  rm -f jq.1; $(MAKE) $(AM_MAKEFLAGS) jq.1;\
 	fi
 else
 jq.1: $(srcdir)/jq.1.default
-	cp $^ $@
+	$(AM_V_GEN) cp $^ $@
 real_docs:
 	@echo "Ruby dependencies not found, cannot build manpage." > /dev/stderr
 	@echo "Follow the instructions in docs/README.md to install them" > /dev/stderr
@@ -77,7 +83,7 @@ endif
 ### Packaging
 
 docs/site.yml: configure.ac
-	sed 's/^jq_version: .*/jq_version: $(VERSION)/' $@ > $@.new
+	sed 's/^jq_version: .*/jq_version: "$(VERSION)"/' $@ > $@.new
 	mv $@.new $@
 
 install-binaries: $(BUILT_SOURCES)
@@ -93,18 +99,16 @@ DOC_FILES = docs/content docs/public docs/templates docs/site.yml	\
 # setup is only used by distribution developers, not package developers.
 # Still, as a matter of allowing patching, its not a bad idea to distribute 
 # the developer setup script in the tarball.
-EXTRA_DIST = config.h.in $(BUILT_SOURCES) $(man_MANS) $(TESTS)	\
+EXTRA_DIST = config.h.in $(man_MANS) $(TESTS)	\
 	$(TEST_LOG_COMPILER) gen_utf8_tables.py jq.spec \
-	$(DOC_FILES)
+	$(DOC_FILES) scripts/version \
+	parser.h parser.c lexer.h lexer.c
 
 # README.md is expected in Github projects, good stuff in it, so we'll 
 # distribute it and install it with the package in the doc directory.
 docdir = ${datadir}/doc/${PACKAGE}
 dist_doc_DATA = README.md COPYING AUTHORS README
 
-releasetag:
-	git tag -s "jq-$$(cat VERSION)" -m "jq release $$(cat VERSION)"
-
 RELEASE ?= 1
 rpm: jq
 	@echo "Packaging jq as an RPM ..."
@@ -119,5 +123,12 @@ dist-clean-local:
 
 # Not sure why this doesn't get cleaned up automatically, guess
 # automake used to man pages which are hand coded?
-clean-local:
+# 'make clean' doesn't delete the manpage if it can't be rebuilt
+.PHONY: clean-local-docs
+clean-local-docs:
+if ENABLE_DOCS
 	rm -f jq.1
+endif 
+
+clean-local: clean-local-docs
+	rm -f version.h .remake-version-h

README.md

@@ -4,20 +4,25 @@ jq
 jq is a command-line JSON processor.
 
 If you want to learn to use jq, read the documentation at
-[http://stedolan.github.com/jq](http://stedolan.github.com/jq). This
+[http://stedolan.github.io/jq](http://stedolan.github.io/jq). This
 documentation is generated from the docs/ folder of this repository.
 
 If you want to hack on jq, feel free, but be warned that its internals
 are not well-documented at the moment. Bring a hard hat and a
 shovel. Also, read the wiki: http://github.com/stedolan/jq/wiki
 
-To build jq, run
-
-    ./configure
-    make
-    sudo make install (optionally)
-
-If you've just checked out the latest version from git (rather than
-using a released source tarball) then you'll need to run this first:
+If you're building directly from the latest git, you'll need flex and
+bison installed. To build, run:
 
     autoreconf
+    ./configure
+    make
+
+After make finishes, you'll be able to use `./jq`. You can also
+install it using:
+
+    sudo make install
+
+If you're not using the latest git version but instead building a
+released tarball (available on the website), then you won't need to
+run `autoreconf` (and shouldn't), and you won't need flex or bison.

builtin.c

@@ -1,29 +1,15 @@
+#include <stdlib.h>
 #include <string.h>
 #include "builtin.h"
 #include "compile.h"
 #include "jq_parser.h"
+#include "bytecode.h"
 #include "locfile.h"
 #include "jv_aux.h"
+#include "jv_file.h"
 #include "jv_unicode.h"
 
 
-
-typedef jv (*func_1)(jv);
-typedef jv (*func_2)(jv,jv);
-typedef jv (*func_3)(jv,jv,jv);
-typedef jv (*func_4)(jv,jv,jv,jv);
-typedef jv (*func_5)(jv,jv,jv,jv,jv);
-jv cfunction_invoke(struct cfunction* function, jv input[]) {
-  switch (function->nargs) {
-  case 1: return ((func_1)function->fptr)(input[0]);
-  case 2: return ((func_2)function->fptr)(input[0], input[1]);
-  case 3: return ((func_3)function->fptr)(input[0], input[1], input[2]);
-  case 4: return ((func_4)function->fptr)(input[0], input[1], input[2], input[3]);
-  case 5: return ((func_5)function->fptr)(input[0], input[1], input[2], input[3], input[4]);
-  default: return jv_invalid_with_msg(jv_string("Function takes too many arguments"));
-  }
-}
-
 static jv type_error(jv bad, const char* msg) {
   jv err = jv_invalid_with_msg(jv_string_fmt("%s %s",
                                              jv_kind_name(jv_get_kind(bad)),
@@ -187,7 +173,7 @@ static jv f_length(jv input) {
   } else if (jv_get_kind(input) == JV_KIND_OBJECT) {
     return jv_number(jv_object_length(input));
   } else if (jv_get_kind(input) == JV_KIND_STRING) {
-    return jv_number(jv_string_length(input));
+    return jv_number(jv_string_length_codepoints(input));
   } else if (jv_get_kind(input) == JV_KIND_NULL) {
     jv_free(input);
     return jv_number(0);
@@ -220,7 +206,7 @@ static jv escape_string(jv input, const char* escapings) {
 
   jv ret = jv_string("");
   const char* i = jv_string_value(input);
-  const char* end = i + jv_string_length(jv_copy(input));
+  const char* end = i + jv_string_length_bytes(jv_copy(input));
   const char* cstart;
   int c = 0;
   while ((i = jvp_utf8_next((cstart = i), end, &c))) {
@@ -299,7 +285,7 @@ static jv f_format(jv input, jv fmt) {
 
     jv line = jv_string("");
     const char* s = jv_string_value(input);
-    for (int i=0; i<jv_string_length(jv_copy(input)); i++) {
+    for (int i=0; i<jv_string_length_bytes(jv_copy(input)); i++) {
       unsigned ch = (unsigned)(unsigned char)*s;
       if (ch < 128 && unreserved[ch]) {
         line = jv_string_append_buf(line, s, 1);
@@ -346,7 +332,7 @@ static jv f_format(jv input, jv fmt) {
     jv line = jv_string("");
     const char b64[64 + 1] = CHARS_ALPHANUM "+/";
     const char* data = jv_string_value(input);
-    int len = jv_string_length(jv_copy(input));
+    int len = jv_string_length_bytes(jv_copy(input));
     for (int i=0; i<len; i+=3) {
       uint32_t code = 0;
       int n = len - i >= 3 ? 3 : len-i;
@@ -520,18 +506,28 @@ static block bind_bytecoded_builtins(block b) {
   }
   {
     struct bytecoded_builtin builtin_def_1arg[] = {
-      {"path", BLOCK(gen_op_simple(DUP), 
+      {"path", BLOCK(gen_op_simple(PATH_BEGIN), 
                      gen_call("arg", gen_noop()),
-                     gen_op_simple(GETPATH))},
+                     gen_op_simple(PATH_END))},
     };
     for (unsigned i=0; i<sizeof(builtin_def_1arg)/sizeof(builtin_def_1arg[0]); i++) {
       builtins = BLOCK(builtins, gen_function(builtin_def_1arg[i].name,
-                                              gen_op_block_unbound(CLOSURE_PARAM, "arg"),
+                                              gen_param("arg"),
                                               builtin_def_1arg[i].code));
     }
   }
+  {
+    block rangevar = gen_op_var_fresh(STOREV, "rangevar");
+    block init = BLOCK(gen_op_simple(DUP), gen_call("start", gen_noop()), rangevar);
+    block range = BLOCK(init, 
+                        gen_call("end", gen_noop()),
+                        gen_op_bound(RANGE, rangevar));
+    builtins = BLOCK(builtins, gen_function("range",
+                                            BLOCK(gen_param("start"), gen_param("end")),
+                                            range));
+  }
   
-  return block_bind(builtins, b, OP_IS_CALL_PSEUDO);
+  return block_bind_referenced(builtins, b, OP_IS_CALL_PSEUDO);
 }
 
 static const char* const jq_builtins[] = {
@@ -542,24 +538,56 @@ static const char* const jq_builtins[] = {
   "def unique: group_by(.) | map(.[0]);",
   "def max_by(f): _max_by_impl(map([f]));",
   "def min_by(f): _min_by_impl(map([f]));",
-  "def add: fold null as $sum (.[] | $sum + .);",
+  "def add: reduce .[] as $x (null; . + $x);",
   "def del(f): delpaths([path(f)]);",
-  "def _assign(paths; value): value as $v | fold . as $obj (path(paths) as $p | $obj | setpath($p; $v));",
-  "def _modify(paths; update): fold . as $obj (path(paths) as $p | $obj | setpath($p; getpath($p) | update));",
+  "def _assign(paths; value): value as $v | reduce path(paths) as $p (.; setpath($p; $v));",
+  "def _modify(paths; update): reduce path(paths) as $p (.; setpath($p; getpath($p) | update));",
   "def recurse(f): ., (f | select(. != null) | recurse(f));",
+  "def to_entries: [keys[] as $k | {key: $k, value: .[$k]}];",
+  "def from_entries: map({(.key): .value}) | add;",
+  "def with_entries(f): to_entries | map(f) | from_entries;",
+  "def reverse: [.[length - 1 - range(0;length)]];",
 };
 
 
-block builtins_bind(block b) {
-  for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) {
-    struct locfile src;
-    locfile_init(&src, jq_builtins[i], strlen(jq_builtins[i]));
-    block funcs;
-    int nerrors = jq_parse_library(&src, &funcs);
-    assert(!nerrors);
-    b = block_bind(funcs, b, OP_IS_CALL_PSEUDO);
-    locfile_free(&src);
+int builtins_bind_one(block* bb, const char* code) {
+  struct locfile src;
+  locfile_init(&src, code, strlen(code));
+  block funcs;
+  int nerrors = jq_parse_library(&src, &funcs);
+  if (nerrors == 0) {
+    *bb = block_bind_referenced(funcs, *bb, OP_IS_CALL_PSEUDO);
   }
-  b = bind_bytecoded_builtins(b);
-  return gen_cbinding(function_list, sizeof(function_list)/sizeof(function_list[0]), b);
+  locfile_free(&src);
+  return nerrors;
+}
+
+int slurp_lib(block* bb) {
+  int nerrors = 0;
+  char* home = getenv("HOME");
+  if (home) {    // silently ignore no $HOME
+    jv filename = jv_string_append_str(jv_string(home), "/.jq");
+    jv data = jv_load_file(jv_string_value(filename), 1);
+    if (jv_is_valid(data)) {
+      nerrors = builtins_bind_one(bb, jv_string_value(data) );
+    }
+    jv_free(filename);
+    jv_free(data);
+  }
+  return nerrors;
+}
+
+int builtins_bind(block* bb) {
+  int nerrors = slurp_lib(bb);
+  if (nerrors) {
+    block_free(*bb);
+    return nerrors;
+  }
+  for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) {
+    nerrors = builtins_bind_one(bb, jq_builtins[i]);
+    assert(!nerrors);
+  }
+  *bb = bind_bytecoded_builtins(*bb);
+  *bb = gen_cbinding(function_list, sizeof(function_list)/sizeof(function_list[0]), *bb);
+  return nerrors;
 }

builtin.h

@@ -1,21 +1,9 @@
 #ifndef BUILTIN_H
 #define BUILTIN_H
 
+#include "bytecode.h"
 #include "compile.h"
 
-block builtins_bind(block);
-
-
-typedef void (*cfunction_ptr)(void);
-
-struct cfunction {
-  cfunction_ptr fptr;
-  const char* name;
-  int nargs;
-};
-
-
-jv cfunction_invoke(struct cfunction* function, jv input[]);
-
+int builtins_bind(block*);
 
 #endif

bytecode.c

@@ -3,9 +3,39 @@
 #include <stdlib.h>
 
 #include "bytecode.h"
-#include "opcode.h"
 #include "jv_alloc.h"
 
+// flags, length
+#define NONE 0, 1
+#define CONSTANT OP_HAS_CONSTANT, 2
+#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
+#define BRANCH OP_HAS_BRANCH, 2
+#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
+#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
+#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
+#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
+
+#define OP(name, imm, in, out) \
+  {name, #name, imm, in, out},
+
+static const struct opcode_description opcode_descriptions[] = {
+#include "opcode_list.h"
+};
+
+static const struct opcode_description invalid_opcode_description = {
+  -1, "#INVALID", 0, 0, 0, 0
+};
+
+
+const struct opcode_description* opcode_describe(opcode op) {
+  if ((int)op >= 0 && (int)op < NUM_OPCODES) {
+    return &opcode_descriptions[op];
+  } else {
+    return &invalid_opcode_description;
+  }
+}
+
+
 static int bytecode_operation_length(uint16_t* codeptr) {
   int length = opcode_describe(*codeptr)->length;
   if (*codeptr == CALL_JQ) {
@@ -14,18 +44,7 @@ static int bytecode_operation_length(uint16_t* codeptr) {
   return length;
 }
 
-void dump_disassembly(int indent, struct bytecode* bc) {
-  if (bc->nclosures > 0) {
-    printf("%*s[params: %d]\n", indent, "", bc->nclosures);
-  }
-  dump_code(indent, bc);
-  for (int i=0; i<bc->nsubfunctions; i++) {
-    printf("%*ssubfn[%d]:\n", indent, "", i);
-    dump_disassembly(indent+2, bc->subfunctions[i]);
-  }
-}
-
-void dump_code(int indent, struct bytecode* bc) {
+static void dump_code(int indent, struct bytecode* bc) {
   int pc = 0;
   while (pc < bc->codelen) {
     printf("%*s", indent, "");
@@ -35,6 +54,43 @@ void dump_code(int indent, struct bytecode* bc) {
   }
 }
 
+static void symbol_table_free(struct symbol_table* syms) {
+  jv_mem_free(syms->cfunctions);
+  jv_free(syms->cfunc_names);
+  jv_mem_free(syms);
+}
+
+void dump_disassembly(int indent, struct bytecode* bc) {
+  if (bc->nclosures > 0) {
+    printf("%*s[params: ", indent, "");
+    jv params = jv_object_get(jv_copy(bc->debuginfo), jv_string("params"));
+    for (int i=0; i<bc->nclosures; i++) {
+      if (i) printf(", ");
+      jv name = jv_array_get(jv_copy(params), i);
+      printf("%s", jv_string_value(name));
+      jv_free(name);
+    }
+    jv_free(params);
+    printf("]\n");
+  }
+  dump_code(indent, bc);
+  for (int i=0; i<bc->nsubfunctions; i++) {
+    struct bytecode* subfn = bc->subfunctions[i];
+    jv name = jv_object_get(jv_copy(subfn->debuginfo), jv_string("name"));
+    printf("%*s%s:%d:\n", indent, "", jv_string_value(name), i);
+    jv_free(name);
+    dump_disassembly(indent+2, subfn);
+  }
+}
+
+static struct bytecode* getlevel(struct bytecode* bc, int level) {
+  while (level > 0) {
+    bc = bc->parent;
+    level--;
+  }
+  return bc;
+}
+
 void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
   int pc = codeptr - bc->code;
   printf("%04d ", pc);
@@ -46,15 +102,28 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
       for (int i=0; i<imm+1; i++) {
         uint16_t level = bc->code[pc++];
         uint16_t idx = bc->code[pc++];
+        jv name;
         if (idx & ARG_NEWCLOSURE) {
-          printf(" subfn[%d]", idx & ~ARG_NEWCLOSURE);
+          idx &= ~ARG_NEWCLOSURE;
+          name = jv_object_get(jv_copy(getlevel(bc,level)->subfunctions[idx]->debuginfo), 
+                               jv_string("name"));
         } else {
-          printf(" param[%d]", idx);
+          name = jv_array_get(jv_object_get(jv_copy(getlevel(bc,level)->debuginfo), 
+                                            jv_string("params")), idx);
         }
+        printf(" %s:%d",
+               jv_string_value(name),
+               idx);
+        jv_free(name);
         if (level) {
           printf("^%d", level);
         }
       }
+    } else if (op->op == CALL_BUILTIN) {
+      int func = bc->code[pc++];
+      jv name = jv_array_get(jv_copy(bc->globals->cfunc_names), func);
+      printf(" %s", jv_string_value(name));
+      jv_free(name);
     } else if (op->flags & OP_HAS_BRANCH) {
       printf(" %04d", pc + imm);
     } else if (op->flags & OP_HAS_CONSTANT) {
@@ -62,7 +131,11 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
       jv_dump(jv_array_get(jv_copy(bc->constants), imm), 0);
     } else if (op->flags & OP_HAS_VARIABLE) {
       uint16_t v = bc->code[pc++];
-      printf(" v%d", v);
+      jv name = jv_array_get(jv_object_get(jv_copy(getlevel(bc,imm)->debuginfo), jv_string("locals")), v);
+      printf(" $%s:%d", 
+             jv_string_value(name), 
+             v);
+      jv_free(name);
       if (imm) {
         printf("^%d", imm);
       }
@@ -72,12 +145,9 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
   }  
 }
 
-void symbol_table_free(struct symbol_table* syms) {
-  jv_mem_free(syms->cfunctions);
-  jv_mem_free(syms);
-}
-
 void bytecode_free(struct bytecode* bc) {
+  if (!bc)
+    return;
   jv_mem_free(bc->code);
   jv_free(bc->constants);
   for (int i=0; i<bc->nsubfunctions; i++)
@@ -85,5 +155,6 @@ void bytecode_free(struct bytecode* bc) {
   if (!bc->parent)
     symbol_table_free(bc->globals);
   jv_mem_free(bc->subfunctions);
+  jv_free(bc->debuginfo);
   jv_mem_free(bc);
 }

bytecode.h

@@ -3,13 +3,56 @@
 #include <stdint.h>
 
 #include "jv.h"
-#include "opcode.h"
-#include "builtin.h"
+
+typedef enum {
+#define OP(name, imm, in, out) name,
+#include "opcode_list.h"
+#undef OP
+} opcode;
+
+enum {
+  NUM_OPCODES = 
+#define OP(name, imm, in, out) +1
+#include "opcode_list.h"
+#undef OP
+};
+
+enum {
+  OP_HAS_CONSTANT = 2,
+  OP_HAS_VARIABLE = 4,
+  OP_HAS_BRANCH = 8,
+  OP_HAS_CFUNC = 32,
+  OP_HAS_UFUNC = 64,
+  OP_IS_CALL_PSEUDO = 128,
+  OP_HAS_BINDING = 1024,
+};
+struct opcode_description {
+  opcode op;
+  const char* name;
+
+  int flags;
+
+  // length in 16-bit units
+  int length;
+
+  int stack_in, stack_out;
+};
+
+const struct opcode_description* opcode_describe(opcode op);
+
 
 #define MAX_CFUNCTION_ARGS 10
+typedef void (*cfunction_ptr)();
+struct cfunction {
+  cfunction_ptr fptr;
+  const char* name;
+  int nargs;
+};
+
 struct symbol_table {
   struct cfunction* cfunctions;
   int ncfunctions;
+  jv cfunc_names;
 };
 
 // The bytecode format matters in:
@@ -33,13 +76,13 @@ struct bytecode {
   int nsubfunctions;
 
   struct bytecode* parent;
+
+  jv debuginfo;
 };
 
 void dump_disassembly(int, struct bytecode* code);
-void dump_code(int, struct bytecode* code);
 void dump_operation(struct bytecode* bc, uint16_t* op);
 
-void symbol_table_free(struct symbol_table* syms);
 void bytecode_free(struct bytecode* bc);
 
 #endif

compile.c

@@ -2,7 +2,6 @@
 #include <assert.h>
 #include <string.h>
 #include <stdlib.h>
-#include "opcode.h"
 #include "compile.h"
 #include "bytecode.h"
 #include "locfile.h"
@@ -153,33 +152,26 @@ void inst_set_target(block b, block target) {
   b.first->imm.target = target.last;
 }
 
-block gen_op_var_unbound(opcode op, const char* name) {
+block gen_op_unbound(opcode op, const char* name) {
+  assert(opcode_describe(op)->flags & OP_HAS_BINDING);
+  inst* i = inst_new(op);
+  i->symbol = strdup(name);
+  return inst_block(i);
+}
+
+block gen_op_var_fresh(opcode op, const char* name) {
   assert(opcode_describe(op)->flags & OP_HAS_VARIABLE);
-  inst* i = inst_new(op);
-  i->symbol = strdup(name);
-  return inst_block(i);
+  return block_bind(gen_op_unbound(op, name),
+                    gen_noop(), OP_HAS_VARIABLE);
 }
 
-block gen_op_var_bound(opcode op, block binder) {
+block gen_op_bound(opcode op, block binder) {
   assert(block_is_single(binder));
-  block b = gen_op_var_unbound(op, binder.first->symbol);
+  block b = gen_op_unbound(op, binder.first->symbol);
   b.first->bound_by = binder.first;
   return b;
 }
 
-block gen_op_block_unbound(opcode op, const char* name) {
-  assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO);
-  inst* i = inst_new(op);
-  i->symbol = strdup(name);
-  return inst_block(i);
-}
-
-block gen_op_block_bound(opcode op, block binder) {
-  assert(block_is_single(binder));
-  block b = gen_op_block_unbound(op, binder.first->symbol);
-  b.first->bound_by = binder.first;
-  return b;
-}
 
 static void inst_join(inst* a, inst* b) {
   assert(a && b);
@@ -216,13 +208,14 @@ int block_has_only_binders(block binders, int bindflags) {
   return 1;
 }
 
-static void block_bind_subblock(block binder, block body, int bindflags) {
+static int block_bind_subblock(block binder, block body, int bindflags) {
   assert(block_is_single(binder));
   assert((opcode_describe(binder.first->op)->flags & bindflags) == bindflags);
   assert(binder.first->symbol);
   assert(binder.first->bound_by == 0 || binder.first->bound_by == binder.first);
 
   binder.first->bound_by = binder.first;
+  int nrefs = 0;
   for (inst* i = body.first; i; i = i->next) {
     int flags = opcode_describe(i->op)->flags;
     if ((flags & bindflags) == bindflags &&
@@ -230,12 +223,14 @@ static void block_bind_subblock(block binder, block body, int bindflags) {
         !strcmp(i->symbol, binder.first->symbol)) {
       // bind this instruction
       i->bound_by = binder.first;
+      nrefs++;
     }
     // binding recurses into closures
-    block_bind_subblock(binder, i->subfn, bindflags);
+    nrefs += block_bind_subblock(binder, i->subfn, bindflags);
     // binding recurses into argument list
-    block_bind_subblock(binder, i->arglist, bindflags);
+    nrefs += block_bind_subblock(binder, i->arglist, bindflags);
   }
+  return nrefs;
 }
 
 static void block_bind_each(block binder, block body, int bindflags) {
@@ -251,6 +246,21 @@ block block_bind(block binder, block body, int bindflags) {
   return block_join(binder, body);
 }
 
+block block_bind_referenced(block binder, block body, int bindflags) {
+  assert(block_has_only_binders(binder, bindflags));
+  bindflags |= OP_HAS_BINDING;
+  block refd = gen_noop();
+  for (inst* curr; (curr = block_take(&binder));) {
+    block b = inst_block(curr);
+    if (block_bind_subblock(b, body, bindflags)) {
+      refd = BLOCK(refd, b);
+    } else {
+      block_free(b);
+    }
+  }
+  return block_join(refd, body);
+}
+
 block gen_function(const char* name, block formals, block body) {
   block_bind_each(formals, body, OP_IS_CALL_PSEUDO);
   inst* i = inst_new(CLOSURE_CREATE);
@@ -262,12 +272,16 @@ block gen_function(const char* name, block formals, block body) {
   return b;
 }
 
+block gen_param(const char* name) {
+  return gen_op_unbound(CLOSURE_PARAM, name);
+}
+
 block gen_lambda(block body) {
   return gen_function("@lambda", gen_noop(), body);
 }
 
 block gen_call(const char* name, block args) {
-  block b = gen_op_block_unbound(CALL_JQ, name);
+  block b = gen_op_unbound(CALL_JQ, name);
   b.first->arglist = args;
   return b;
 }
@@ -275,7 +289,7 @@ block gen_call(const char* name, block args) {
 
 
 block gen_subexp(block a) {
-  return BLOCK(gen_op_simple(DUP), a, gen_op_simple(SWAP));
+  return BLOCK(gen_op_simple(SUBEXP_BEGIN), a, gen_op_simple(SUBEXP_END));
 }
 
 block gen_both(block a, block b) {
@@ -288,41 +302,46 @@ block gen_both(block a, block b) {
 
 
 block gen_collect(block expr) {
-  block array_var = block_bind(gen_op_var_unbound(STOREV, "collect"),
-                               gen_noop(), OP_HAS_VARIABLE);
+  block array_var = gen_op_var_fresh(STOREV, "collect");
   block c = BLOCK(gen_op_simple(DUP), gen_const(jv_array()), array_var);
 
-  block tail = BLOCK(gen_op_var_bound(APPEND, array_var),
+  block tail = BLOCK(gen_op_bound(APPEND, array_var),
                      gen_op_simple(BACKTRACK));
 
   return BLOCK(c,
                gen_op_target(FORK, tail),
                expr, 
                tail,
-               gen_op_var_bound(LOADV, array_var));
+               gen_op_bound(LOADVN, array_var));
 }
 
-block gen_fold(const char* varname, block init, block fold) {
-  block loop = BLOCK(fold, gen_op_var_unbound(STOREV, varname), gen_op_simple(BACKTRACK));
+block gen_reduce(const char* varname, block source, block init, block body) {
+  block res_var = gen_op_var_fresh(STOREV, "reduce");
+  block loop = BLOCK(gen_op_simple(DUP),
+                     source,
+                     block_bind(gen_op_unbound(STOREV, varname),
+                                BLOCK(gen_op_bound(LOADVN, res_var),
+                                      body,
+                                      gen_op_bound(STOREV, res_var)),
+                                OP_HAS_VARIABLE),
+                     gen_op_simple(BACKTRACK));
   return BLOCK(gen_op_simple(DUP),
                init,
-               block_bind(gen_op_var_unbound(STOREV, varname),
-                          BLOCK(gen_op_target(FORK, loop),
-                                loop,
-                                gen_op_var_unbound(LOADV, varname)),
-                          OP_HAS_VARIABLE));
+               res_var,
+               gen_op_target(FORK, loop),
+               loop,
+               gen_op_bound(LOADVN, res_var));
 }
 
 block gen_definedor(block a, block b) {
   // var found := false
-  block found_var = block_bind(gen_op_var_unbound(STOREV, "found"),
-                               gen_noop(), OP_HAS_VARIABLE);
+  block found_var = gen_op_var_fresh(STOREV, "found");
   block init = BLOCK(gen_op_simple(DUP), gen_const(jv_false()), found_var);
 
   // if found, backtrack. Otherwise execute b
   block backtrack = gen_op_simple(BACKTRACK);
   block tail = BLOCK(gen_op_simple(DUP),
-                     gen_op_var_bound(LOADV, found_var),
+                     gen_op_bound(LOADV, found_var),
                      gen_op_target(JUMP_F, backtrack),
                      backtrack,
                      gen_op_simple(POP),
@@ -334,7 +353,7 @@ block gen_definedor(block a, block b) {
   // found := true, produce result
   block if_found = BLOCK(gen_op_simple(DUP),
                          gen_const(jv_true()),
-                         gen_op_var_bound(STOREV, found_var),
+                         gen_op_bound(STOREV, found_var),
                          gen_op_target(JUMP, tail));
 
   return BLOCK(init,
@@ -373,7 +392,7 @@ block gen_or(block a, block b) {
 
 block gen_var_binding(block var, const char* name, block body) {
   return BLOCK(gen_op_simple(DUP), var,
-               block_bind(gen_op_var_unbound(STOREV, name),
+               block_bind(gen_op_unbound(STOREV, name),
                           body, OP_HAS_VARIABLE));
 }
 
@@ -448,7 +467,7 @@ static int expand_call_arglist(struct locfile* locations, block* b) {
             break;
           case CLOSURE_CREATE:
             block_append(&prelude, b);
-            block_append(&callargs, gen_op_block_bound(CLOSURE_REF, b));
+            block_append(&callargs, gen_op_bound(CLOSURE_REF, b));
             break;
           }
           actual_args++;
@@ -507,13 +526,8 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
   int var_frame_idx = 0;
   bc->nsubfunctions = 0;
   errors += expand_call_arglist(locations, &b);
-  if (bc->parent) {
-    // functions should end in a return
-    b = BLOCK(b, gen_op_simple(RET));
-  } else {
-    // the toplevel should YIELD;BACKTRACK; when it finds an answer
-    b = BLOCK(b, gen_op_simple(YIELD), gen_op_simple(BACKTRACK));
-  }
+  b = BLOCK(b, gen_op_simple(RET));
+  jv localnames = jv_array();
   for (inst* curr = b.first; curr; curr = curr->next) {
     if (!curr->next) assert(curr == b.last);
     int length = opcode_describe(curr->op)->length;
@@ -531,6 +545,7 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
     if ((opcode_describe(curr->op)->flags & OP_HAS_VARIABLE) &&
         curr->bound_by == curr) {
       curr->imm.intval = var_frame_idx++;
+      localnames = jv_array_append(localnames, jv_string(curr->symbol));
     }
 
     if (curr->op == CLOSURE_CREATE) {
@@ -540,10 +555,13 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
     if (curr->op == CLOSURE_CREATE_C) {
       assert(curr->bound_by == curr);
       int idx = bc->globals->ncfunctions++;
+      bc->globals->cfunc_names = jv_array_append(bc->globals->cfunc_names,
+                                                 jv_string(curr->symbol));
       bc->globals->cfunctions[idx] = *curr->imm.cfunc;
       curr->imm.intval = idx;
     }
   }
+  bc->debuginfo = jv_object_set(bc->debuginfo, jv_string("locals"), localnames);
   if (bc->nsubfunctions) {
     bc->subfunctions = jv_mem_alloc(sizeof(struct bytecode*) * bc->nsubfunctions);
     for (inst* curr = b.first; curr; curr = curr->next) {
@@ -553,12 +571,16 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
         subfn->globals = bc->globals;
         subfn->parent = bc;
         subfn->nclosures = 0;
+        subfn->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_string(curr->symbol));
+        jv params = jv_array();
         for (inst* param = curr->arglist.first; param; param = param->next) {
           assert(param->op == CLOSURE_PARAM);
           assert(param->bound_by == param);
           param->imm.intval = subfn->nclosures++;
           param->compiled = subfn;
+          params = jv_array_append(params, jv_string(param->symbol));
         }
+        subfn->debuginfo = jv_object_set(subfn->debuginfo, jv_string("params"), params);
         errors += compile(locations, subfn, curr->subfn);
         curr->subfn = gen_noop();
       }
@@ -626,6 +648,8 @@ int block_compile(block b, struct locfile* locations, struct bytecode** out) {
   int ncfunc = count_cfunctions(b);
   bc->globals->ncfunctions = 0;
   bc->globals->cfunctions = jv_mem_alloc(sizeof(struct cfunction) * ncfunc);
+  bc->globals->cfunc_names = jv_array();
+  bc->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_null());
   int nerrors = compile(locations, bc, b);
   assert(bc->globals->ncfunctions == ncfunc);
   if (nerrors > 0) {

compile.h

@@ -2,13 +2,9 @@
 #define COMPILE_H
 #include <stdint.h>
 #include "jv.h"
-#include "opcode.h"
+#include "bytecode.h"
 #include "locfile.h"
 
-struct bytecode;
-struct symbol_table;
-struct cfunction;
-
 struct inst;
 typedef struct inst inst;
 
@@ -24,17 +20,18 @@ block gen_noop();
 block gen_op_simple(opcode op);
 block gen_const(jv constant);
 block gen_op_target(opcode op, block target);
-block gen_op_var_unbound(opcode op, const char* name);
-block gen_op_var_bound(opcode op, block binder);
-block gen_op_block_unbound(opcode op, const char* name);
+block gen_op_unbound(opcode op, const char* name);
+block gen_op_bound(opcode op, block binder);
+block gen_op_var_fresh(opcode op, const char* name);
 
 block gen_function(const char* name, block formals, block body);
+block gen_param(const char* name);
 block gen_lambda(block body);
 block gen_call(const char* name, block body);
 block gen_subexp(block a);
 block gen_both(block a, block b);
 block gen_collect(block expr);
-block gen_fold(const char* varname, block init, block body);
+block gen_reduce(const char* varname, block source, block init, block body);
 block gen_definedor(block a, block b);
 block gen_condbranch(block iftrue, block iffalse);
 block gen_and(block a, block b);
@@ -50,6 +47,7 @@ void block_append(block* b, block b2);
 block block_join(block a, block b);
 int block_has_only_binders(block, int bindflags);
 block block_bind(block binder, block body, int bindflags);
+block block_bind_referenced(block binder, block body, int bindflags);
 
 int block_compile(block, struct locfile*, struct bytecode**);
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([jq], [1.2], [mu@netsoc.tcd.ie],
+AC_INIT([jq], [1.3], [mu@netsoc.tcd.ie],
              [jq], [http://stedolan.github.com/jq/])
 
 dnl Created autoconf implementation thompson@dtosolutions, 26NOV12
@@ -39,10 +39,13 @@ AC_ARG_ENABLE([docs],
 
 AS_IF([test "x$enable_docs" != "xno"],[
    AC_CHECK_PROGS(bundle_cmd, bundle)
-   AS_IF([test "x$bundle_cmd" = "x" || \
-      ! ( cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null )],[
-         AC_MSG_WARN([no bundler])
-         cat <<EOF
+
+   AC_CACHE_CHECK([for Ruby dependencies], [jq_cv_ruby_deps],
+     [jq_cv_ruby_deps=yes;
+        AS_IF([test "x$bundle_cmd" = "x" || \
+        ! bmsg="`cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null`"],[
+           AC_MSG_WARN([$bmsg])
+           cat <<EOF
 *****************************************************************
 *  Ruby dependencies for building jq documentation not found.   *
 *  You can still build, install and hack on jq, but the manpage *
@@ -50,8 +53,12 @@ AS_IF([test "x$enable_docs" != "xno"],[
 *  See docs/README.md for how to install the docs dependencies. *
 *****************************************************************
 EOF
-         enable_docs=no
-   ])
+           jq_cv_ruby_deps=no
+     ])])
+
+   if test "x$jq_cv_ruby_deps" != "xyes"; then
+     enable_docs=no
+   fi
 ])
 
 AM_CONDITIONAL([ENABLE_DOCS], [test "x$enable_docs" != xno])

docs/content/2.download/default.yml

@@ -27,7 +27,7 @@ body:
          manually by following instructions on [Arch Linux's
          Wiki](https://wiki.archlinux.org/index.php/Arch_UseRepository).
 
-       * Debian/Ubuntu packages are coming Real Soon Now.
+       * jq 1.2 is [in Debian](http://packages.debian.org/jq)
 
       ### OS X
 
@@ -42,18 +42,28 @@ body:
 
       ### From source on Linux or OS X
 
+       * [Source tarball for jq $JQ_VERSION](source/jq-$JQ_VERSION.tar.gz)
+
+      You can build it using the usual `./configure && make && sudo
+      make install` rigmarole.
+
+      If you're interested in using the lastest development version, try:
+
           git clone https://github.com/stedolan/jq.git
           cd jq
-          make && sudo make install
+          autoreconf
+          ./configure
+          make
+          sudo make install
 
       To build it from a git clone, you'll need to install a few
       packages first:
 
        * [Flex](http://www.gnu.org/software/flex/)
        * [Bison](http://www.gnu.org/software/bison/)
-       * [Python](http://www.python.org)
        * [GCC](http://gcc.gnu.org)
        * [Make](http://www.gnu.org/software/make)
+       * [Autotools](http://www.gnu.org/software/automake)
 
       For Linux systems, these will all be in your system's package
       manager, and if you do development on the machine they're most
@@ -73,6 +83,8 @@ body:
 
       jq's documentation is compiled into static HTML using
       [Bonsai](http://www.tinytree.info). To view the documentation
-      locally, run `rake serve` from the docs/ subdirectory.
+      locally, run `rake serve` from the docs/ subdirectory. You'll
+      need a few Ruby dependencies, which can be installed by
+      following the instructions in `docs/README.md`.
 
 

docs/content/3.manual/manual.yml

@@ -113,7 +113,11 @@ sections:
         "\u03bc"). Using this option, you can force jq to produce pure
         ASCII output with every non-ASCII character replaced with the
         equivalent escape sequence.
-  
+
+      * `--sort-keys` / `-S`:
+        
+        Output the fields of each object with the keys in sorted order.
+
       * `--raw-output` / `-r`:
   
         With this option, if the filter's result is a string then it
@@ -127,6 +131,14 @@ sections:
         variable. If you run jq with `--arg foo bar`, then `$foo` is
         available in the program and has the value `"bar"`.
 
+      * `--argfile name filename`:
+
+        This option passes the first value from the named file as a
+        value to the jq program as a predefined variable. If you run jq
+        with `--argfile foo bar`, then `$foo` is available in the
+        program and has the value resulting from parsing the content of
+        the file named `bar`.
+
   - title: Basic filters
     entries:
       - title: "`.`"
@@ -148,10 +160,13 @@ sections:
       - title: "`.foo`"
         body: |
           
-          The simplest *useful* filter is .foo. When given a
+          The simplest *useful* filter is `.foo`. When given a
           JSON object (aka dictionary or hash) as input, it produces
           the value at the key "foo", or null if there's none present.
 
+          If the key contains special characters, you need to surround
+          it with double quotes like this: `."foo$"`.
+          
         examples:
           - program: '.foo'
             input: '{"foo": 42, "bar": "less interesting data"}'
@@ -159,8 +174,11 @@ sections:
           - program: '.foo'
             input: '{"notfoo": true, "alsonotfoo": false}'
             output: ['null']
+          - program: '."foo"'
+            input: '{"foo": 42}'
+            output: [42]
 
-      - title: "`.[foo]`"
+      - title: "`.[foo]`, `.[2]`, `.[10:15]`"
         body: |
           
           You can also look up fields of an object using syntax like
@@ -169,6 +187,13 @@ sections:
           integer. Arrays are zero-based (like javascript), so `.[2]`
           returns the third element of the array.
 
+          The `.[10:15]` syntax can be used to return a subarray of an
+          array. The array returned by `.[10:15]` will be of length 5,
+          containing the elements from index 10 (inclusive) to index
+          15 (exclusive). Either index may be negative (in which case
+          it counts backwards from the end of the array), or omitted
+          (in which case it refers to the start or end of the array).
+
         examples:
           - program: '.[0]'
             input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
@@ -178,6 +203,18 @@ sections:
             input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
             output: ['null']
 
+          - program: '.[2:4]'
+            input: '["a","b","c","d","e"]'
+            output: ['["c", "d"]']
+          
+          - program: '.[:3]'
+            input: '["a","b","c","d","e"]'
+            output: ['["a", "b", "c"]']
+
+          - program: '.[-2:]'
+            input: '["a","b","c","d","e"]'
+            output: ['["d", "e"]']
+
       - title: "`.[]`"
         body: |
           
@@ -202,8 +239,8 @@ sections:
             output: []
 
           - program: '.[]'
-            input: '{"a": 1, "b": 2}'
-            output: ['[1, 2]']
+            input: '{"a": 1, "b": 1}'
+            output: ['1', '1']
 
       - title: "`,`"
         body: |
@@ -478,6 +515,31 @@ sections:
             input: '[[0,1], ["a","b","c"]]'
             output: ['[false, true]']
 
+      - title: `to_entries`, `from_entries`, `with_entries`
+        body: |
+          
+          These functions convert between an object and an array of
+          key-value pairs. If `to_entries` is passed an object, then
+          for each `k: v` entry in the input, the output array
+          includes `{"key": k, "value": v}`. 
+
+          `from_entries` does the opposite conversion, and
+          `with_entries(foo)` is a shorthand for `to_entries |
+          map(foo) | from_entries`, useful for doing some operation to
+          all keys and values of an object.
+
+        examples:
+          - program: 'to_entries'
+            input: '{"a": 1, "b": 2}'
+            output: ['[{"key":"a", "value":1}, {"key":"b", "value":2}]']
+          - program: 'from_entries'
+            input: '[{"key":"a", "value":1}, {"key":"b", "value":2}]'
+            output: ['{"a": 1, "b": 2}']
+          - program: 'with_entries(.key |= "KEY_" + .)'
+            input: '{"a": 1, "b": 2}'
+            output: ['{"KEY_a": 1, "KEY_b": 2}']
+
+
       - title: `select`
         body: |
           
@@ -544,6 +606,22 @@ sections:
           - program: add
             input: '[]'
             output: ["null"]
+
+      - title: `range`
+        body: |
+          
+          The `range` function produces a range of numbers. `range(4;10)`
+          produces 6 numbers, from 4 (inclusive) to 10 (exclusive). The numbers
+          are produced as separate outputs. Use `[range(4;10)]` to get a range as 
+          an array.
+
+        examples:
+          - program: 'range(2;4)'
+            input: 'null'
+            output: ['2', '3']
+          - program: '[range(2;4)]'
+            input: 'null'
+            output: ['[2,3]']
             
       - title: `tonumber`
         body: |
@@ -658,6 +736,16 @@ sections:
             input: '[1,2,5,3,5,3,1,3]'
             output: ['[1,2,3,5]']
 
+      - title: `reverse`
+        body: |
+          
+          This function reverses an array.
+
+        examples:
+          - program: 'reverse'
+            input: '[1,2,3,4]'
+            output: ['[4,3,2,1]']
+
       - title: `contains`
         body: |
 
@@ -801,7 +889,7 @@ sections:
 
           - program: '@sh "echo \(.)"'
             input: "\"O'Hara's Ale\""
-            output: ["\"echo 'O'\\''Hara'\\''s Ale\""]
+            output: ["\"echo 'O'\\\\''Hara'\\\\''s Ale'\""]
         
   - title: Conditionals and Comparisons
     entries:
@@ -898,9 +986,12 @@ sections:
           - program: '(true, false) or false'
             input: 'null'
             output: ['true', 'false']
-          - program: '(true, false) and (true, false)'
+#          - program: '(true, false) and (true, false)'
+#            input: 'null'
+#            output: ['true', 'false', 'false', 'false']
+          - program: '(true, true) and (true, false)'
             input: 'null'
-            output: ['true', 'false', 'false', 'false']
+            output: ['true', 'false', 'true', 'false']
           - program: '[true, false | not]'
             input: 'null'
             output: ['[false, true]']
@@ -943,7 +1034,7 @@ sections:
       (many jq functions such as `map` and `find` are in fact written
       in jq).
 
-      Finally, jq has a `fold` operation, which is very powerful but a
+      Finally, jq has a `reduce` operation, which is very powerful but a
       bit tricky. Again, it's mostly used internally, to define some
       useful bits of jq's standard library.
 
@@ -1040,7 +1131,7 @@ sections:
           If you want the value-argument behaviour for defining simple
           functions, you can just use a variable:
 
-              def addvalue(f): f as $value | map(, + $value);
+              def addvalue(f): f as $value | map(. + $value);
 
           With that definition, `addvalue(.foo)` will add the current
           input's `.foo` field to each element of the array.
@@ -1053,36 +1144,26 @@ sections:
             input: '[[1,2],[10,20]]'
             output: ['[[1,2,1,2], [10,20,1,2]]']
 
-      - title: Fold
+      - title: Reduce
         body: |
         
-          The `fold` syntax in jq allows you to combine all of the
+          The `reduce` syntax in jq allows you to combine all of the
           results of an expression by accumulating them into a single
           answer. As an example, we'll pass `[3,2,1]` to this expression:
 
-              fold 0 as $sum (.[] | $sum + .)
+              reduce .[] as $item (0; . + $item)
           
-          The variable `$sum` is first given the value `0`. The body
-          of the fold (i.e. `.[] | $sum + .`) is evaluated. `.[]`
-          produces three results, `3`, `2`, and `1`. For the first
-          one, `$sum + .` gives `3`.
+          For each result that `.[]` produces, `. + $item` is run to
+          accumulate a running total, starting from 0. In this
+          example, `.[]` produces the results 3, 2, and 1, so the
+          effect is similar to running something like this:
 
-          Having produced this answer, jq backtracks to find the next
-          result as per usual. However, this time, `$sum` is set to
-          the previous value of the body, so `$sum + .` gives
-          `5`. After the final backtracking, `$sum + .` gives
-          `6`. This final value is used as the value of the entire
-          `fold` expression, so the above filter returns `6`.
-
-          More formally, in order to evaluate `fold INIT as $VAR
-          (BODY)`, jq first sets `$VAR` to the value of `INIT`. It
-          then runs through `BODY`. Each time `BODY` produces a value,
-          `$VAR` is set to that value and jq backtracks to find the
-          next one. When `BODY` stops producing values, the final
-          value of `$VAR` is the result of the entire expression.
+              0 | (3 as $item | . + $item) | 
+                  (2 as $item | . + $item) |
+                  (1 as $item | . + $item)
 
         examples:
-          - program: 'fold 0 as $sum (.[] | $sum + .)'
+          - program: 'reduce .[] as $item (0; . + $item)'
             input: '[10,2,5,3]'
             output: ['20']
       

docs/content/index/index.yml

@@ -24,4 +24,11 @@ body3: |
   
 tail: |
   Go read the [tutorial](tutorial/) for more, or the [manual](manual/)
-  for *way* more.
+  for *way* more.
+
+news:
+  - date: 2013-05-19
+    body: |
+      
+      jq 1.3 (finally) released! Get it on the [download](download/) page.
+

docs/public/css/base.scss

@@ -89,4 +89,11 @@ h3 code {
 
 footer {
     text-align: center;
+}
+
+#news {
+    font-size: 12pt;
+    .date {
+          font-style: italic;
+    }
 }

docs/site.yml

@@ -4,7 +4,7 @@
 
 # This line is modified by the Makefile. To change the version number,
 # edit the Autoconf version number at the top of configure.ac
-jq_version: 1.2
+jq_version: "1.3"
 
 root: '/jq'
 footer: |

docs/templates/default.liquid

@@ -10,7 +10,7 @@
         <h1>{{headline}}</h1>
         {% for item in body %}
           {% if item.text %}
-            {{ item.text | markdownify }}
+            {{ item.text | replace: '$JQ_VERSION', jq_version | markdownify }}
           {% endif %}
         
           {% if item.command %}

docs/templates/index.liquid

@@ -35,6 +35,16 @@
       <div class="row" style="text-align:center; margin-top: 30px">
         {{tail | markdownify}}
       </div>
+      <div class="row">
+        <h2>News</h2>
+        <div id="news">
+          <ul>
+            {% for item in news %}
+            <li><span class="date">{{ item.date | date: '%m %B %Y' }}</span>{{item.body | markdownify}}</li>
+            {% endfor %}
+          </ul>
+        </div>
+      </div>
     </div>
     {% include "shared/footer" %}    
   </body>

exec_stack.h

@@ -0,0 +1,82 @@
+#ifndef EXEC_STACK_H
+#define EXEC_STACK_H
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct determine_alignment {
+  char x;
+  union { int i; double d; uint64_t u64; size_t sz; void* ptr; } u;
+};
+enum {ALIGNMENT = offsetof(struct determine_alignment, u)};
+
+static size_t align_round_up(size_t sz) {
+  return ((sz + (ALIGNMENT - 1)) / ALIGNMENT) * ALIGNMENT;
+}
+
+typedef int stack_ptr;
+
+struct stack {
+  char* mem_end; // one-past-the-end of allocated region
+  stack_ptr bound;
+  stack_ptr limit; // 0 - stack is empty
+};
+
+static void stack_init(struct stack* s) {
+  s->mem_end = 0;
+  s->bound = ALIGNMENT;
+  s->limit = 0;
+}
+
+static void stack_reset(struct stack* s) {
+  assert(s->limit == 0 && "stack freed while not empty");
+  char* mem_start = s->mem_end - ( -s->bound + ALIGNMENT);
+  free(mem_start);
+  stack_init(s);
+}
+
+static int stack_pop_will_free(struct stack* s, stack_ptr p) {
+  return p == s->limit;
+}
+
+static void* stack_block(struct stack* s, stack_ptr p) {
+  return (void*)(s->mem_end + p);
+}
+
+static stack_ptr* stack_block_next(struct stack* s, stack_ptr p) {
+  return &((stack_ptr*)stack_block(s, p))[-1];
+}
+
+static void stack_reallocate(struct stack* s, size_t sz) {
+  int old_mem_length = -(s->bound) + ALIGNMENT;
+  char* old_mem_start = s->mem_end - old_mem_length;
+  
+  int new_mem_length = align_round_up((old_mem_length + sz + 256) * 2);
+  char* new_mem_start = realloc(old_mem_start, new_mem_length);
+  memmove(new_mem_start + (new_mem_length - old_mem_length),
+            new_mem_start, old_mem_length);
+  s->mem_end = new_mem_start + new_mem_length;
+  s->bound = -(new_mem_length - ALIGNMENT);
+}
+
+static stack_ptr stack_push_block(struct stack* s, stack_ptr p, size_t sz) {
+  int alloc_sz = align_round_up(sz) + ALIGNMENT;
+  stack_ptr r = s->limit - alloc_sz;
+  if (r < s->bound) {
+    stack_reallocate(s, alloc_sz);
+  }
+  s->limit = r;
+  *stack_block_next(s, r) = p;
+  return r;
+}
+
+static stack_ptr stack_pop_block(struct stack* s, stack_ptr p, size_t sz) {
+  stack_ptr r = *stack_block_next(s, p);
+  if (p == s->limit) {
+    int alloc_sz = align_round_up(sz) + ALIGNMENT;
+    s->limit += alloc_sz;
+  }
+  return r;
+}
+#endif

execute.c

@@ -5,12 +5,9 @@
 
 #include "execute.h"
 
-#include "opcode.h"
+#include "exec_stack.h"
 #include "bytecode.h"
 
-#include "forkable_stack.h"
-#include "frame_layout.h"
-
 #include "jv_alloc.h"
 #include "jq_parser.h"
 #include "locfile.h"
@@ -19,128 +16,233 @@
 #include "parser.h"
 #include "builtin.h"
 
-typedef struct {
-  jv value;
-  int pathidx;
-} stackval;
-
 struct jq_state {
-  struct forkable_stack data_stk;
-  struct forkable_stack frame_stk;
-  struct forkable_stack fork_stk;
-  jv* pathbuf;
-  int pathsize; // number of allocated elements
+  void (*nomem_handler)(void *);
+  void *nomem_handler_data;
+  struct bytecode* bc;
+
+  struct stack stk;
+  stack_ptr curr_frame;
+  stack_ptr stk_top;
+  stack_ptr fork_top;
+
+  jv path;
+  int subexp_nest;
   int debug_trace_enabled;
+  int initial_execution;
 };
 
-int path_push(jq_state *jq, stackval sv, jv val) {
-  int pos = sv.pathidx;
-  assert(pos <= jq->pathsize);
-  assert(pos >= 0);
-  if (pos == jq->pathsize) {
-    int oldpathsize = jq->pathsize;
-    jq->pathsize = oldpathsize ? oldpathsize * 2 : 100;
-    jq->pathbuf = jv_mem_realloc(jq->pathbuf, sizeof(jq->pathbuf[0]) * jq->pathsize);
-    for (int i=oldpathsize; i<jq->pathsize; i++) {
-      jq->pathbuf[i] = jv_invalid();
+struct closure {
+  struct bytecode* bc;
+  stack_ptr env;
+};
+
+union frame_entry {
+  struct closure closure;
+  jv localvar;
+};
+
+struct frame {
+  struct bytecode* bc;
+  stack_ptr env;
+  stack_ptr retdata;
+  uint16_t* retaddr;
+  /* bc->nclosures closures followed by bc->nlocals local variables */
+  union frame_entry entries[0]; 
+};
+
+static int frame_size(struct bytecode* bc) {
+  return sizeof(struct frame) + sizeof(union frame_entry) * (bc->nclosures + bc->nlocals);
+}
+
+static struct frame* frame_current(struct jq_state* jq) {
+  struct frame* fp = stack_block(&jq->stk, jq->curr_frame);
+
+  stack_ptr next = *stack_block_next(&jq->stk, jq->curr_frame);
+  if (next) {
+    struct frame* fpnext = stack_block(&jq->stk, next);
+    struct bytecode* bc = fpnext->bc;
+    assert(fp->retaddr >= bc->code && fp->retaddr < bc->code + bc->codelen);
+  } else {
+    assert(fp->retaddr == 0);
+  }
+  return fp;
+}
+
+static stack_ptr frame_get_level(struct jq_state* jq, int level) {
+  stack_ptr fr = jq->curr_frame;
+  for (int i=0; i<level; i++) {
+    struct frame* fp = stack_block(&jq->stk, fr);
+    fr = fp->env;
+  }
+  return fr;
+}
+
+static jv* frame_local_var(struct jq_state* jq, int var, int level) {
+  struct frame* fr = stack_block(&jq->stk, frame_get_level(jq, level));
+  assert(var >= 0);
+  assert(var < fr->bc->nlocals);
+  return &fr->entries[fr->bc->nclosures + var].localvar;
+}
+
+static struct closure make_closure(struct jq_state* jq, uint16_t* pc) {
+  uint16_t level = *pc++;
+  uint16_t idx = *pc++;
+  stack_ptr fridx = frame_get_level(jq, level);
+  struct frame* fr = stack_block(&jq->stk, fridx);
+  if (idx & ARG_NEWCLOSURE) {
+    int subfn_idx = idx & ~ARG_NEWCLOSURE;
+    assert(subfn_idx < fr->bc->nsubfunctions);
+    struct closure cl = {fr->bc->subfunctions[subfn_idx],
+                         fridx};
+    return cl;
+  } else {
+    int closure = idx;
+    assert(closure >= 0);
+    assert(closure < fr->bc->nclosures);
+    return fr->entries[closure].closure;
+  }
+}
+
+static struct frame* frame_push(struct jq_state* jq, struct closure callee, 
+                                uint16_t* argdef, int nargs) {
+  stack_ptr new_frame_idx = stack_push_block(&jq->stk, jq->curr_frame, frame_size(callee.bc));
+  struct frame* new_frame = stack_block(&jq->stk, new_frame_idx);
+  new_frame->bc = callee.bc;
+  new_frame->env = callee.env;
+  assert(nargs == new_frame->bc->nclosures);
+  union frame_entry* entries = new_frame->entries;
+  for (int i=0; i<nargs; i++) {
+    entries->closure = make_closure(jq, argdef + i * 2);
+    entries++;
+  }
+  for (int i=0; i<callee.bc->nlocals; i++) {
+    entries->localvar = jv_invalid();
+    entries++;
+  }
+  jq->curr_frame = new_frame_idx;
+  return new_frame;
+}
+
+static void frame_pop(struct jq_state* jq) {
+  assert(jq->curr_frame);
+  struct frame* fp = frame_current(jq);
+  if (stack_pop_will_free(&jq->stk, jq->curr_frame)) {
+    int nlocals = fp->bc->nlocals;
+    for (int i=0; i<nlocals; i++) {
+      jv_free(*frame_local_var(jq, i, 0));
     }
   }
-  jv_free(jq->pathbuf[pos]);
-  jq->pathbuf[pos] = val;
-  return pos + 1;
+  jq->curr_frame = stack_pop_block(&jq->stk, jq->curr_frame, frame_size(fp->bc));
 }
 
-stackval stackval_replace(stackval value, jv newjs) {
-  jv_free(value.value);
-  stackval s = {newjs, value.pathidx};
-  return s;
+void stack_push(jq_state *jq, jv val) {
+  assert(jv_is_valid(val));
+  jq->stk_top = stack_push_block(&jq->stk, jq->stk_top, sizeof(jv));
+  jv* sval = stack_block(&jq->stk, jq->stk_top);
+  *sval = val;
 }
 
-
-// Probably all uses of this function are bugs
-stackval stackval_root(jv v) {
-  stackval s = {v, 0};
-  return s;
-}
-
-typedef struct {
-  FORKABLE_STACK_HEADER;
-  stackval sv;
-} data_stk_elem;
-
-void stack_push(jq_state *jq, stackval val) {
-  assert(jv_is_valid(val.value));
-  data_stk_elem* s = forkable_stack_push(&jq->data_stk, sizeof(data_stk_elem));
-  s->sv = val;
-}
-
-stackval stack_pop(jq_state *jq) {
-  data_stk_elem* s = forkable_stack_peek(&jq->data_stk);
-  stackval sv = s->sv;
-  if (!forkable_stack_pop_will_free(&jq->data_stk)) {
-    sv.value = jv_copy(sv.value);
+jv stack_pop(jq_state *jq) {
+  jv* sval = stack_block(&jq->stk, jq->stk_top);
+  jv val = *sval;
+  if (!stack_pop_will_free(&jq->stk, jq->stk_top)) {
+    val = jv_copy(val);
   }
-  forkable_stack_pop(&jq->data_stk);
-  assert(jv_is_valid(sv.value));
-  return sv;
+  jq->stk_top = stack_pop_block(&jq->stk, jq->stk_top, sizeof(jv));
+  assert(jv_is_valid(val));
+  return val;
 }
 
 
-
 struct forkpoint {
-  FORKABLE_STACK_HEADER;
-  struct forkable_stack_state saved_data_stack;
-  struct forkable_stack_state saved_call_stack;
+  stack_ptr saved_data_stack;
+  stack_ptr saved_curr_frame;
+  int path_len, subexp_nest;
+  uint16_t* return_address;
 };
 
+struct stack_pos {
+  stack_ptr saved_data_stack, saved_curr_frame;
+};
 
-void stack_save(jq_state *jq){
-  struct forkpoint* fork = forkable_stack_push(&jq->fork_stk, sizeof(struct forkpoint));
-  forkable_stack_save(&jq->data_stk, &fork->saved_data_stack);
-  forkable_stack_save(&jq->frame_stk, &fork->saved_call_stack);
+struct stack_pos stack_get_pos(jq_state* jq) {
+  struct stack_pos sp = {jq->stk_top, jq->curr_frame};
+  return sp;
 }
 
-void stack_switch(jq_state *jq) {
-  struct forkpoint* fork = forkable_stack_peek(&jq->fork_stk);
-  forkable_stack_switch(&jq->data_stk, &fork->saved_data_stack);
-  forkable_stack_switch(&jq->frame_stk, &fork->saved_call_stack);
+void stack_save(jq_state *jq, uint16_t* retaddr, struct stack_pos sp){
+  jq->fork_top = stack_push_block(&jq->stk, jq->fork_top, sizeof(struct forkpoint));
+  struct forkpoint* fork = stack_block(&jq->stk, jq->fork_top);
+  fork->saved_data_stack = jq->stk_top;
+  fork->saved_curr_frame = jq->curr_frame;
+  fork->path_len = 
+    jv_get_kind(jq->path) == JV_KIND_ARRAY ? jv_array_length(jv_copy(jq->path)) : 0;
+  fork->subexp_nest = jq->subexp_nest;
+  fork->return_address = retaddr;
+  jq->stk_top = sp.saved_data_stack;
+  jq->curr_frame = sp.saved_curr_frame;
 }
 
-int stack_restore(jq_state *jq){
-  while (!forkable_stack_empty(&jq->data_stk) && 
-         forkable_stack_pop_will_free(&jq->data_stk)) {
-    jv_free(stack_pop(jq).value);
+void path_append(jq_state* jq, jv component) {
+  if (jq->subexp_nest == 0 && jv_get_kind(jq->path) == JV_KIND_ARRAY) {
+    int n1 = jv_array_length(jv_copy(jq->path));
+    jq->path = jv_array_append(jq->path, component);
+    int n2 = jv_array_length(jv_copy(jq->path));
+    assert(n2 == n1 + 1);
+  } else {
+    jv_free(component);
   }
-  while (!forkable_stack_empty(&jq->frame_stk) && 
-         forkable_stack_pop_will_free(&jq->frame_stk)) {
-    frame_pop(&jq->frame_stk);
+}
+
+uint16_t* stack_restore(jq_state *jq){
+  while (!stack_pop_will_free(&jq->stk, jq->fork_top)) {
+    if (stack_pop_will_free(&jq->stk, jq->stk_top)) {
+      jv_free(stack_pop(jq));
+    } else if (stack_pop_will_free(&jq->stk, jq->curr_frame)) {
+      frame_pop(jq);
+    } else {
+      assert(0);
+    }
   }
-  if (forkable_stack_empty(&jq->fork_stk)) {
+
+  if (jq->fork_top == 0) {
     return 0;
-  } else {
-    struct forkpoint* fork = forkable_stack_peek(&jq->fork_stk);
-    forkable_stack_restore(&jq->data_stk, &fork->saved_data_stack);
-    forkable_stack_restore(&jq->frame_stk, &fork->saved_call_stack);
-    forkable_stack_pop(&jq->fork_stk);
-    return 1;
   }
+
+  struct forkpoint* fork = stack_block(&jq->stk, jq->fork_top);
+  uint16_t* retaddr = fork->return_address;
+  jq->stk_top = fork->saved_data_stack;
+  jq->curr_frame = fork->saved_curr_frame;
+  int path_len = fork->path_len;
+  if (jv_get_kind(jq->path) == JV_KIND_ARRAY) {
+    assert(path_len >= 0);
+    jq->path = jv_array_slice(jq->path, 0, path_len);
+  } else {
+    assert(path_len == 0);
+  }
+  jq->subexp_nest = fork->subexp_nest;
+  jq->fork_top = stack_pop_block(&jq->stk, jq->fork_top, sizeof(struct forkpoint));
+  return retaddr;
 }
 
-static struct closure make_closure(struct forkable_stack* stk, frame_ptr fr, uint16_t* pc) {
-  uint16_t level = *pc++;
-  uint16_t idx = *pc++;
-  fr = frame_get_level(stk, fr, level);
-  if (idx & ARG_NEWCLOSURE) {
-    int subfn_idx = idx & ~ARG_NEWCLOSURE;
-    assert(subfn_idx < frame_self(fr)->bc->nsubfunctions);
-    struct closure cl = {frame_self(fr)->bc->subfunctions[subfn_idx],
-                         forkable_stack_to_idx(stk, fr)};
-    return cl;
-  } else {
-    return *frame_closure_arg(fr, idx);
-  }
+static void jq_reset(jq_state *jq) {
+  while (stack_restore(jq)) {}
+
+  assert(jq->stk_top == 0);
+  assert(jq->fork_top == 0);
+  assert(jq->curr_frame == 0);
+  stack_reset(&jq->stk);
+
+  if (jv_get_kind(jq->path) != JV_KIND_INVALID)
+    jv_free(jq->path);
+  jq->path = jv_null();
+  jq->subexp_nest = 0;
 }
 
+
+
+
 void print_error(jv value) {
   assert(!jv_is_valid(value));
   jv msg = jv_invalid_get_msg(value);
@@ -154,39 +256,44 @@ void print_error(jv value) {
 jv jq_next(jq_state *jq) {
   jv cfunc_input[MAX_CFUNCTION_ARGS];
 
-  assert(!forkable_stack_empty(&jq->frame_stk));
-  uint16_t* pc = *frame_current_retaddr(&jq->frame_stk);
-  frame_pop(&jq->frame_stk);
-  
-  assert(!forkable_stack_empty(&jq->frame_stk));
+  jv_nomem_handler(jq->nomem_handler, jq->nomem_handler_data);
 
-  int backtracking = 0;
+  uint16_t* pc = stack_restore(jq);
+  assert(pc);
+
+  int backtracking = !jq->initial_execution;
+  jq->initial_execution = 0;
   while (1) {
     uint16_t opcode = *pc;
 
     if (jq->debug_trace_enabled) {
-      dump_operation(frame_current_bytecode(&jq->frame_stk), pc);
+      dump_operation(frame_current(jq)->bc, pc);
       printf("\t");
       const struct opcode_description* opdesc = opcode_describe(opcode);
-      data_stk_elem* param = 0;
-      int stack_in = opdesc->stack_in;
-      if (stack_in == -1) stack_in = pc[1];
-      for (int i=0; i<stack_in; i++) {
-        if (i == 0) {
-          param = forkable_stack_peek(&jq->data_stk);
-        } else {
-          printf(" | ");
-          param = forkable_stack_peek_next(&jq->data_stk, param);
+      stack_ptr param = 0;
+      if (!backtracking) {
+        int stack_in = opdesc->stack_in;
+        if (stack_in == -1) stack_in = pc[1];
+        for (int i=0; i<stack_in; i++) {
+          if (i == 0) {
+            param = jq->stk_top;
+          } else {
+            printf(" | ");
+            param = *stack_block_next(&jq->stk, param);
+          }
+          if (!param) break;
+          jv_dump(jv_copy(*(jv*)stack_block(&jq->stk, param)), 0);
+          //printf("<%d>", jv_get_refcnt(param->val));
+          //printf(" -- ");
+          //jv_dump(jv_copy(jq->path), 0);
         }
-        if (!param) break;
-        jv_dump(jv_copy(param->sv.value), 0);
-        printf("<%d>", jv_get_refcnt(param->sv.value));
+      } else {
+        printf("\t<backtracking>");
       }
 
-      if (backtracking) printf("\t<backtracking>");
-
       printf("\n");
     }
+
     if (backtracking) {
       opcode = ON_BACKTRACK(opcode);
       backtracking = 0;
@@ -197,67 +304,107 @@ jv jq_next(jq_state *jq) {
     default: assert(0 && "invalid instruction");
 
     case LOADK: {
-      jv v = jv_array_get(jv_copy(frame_current_bytecode(&jq->frame_stk)->constants), *pc++);
+      jv v = jv_array_get(jv_copy(frame_current(jq)->bc->constants), *pc++);
       assert(jv_is_valid(v));
-      stack_push(jq, stackval_replace(stack_pop(jq), v));
+      jv_free(stack_pop(jq));
+      stack_push(jq, v);
       break;
     }
 
     case DUP: {
-      stackval v = stack_pop(jq);
-      stackval v2 = v;
-      v2.value = jv_copy(v.value);
+      jv v = stack_pop(jq);
+      stack_push(jq, jv_copy(v));
       stack_push(jq, v);
-      stack_push(jq, v2);
       break;
     }
 
     case DUP2: {
-      stackval keep = stack_pop(jq);
-      stackval v = stack_pop(jq);
-      stackval v2 = v;
-      v2.value = jv_copy(v.value);
-      stack_push(jq, v);
+      jv keep = stack_pop(jq);
+      jv v = stack_pop(jq);
+      stack_push(jq, jv_copy(v));
       stack_push(jq, keep);
-      stack_push(jq, v2);
+      stack_push(jq, v);
       break;
     }
 
-    case SWAP: {
-      stackval a = stack_pop(jq);
-      stackval b = stack_pop(jq);
+    case SUBEXP_BEGIN: {
+      jv v = stack_pop(jq);
+      stack_push(jq, jv_copy(v));
+      stack_push(jq, v);
+      jq->subexp_nest++;
+      break;
+    }
+
+    case SUBEXP_END: {
+      assert(jq->subexp_nest > 0);
+      jq->subexp_nest--;
+      jv a = stack_pop(jq);
+      jv b = stack_pop(jq);
       stack_push(jq, a);
       stack_push(jq, b);
       break;
     }
       
     case POP: {
-      jv_free(stack_pop(jq).value);
+      jv_free(stack_pop(jq));
       break;
     }
 
     case APPEND: {
-      // FIXME paths
-      jv v = stack_pop(jq).value;
+      jv v = stack_pop(jq);
       uint16_t level = *pc++;
       uint16_t vidx = *pc++;
-      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
-      jv* var = frame_local_var(fp, vidx);
+      jv* var = frame_local_var(jq, vidx, level);
       assert(jv_get_kind(*var) == JV_KIND_ARRAY);
       *var = jv_array_append(*var, v);
       break;
     }
 
     case INSERT: {
-      stackval stktop = stack_pop(jq);
-      jv v = stack_pop(jq).value;
-      jv k = stack_pop(jq).value;
-      stackval objv = stack_pop(jq);
-      assert(jv_get_kind(k) == JV_KIND_STRING);
-      assert(jv_get_kind(objv.value) == JV_KIND_OBJECT);
-      objv.value = jv_object_set(objv.value, k, v);
-      stack_push(jq, objv);
-      stack_push(jq, stktop);
+      jv stktop = stack_pop(jq);
+      jv v = stack_pop(jq);
+      jv k = stack_pop(jq);
+      jv objv = stack_pop(jq);
+      assert(jv_get_kind(objv) == JV_KIND_OBJECT);
+      if (jv_get_kind(k) == JV_KIND_STRING) {
+        stack_push(jq, jv_object_set(objv, k, v));
+        stack_push(jq, stktop);
+      } else {
+        print_error(jv_invalid_with_msg(jv_string_fmt("Cannot use %s as object key",
+                                                      jv_kind_name(jv_get_kind(k)))));
+        jv_free(stktop);
+        jv_free(v);
+        jv_free(k);
+        jv_free(objv);
+        goto do_backtrack;
+      }
+      break;
+    }
+
+    case ON_BACKTRACK(RANGE):
+    case RANGE: {
+      uint16_t level = *pc++;
+      uint16_t v = *pc++;
+      jv* var = frame_local_var(jq, v, level);
+      jv max = stack_pop(jq);
+      if (jv_get_kind(*var) != JV_KIND_NUMBER ||
+          jv_get_kind(max) != JV_KIND_NUMBER) {
+        print_error(jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric")));
+        jv_free(max);
+        goto do_backtrack;
+      } else if (jv_number_value(jv_copy(*var)) >= jv_number_value(jv_copy(max))) {
+        /* finished iterating */
+        goto do_backtrack;
+      } else {
+        jv curr = jv_copy(*var);
+        *var = jv_number(jv_number_value(*var) + 1);
+
+        struct stack_pos spos = stack_get_pos(jq);
+        stack_push(jq, jv_copy(max));
+        stack_save(jq, pc - 3, spos);
+
+        stack_push(jq, curr);
+      }
       break;
     }
 
@@ -265,55 +412,94 @@ jv jq_next(jq_state *jq) {
     case LOADV: {
       uint16_t level = *pc++;
       uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
-      jv* var = frame_local_var(fp, v);
+      jv* var = frame_local_var(jq, v, level);
       if (jq->debug_trace_enabled) {
         printf("V%d = ", v);
         jv_dump(jv_copy(*var), 0);
         printf("\n");
       }
-      stack_push(jq, stackval_replace(stack_pop(jq), jv_copy(*var)));
+      jv_free(stack_pop(jq));
+      stack_push(jq, jv_copy(*var));
+      break;
+    }
+
+      // Does a load but replaces the variable with null
+    case LOADVN: {
+      uint16_t level = *pc++;
+      uint16_t v = *pc++;
+      jv* var = frame_local_var(jq, v, level);
+      if (jq->debug_trace_enabled) {
+        printf("V%d = ", v);
+        jv_dump(jv_copy(*var), 0);
+        printf("\n");
+      }
+      jv_free(stack_pop(jq));
+      stack_push(jq, *var);
+      *var = jv_null();
       break;
     }
 
     case STOREV: {
       uint16_t level = *pc++;
       uint16_t v = *pc++;
-      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
-      jv* var = frame_local_var(fp, v);
-      stackval val = stack_pop(jq);
+      jv* var = frame_local_var(jq, v, level);
+      jv val = stack_pop(jq);
       if (jq->debug_trace_enabled) {
         printf("V%d = ", v);
-        jv_dump(jv_copy(val.value), 0);
+        jv_dump(jv_copy(val), 0);
         printf("\n");
       }
       jv_free(*var);
-      *var = val.value;
+      *var = val;
       break;
     }
 
-    case GETPATH: {
-      stackval path_end = stack_pop(jq);
-      stackval path_start = stack_pop(jq);
-      jv_free(path_end.value);
-      jv path = jv_array();
-      for (int i=path_start.pathidx; i<path_end.pathidx; i++) {
-        path = jv_array_set(path, i, jv_copy(jq->pathbuf[i]));
-      }
-      stack_push(jq, stackval_replace(path_start, path));
+    case PATH_BEGIN: {
+      jv v = stack_pop(jq);
+      stack_push(jq, jq->path);
+
+      stack_save(jq, pc - 1, stack_get_pos(jq));
+
+      stack_push(jq, jv_number(jq->subexp_nest));
+      stack_push(jq, v);
+
+      jq->path = jv_array();
+      jq->subexp_nest = 0;
       break;
     }
 
+    case PATH_END: {
+      jv v = stack_pop(jq);
+      jv_free(v); // discard value, only keep path
+
+      int old_subexp_nest = (int)jv_number_value(stack_pop(jq));
+
+      jv path = jq->path;
+      jq->path = stack_pop(jq);
+
+      struct stack_pos spos = stack_get_pos(jq);
+      stack_push(jq, jv_copy(path));
+      stack_save(jq, pc - 1, spos);
+
+      stack_push(jq, path);
+      jq->subexp_nest = old_subexp_nest;
+      break;
+    }
+
+    case ON_BACKTRACK(PATH_BEGIN):
+    case ON_BACKTRACK(PATH_END): {
+      jv_free(jq->path);
+      jq->path = stack_pop(jq);
+      goto do_backtrack;
+    }
+
     case INDEX: {
-      stackval t = stack_pop(jq);
-      jv k = stack_pop(jq).value;
-      int pathidx = path_push(jq, t, jv_copy(k));
-      jv v = jv_get(t.value, k);
+      jv t = stack_pop(jq);
+      jv k = stack_pop(jq);
+      path_append(jq, jv_copy(k));
+      jv v = jv_get(t, k);
       if (jv_is_valid(v)) {
-        stackval sv;
-        sv.value = v;
-        sv.pathidx = pathidx;
-        stack_push(jq, sv);
+        stack_push(jq, v);
       } else {
         print_error(v);
         goto do_backtrack;
@@ -330,8 +516,8 @@ jv jq_next(jq_state *jq) {
 
     case JUMP_F: {
       uint16_t offset = *pc++;
-      stackval t = stack_pop(jq);
-      jv_kind kind = jv_get_kind(t.value);
+      jv t = stack_pop(jq);
+      jv_kind kind = jv_get_kind(t);
       if (kind == JV_KIND_FALSE || kind == JV_KIND_NULL) {
         pc += offset;
       }
@@ -340,69 +526,70 @@ jv jq_next(jq_state *jq) {
     }
 
     case EACH: 
-      stack_push(jq, stackval_root(jv_number(-1)));
+      stack_push(jq, jv_number(-1));
       // fallthrough
     case ON_BACKTRACK(EACH): {
-      int idx = jv_number_value(stack_pop(jq).value);
-      stackval container = stack_pop(jq);
+      int idx = jv_number_value(stack_pop(jq));
+      jv container = stack_pop(jq);
 
-      int keep_going;
+      int keep_going, is_last = 0;
       jv key, value;
-      if (jv_get_kind(container.value) == JV_KIND_ARRAY) {
+      if (jv_get_kind(container) == JV_KIND_ARRAY) {
         if (opcode == EACH) idx = 0;
         else idx = idx + 1;
-        keep_going = idx < jv_array_length(jv_copy(container.value));
+        int len = jv_array_length(jv_copy(container));
+        keep_going = idx < len;
+        is_last = idx == len - 1;
         if (keep_going) {
           key = jv_number(idx);
-          value = jv_array_get(jv_copy(container.value), idx);
+          value = jv_array_get(jv_copy(container), idx);
         }
-      } else if (jv_get_kind(container.value) == JV_KIND_OBJECT) {
-        if (opcode == EACH) idx = jv_object_iter(container.value);
-        else idx = jv_object_iter_next(container.value, idx);
-        keep_going = jv_object_iter_valid(container.value, idx);
+      } else if (jv_get_kind(container) == JV_KIND_OBJECT) {
+        if (opcode == EACH) idx = jv_object_iter(container);
+        else idx = jv_object_iter_next(container, idx);
+        keep_going = jv_object_iter_valid(container, idx);
         if (keep_going) {
-          key = jv_object_iter_key(container.value, idx);
-          value = jv_object_iter_value(container.value, idx);
+          key = jv_object_iter_key(container, idx);
+          value = jv_object_iter_value(container, idx);
         }
       } else {
         assert(opcode == EACH);
         print_error(jv_invalid_with_msg(jv_string_fmt("Cannot iterate over %s",
-                                                      jv_kind_name(jv_get_kind(container.value)))));
+                                                      jv_kind_name(jv_get_kind(container)))));
         keep_going = 0;
       }
 
       if (!keep_going) {
-        jv_free(container.value);
+        jv_free(container);
         goto do_backtrack;
+      } else if (is_last) {
+        // we don't need to make a backtrack point
+        jv_free(container);
+        path_append(jq, key);
+        stack_push(jq, value);
       } else {
-        stack_save(jq);
+        struct stack_pos spos = stack_get_pos(jq);
         stack_push(jq, container);
-        stack_push(jq, stackval_root(jv_number(idx)));
-        frame_push_backtrack(&jq->frame_stk, pc - 1);
-        stack_switch(jq);
-        
-        stackval sv = {value,
-                       path_push(jq, container, key)};
-        stack_push(jq, sv);
+        stack_push(jq, jv_number(idx));
+        stack_save(jq, pc - 1, spos);
+        path_append(jq, key);
+        stack_push(jq, value);
       }
       break;
     }
 
     do_backtrack:
     case BACKTRACK: {
-      if (!stack_restore(jq)) {
+      pc = stack_restore(jq);
+      if (!pc) {
         return jv_invalid();
       }
-      pc = *frame_current_retaddr(&jq->frame_stk);
-      frame_pop(&jq->frame_stk);
       backtracking = 1;
       break;
     }
 
     case FORK: {
-      stack_save(jq);
-      frame_push_backtrack(&jq->frame_stk, pc - 1);
-      stack_switch(jq);
+      stack_save(jq, pc - 1, stack_get_pos(jq));
       pc++; // skip offset this time
       break;
     }
@@ -412,77 +599,119 @@ jv jq_next(jq_state *jq) {
       pc += offset;
       break;
     }
-
-    case YIELD: {
-      jv value = stack_pop(jq).value;
-      frame_push_backtrack(&jq->frame_stk, pc);
-      return value;
-    }
       
     case CALL_BUILTIN: {
       int nargs = *pc++;
-      stackval top = stack_pop(jq);
-      cfunc_input[0] = top.value;
+      jv top = stack_pop(jq);
+      jv* in = cfunc_input;
+      in[0] = top;
       for (int i = 1; i < nargs; i++) {
-        cfunc_input[i] = stack_pop(jq).value;
+        in[i] = stack_pop(jq);
       }
-      struct cfunction* func = &frame_current_bytecode(&jq->frame_stk)->globals->cfunctions[*pc++];
-      top.value = cfunction_invoke(func, cfunc_input);
-      if (jv_is_valid(top.value)) {
+      struct cfunction* function = &frame_current(jq)->bc->globals->cfunctions[*pc++];
+      typedef jv (*func_1)(jv);
+      typedef jv (*func_2)(jv,jv);
+      typedef jv (*func_3)(jv,jv,jv);
+      typedef jv (*func_4)(jv,jv,jv,jv);
+      typedef jv (*func_5)(jv,jv,jv,jv,jv);
+      switch (function->nargs) {
+      case 1: top = ((func_1)function->fptr)(in[0]); break;
+      case 2: top = ((func_2)function->fptr)(in[0], in[1]); break;
+      case 3: top = ((func_3)function->fptr)(in[0], in[1], in[2]); break;
+      case 4: top = ((func_4)function->fptr)(in[0], in[1], in[2], in[3]); break;
+      case 5: top = ((func_5)function->fptr)(in[0], in[1], in[2], in[3], in[4]); break;
+      default: return jv_invalid_with_msg(jv_string("Function takes too many arguments"));
+      }
+      
+      if (jv_is_valid(top)) {
         stack_push(jq, top);
       } else {
-        print_error(top.value);
+        print_error(top);
         goto do_backtrack;
       }
       break;
     }
 
     case CALL_JQ: {
+      jv input = stack_pop(jq);
       uint16_t nclosures = *pc++;
       uint16_t* retaddr = pc + 2 + nclosures*2;
-      frame_ptr new_frame = frame_push(&jq->frame_stk, 
-                                       make_closure(&jq->frame_stk, frame_current(&jq->frame_stk), pc),
-                                       retaddr);
-      pc += 2;
-      frame_ptr old_frame = forkable_stack_peek_next(&jq->frame_stk, new_frame);
-      assert(nclosures == frame_self(new_frame)->bc->nclosures);
-      for (int i=0; i<nclosures; i++) {
-        *frame_closure_arg(new_frame, i) = make_closure(&jq->frame_stk, old_frame, pc);
-        pc += 2;
-      }
-
-      pc = frame_current_bytecode(&jq->frame_stk)->code;
+      struct frame* new_frame = frame_push(jq, make_closure(jq, pc),
+                                           pc + 2, nclosures);
+      new_frame->retdata = jq->stk_top;
+      new_frame->retaddr = retaddr;
+      pc = new_frame->bc->code;
+      stack_push(jq, input);
       break;
     }
 
     case RET: {
-      pc = *frame_current_retaddr(&jq->frame_stk);
-      frame_pop(&jq->frame_stk);
+      jv value = stack_pop(jq);
+      assert(jq->stk_top == frame_current(jq)->retdata);
+      uint16_t* retaddr = frame_current(jq)->retaddr;
+      if (retaddr) {
+        // function return
+        pc = retaddr;
+        frame_pop(jq);
+      } else {
+        // top-level return, yielding value
+        struct stack_pos spos = stack_get_pos(jq);
+        stack_push(jq, jv_null());
+        stack_save(jq, pc - 1, spos);
+        return value;
+      }
+      stack_push(jq, value);
       break;
     }
+    case ON_BACKTRACK(RET): {
+      // resumed after top-level return
+      goto do_backtrack;
+    }
     }
   }
 }
 
+jq_state *jq_init(void) {
+  jq_state *jq;
+  jq = jv_mem_alloc_unguarded(sizeof(*jq));
+  if (jq == NULL)
+    return NULL;
 
-void jq_init(struct bytecode* bc, jv input, jq_state **jq, int flags) {
-  jq_state *new_jq;
-  new_jq = jv_mem_alloc(sizeof(*new_jq));
-  memset(new_jq, 0, sizeof(*new_jq));
-  forkable_stack_init(&new_jq->data_stk, sizeof(stackval) * 100);
-  forkable_stack_init(&new_jq->frame_stk, 1024);
-  forkable_stack_init(&new_jq->fork_stk, 1024);
+  jq->bc = 0;
+
+  stack_init(&jq->stk);
+  jq->stk_top = 0;
+  jq->fork_top = 0;
+  jq->curr_frame = 0;
+
+  jq->path = jv_null();
+  return jq;
+}
+
+void jq_set_nomem_handler(jq_state *jq, void (*nomem_handler)(void *), void *data) {
+  jv_nomem_handler(nomem_handler, data);
+  jq->nomem_handler = nomem_handler;
+  jq->nomem_handler_data = data;
+}
+
+
+void jq_start(jq_state *jq, jv input, int flags) {
+  jv_nomem_handler(jq->nomem_handler, jq->nomem_handler_data);
+  jq_reset(jq);
   
-  stack_push(new_jq, stackval_root(input));
-  struct closure top = {bc, -1};
-  frame_push(&new_jq->frame_stk, top, 0);
-  frame_push_backtrack(&new_jq->frame_stk, bc->code);
+  struct closure top = {jq->bc, -1};
+  struct frame* top_frame = frame_push(jq, top, 0, 0);
+  top_frame->retdata = 0;
+  top_frame->retaddr = 0;
+
+  stack_push(jq, input);
+  stack_save(jq, jq->bc->code, stack_get_pos(jq));
   if (flags & JQ_DEBUG_TRACE) {
-    new_jq->debug_trace_enabled = 1;
+    jq->debug_trace_enabled = 1;
   } else {
-    new_jq->debug_trace_enabled = 0;
+    jq->debug_trace_enabled = 0;
   }
-  *jq = new_jq;
+  jq->initial_execution = 1;
 }
 
 void jq_teardown(jq_state **jq) {
@@ -491,28 +720,24 @@ void jq_teardown(jq_state **jq) {
     return;
   *jq = NULL;
 
-  while (stack_restore(old_jq)) {}
+  jq_reset(old_jq);
+  bytecode_free(old_jq->bc);
+  old_jq->bc = 0;
 
-  assert(forkable_stack_empty(&old_jq->fork_stk));
-  assert(forkable_stack_empty(&old_jq->data_stk));
-  assert(forkable_stack_empty(&old_jq->frame_stk));
-  forkable_stack_free(&old_jq->fork_stk);
-  forkable_stack_free(&old_jq->data_stk);
-  forkable_stack_free(&old_jq->frame_stk);
-
-  for (int i=0; i<old_jq->pathsize; i++) {
-    jv_free(old_jq->pathbuf[i]);
-  }
-  jv_mem_free(old_jq->pathbuf);
   jv_mem_free(old_jq);
 }
 
-struct bytecode* jq_compile_args(const char* str, jv args) {
+int jq_compile_args(jq_state *jq, const char* str, jv args) {
+  jv_nomem_handler(jq->nomem_handler, jq->nomem_handler_data);
   assert(jv_get_kind(args) == JV_KIND_ARRAY);
   struct locfile locations;
   locfile_init(&locations, str, strlen(str));
   block program;
-  struct bytecode* bc = 0;
+  jq_reset(jq);
+  if (jq->bc) {
+    bytecode_free(jq->bc);
+    jq->bc = 0;
+  }
   int nerrors = jq_parse(&locations, &program);
   if (nerrors == 0) {
     for (int i=0; i<jv_array_length(jv_copy(args)); i++) {
@@ -523,16 +748,22 @@ struct bytecode* jq_compile_args(const char* str, jv args) {
       jv_free(name);
     }
     jv_free(args);
-    program = builtins_bind(program);
-    nerrors = block_compile(program, &locations, &bc);
+    nerrors = builtins_bind(&program);
+    if (nerrors == 0) {
+      nerrors = block_compile(program, &locations, &jq->bc);
+    }
   }
   if (nerrors) {
     fprintf(stderr, "%d compile %s\n", nerrors, nerrors > 1 ? "errors" : "error");
   }
   locfile_free(&locations);
-  return bc;
+  return jq->bc != NULL;
 }
 
-struct bytecode* jq_compile(const char* str) {
-  return jq_compile_args(str, jv_array());
+int jq_compile(jq_state *jq, const char* str) {
+  return jq_compile_args(jq, str, jv_array());
+}
+
+void jq_dump_disassembly(jq_state *jq, int indent) {
+  dump_disassembly(indent, jq->bc);
 }

execute.h

@@ -2,19 +2,20 @@
 #define EXECUTE_H
 #include "bytecode.h"
 
+typedef struct jq_state jq_state;
 
-
-struct bytecode* jq_compile(const char* str);
+int jq_compile(jq_state *, const char* str);
 
 /* args must be an array of the form [{name:"foo", value:"thing"}, {name:"bar",value:3}] */
-struct bytecode* jq_compile_args(const char* str, jv args);
+int jq_compile_args(jq_state *, const char* str, jv args);
 
-typedef struct jq_state jq_state;
 enum {JQ_DEBUG_TRACE = 1};
 
-void jq_init(struct bytecode* bc, jv value, jq_state **, int flags);
+jq_state *jq_init(void);
+void jq_set_nomem_handler(jq_state *, void (*)(void *), void *);
+void jq_start(jq_state *, jv value, int flags);
 jv jq_next(jq_state *);
 void jq_teardown(jq_state **);
-
+void jq_dump_disassembly(jq_state *, int);
 
 #endif

forkable_stack.h

@@ -1,172 +0,0 @@
-#ifndef FORKABLE_STACK_H
-#define FORKABLE_STACK_H
-#include <stddef.h>
-#include <assert.h>
-#include <string.h>
-#include "jv_alloc.h"
-
-struct forkable_stack_header {
-  /* distance in bytes between this header and the header
-     of the next object on the stack */
-  int next_delta;
-};
-
-#define FORKABLE_STACK_HEADER struct forkable_stack_header fk_header_
-
-struct forkable_stack {
-  // Stack grows down from stk+length
-  char* stk;
-
-  // stk+length is just past end of allocated area
-  // stk = jv_mem_alloc(length)
-  int length;
-
-  // stk+pos is header of top object, or stk+length if empty
-  int pos;
-
-  // everything after-or-including stk+savedlimit must be preserved
-  int savedlimit;
-};
-
-static void forkable_stack_check(struct forkable_stack* s) {
-  assert(s->stk);
-  assert(s->length > 0);
-  assert(s->pos >= 0 && s->pos <= s->length);
-  assert(s->savedlimit >= 0 && s->savedlimit <= s->length);
-}
-
-static int forkable_stack_empty(struct forkable_stack* s) {
-  return s->pos == s->length;
-}
-
-static void forkable_stack_init(struct forkable_stack* s, size_t sz) {
-  s->stk = jv_mem_alloc(sz);
-  s->length = sz;
-  s->pos = s->length;
-  s->savedlimit = s->length;
-  forkable_stack_check(s);
-}
-
-static void forkable_stack_free(struct forkable_stack* s) {
-  assert(forkable_stack_empty(s));
-  assert(s->savedlimit == s->length);
-  jv_mem_free(s->stk);
-  s->stk = 0;
-}
-
-static void* forkable_stack_push(struct forkable_stack* s, size_t sz_size) {
-  assert(sz_size > 0 && sz_size % sizeof(struct forkable_stack_header) == 0);
-  int size = (int)sz_size;
-  forkable_stack_check(s);
-  int curr = s->pos < s->savedlimit ? s->pos : s->savedlimit;
-  if (curr - size < 0) {
-    int oldlen = s->length;
-    s->length = (size + oldlen + 1024) * 2;
-    s->stk = jv_mem_realloc(s->stk, s->length);
-    int shift = s->length - oldlen;
-    memmove(s->stk + shift, s->stk, oldlen);
-    s->pos += shift;
-    s->savedlimit += shift;
-    curr += shift;
-  }
-  int newpos = curr - size;
-  assert(newpos < s->pos);
-  void* ret = (void*)(s->stk + newpos);
-  ((struct forkable_stack_header*)ret)->next_delta = s->pos - newpos;
-  s->pos = newpos;
-  forkable_stack_check(s);
-  return ret;
-}
-
-static void* forkable_stack_peek(struct forkable_stack* s) {
-  assert(!forkable_stack_empty(s));
-  forkable_stack_check(s);
-  return (void*)(s->stk + s->pos);
-}
-
-static void* forkable_stack_peek_next(struct forkable_stack* s, void* top) {
-  forkable_stack_check(s);
-  struct forkable_stack_header* elem = top;
-  int elempos = (char*)top - s->stk;
-  assert(elempos >= 0 && elempos < s->length);
-  assert(elempos + elem->next_delta <= s->length);
-  if (elempos + elem->next_delta < s->length) {
-    return (void*)(s->stk + elempos + elem->next_delta);
-  } else {
-    return 0;
-  }
-}
-
-// Returns 1 if the next forkable_stack_pop will permanently remove an
-// object from the stack (i.e. the top object was not saved with a fork)
-static int forkable_stack_pop_will_free(struct forkable_stack* s) {
-  return s->pos < s->savedlimit;
-}
-
-static void forkable_stack_pop(struct forkable_stack* s) {
-  forkable_stack_check(s);
-  struct forkable_stack_header* elem = forkable_stack_peek(s);
-  int reclaim_upto = s->pos + elem->next_delta < s->savedlimit ? 
-    s->pos + elem->next_delta : s->savedlimit;
-  assert(reclaim_upto <= s->length);
-  int reclaimed = reclaim_upto > s->pos ? reclaim_upto - s->pos : 0;
-  assert(0 <= reclaimed && s->pos + reclaimed <= s->length);
-  // s->pos <= i < reclaim_upto means that position i is to be freed
-  assert((reclaimed > 0) == forkable_stack_pop_will_free(s));
-  int new_pos = s->pos + elem->next_delta;
-  jv_mem_invalidate(elem, reclaimed);
-  s->pos = new_pos;
-}
-
-
-
-struct forkable_stack_state {
-  // We save the previous pos, savedlimit as
-  // length-pos, length-savedlimit since these
-  // values are stable across stack reallocations.
-  int prev_pos_delta, prev_limit_delta;
-};
-
-static void forkable_stack_save(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  state->prev_pos_delta = s->length - s->pos;
-  state->prev_limit_delta = s->length - s->savedlimit;
-  if (s->pos < s->savedlimit) s->savedlimit = s->pos;
-}
-
-static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  int curr_pos = s->pos;
-  s->pos = s->length - state->prev_pos_delta;
-  state->prev_pos_delta = s->length - curr_pos;
-
-  int curr_limit = s->savedlimit;
-  if (curr_pos < curr_limit) s->savedlimit = curr_pos;
-  //state->prevlimit = curr_limit; FIXME clean up
-  forkable_stack_check(s);
-}
-
-static void forkable_stack_restore(struct forkable_stack* s, struct forkable_stack_state* state) {
-  forkable_stack_check(s);
-  assert(s->savedlimit <= s->length - state->prev_pos_delta);
-  assert(s->savedlimit <= s->length - state->prev_limit_delta);
-  s->pos = s->length - state->prev_pos_delta;
-  s->savedlimit = s->length - state->prev_limit_delta;
-  forkable_stack_check(s);
-}
-
-typedef int stack_idx;
-
-static stack_idx forkable_stack_to_idx(struct forkable_stack* s, void* ptr) {
-  char* item = ptr;
-  int pos = item - s->stk;
-  assert(pos >= 0 && pos < s->length);
-  return s->length - pos;
-}
-
-static void* forkable_stack_from_idx(struct forkable_stack* s, stack_idx idx) {
-  assert(idx >= 1 && idx <= s->length);
-  return &s->stk[s->length - idx];
-}
-
-#endif

frame_layout.h

@@ -1,118 +0,0 @@
-#ifndef FRAME_LAYOUT_H
-#include "forkable_stack.h"
-#include "bytecode.h"
-
-struct closure {
-  struct bytecode* bc;
-  stack_idx env;
-};
-
-struct continuation {
-  struct bytecode* bc;
-  stack_idx env;
-  uint16_t* retaddr;
-  // FIXME: probably not necessary as a separate field
-  int is_backtrack_frame;
-};
-
-typedef union frame_elem {
-  FORKABLE_STACK_HEADER;
-  struct continuation cont;
-  struct closure closure;
-  jv jsonval;
-} *frame_ptr;
-
-/*
- * Frame layout
- * fr[0] - FORKABLE_STACK_HEADER (next pointer)
- * fr[1] - self (used to store return addresses, etc)
- * fr[2...nclosures+2] - closure params
- * fr[nclosures+2..nclosures+nlocals+2] - local variables
- */
-
-static int frame_size(struct bytecode* bc) {
-  return sizeof(union frame_elem) * (bc->nclosures + bc->nlocals + 2);
-}
-
-static struct continuation* frame_self(frame_ptr fr) {
-  return &fr[1].cont;
-}
-
-static struct closure* frame_closure_arg(frame_ptr fr, int closure) {
-  assert(closure >= 0);
-  assert(closure < frame_self(fr)->bc->nclosures);
-  return &fr[2+closure].closure;
-}
-
-static jv* frame_local_var(frame_ptr fr, int var) {
-  assert(var >= 0);
-  assert(var < frame_self(fr)->bc->nlocals);
-  return &fr[2 + frame_self(fr)->bc->nclosures + var].jsonval;
-}
-
-
-static frame_ptr frame_current(struct forkable_stack* stk) {
-  frame_ptr fp = forkable_stack_peek(stk);
-  if (forkable_stack_peek_next(stk, fp)) {
-    struct bytecode* bc = frame_self(forkable_stack_peek_next(stk, fp))->bc;
-    assert(frame_self(fp)->retaddr >= bc->code && frame_self(fp)->retaddr < bc->code + bc->codelen);
-  } else {
-    assert(frame_self(fp)->retaddr == 0);
-  }
-  return fp;
-}
-
-static struct bytecode* frame_current_bytecode(struct forkable_stack* stk) {
-  return frame_self(frame_current(stk))->bc;
-}
-static uint16_t** frame_current_retaddr(struct forkable_stack* stk) {
-  return &frame_self(frame_current(stk))->retaddr;
-}
-
-static frame_ptr frame_get_parent(struct forkable_stack* stk, frame_ptr fr) {
-  return forkable_stack_from_idx(stk, frame_self(fr)->env);
-}
-
-static frame_ptr frame_get_level(struct forkable_stack* stk, frame_ptr fr, int level) {
-  for (int i=0; i<level; i++) {
-    fr = frame_get_parent(stk, fr);
-  }
-  return fr;
-}
-
-static frame_ptr frame_push(struct forkable_stack* stk, struct closure cl, uint16_t* retaddr) {
-  frame_ptr fp = forkable_stack_push(stk, frame_size(cl.bc));
-  struct continuation* cc = frame_self(fp);
-  cc->bc = cl.bc;
-  cc->env = cl.env;
-  cc->retaddr = retaddr;
-  cc->is_backtrack_frame = 0;
-  for (int i=0; i<cl.bc->nlocals; i++) {
-    *frame_local_var(fp, i) = jv_invalid();
-  }
-  return fp;
-}
-
-static frame_ptr frame_push_backtrack(struct forkable_stack* stk, uint16_t* retaddr) {
-  struct continuation cc = *frame_self(frame_current(stk));
-  frame_ptr fp = forkable_stack_push(stk, sizeof(union frame_elem) * 2);
-  assert(!cc.is_backtrack_frame);
-  cc.is_backtrack_frame = 1;
-  cc.retaddr = retaddr;
-  *frame_self(fp) = cc;
-  return fp;
-}
-
-static void frame_pop(struct forkable_stack* stk) {
-  frame_ptr fp = frame_current(stk);
-  if (forkable_stack_pop_will_free(stk) && 
-      !frame_self(fp)->is_backtrack_frame) {
-    int nlocals = frame_self(fp)->bc->nlocals;
-    for (int i=0; i<nlocals; i++) {
-      jv_free(*frame_local_var(fp, i));
-    }
-  }
-  forkable_stack_pop(stk);
-}
-
-#endif

jq_parser.h

@@ -1,5 +1,7 @@
 #ifndef JQ_PARSER_H
 #define JQ_PARSER_H
+#include "locfile.h"
+#include "compile.h"
 
 int jq_parse(struct locfile* source, block* answer);
 int jq_parse_library(struct locfile* locations, block* answer);

jq_test.c

@@ -10,8 +10,16 @@ static void run_jq_tests();
 
 
 int jq_testsuite(int argc, char* argv[]) {
+  FILE *testdata = stdin;
   jv_test();
-  run_jq_tests(stdin);
+  if (argc > 2) {
+    testdata = fopen(argv[2], "r");
+    if (!testdata) {
+      perror("fopen");
+      exit(1);
+    }
+  }
+  run_jq_tests(testdata);
   return 0;
 }
 
@@ -27,6 +35,9 @@ static void run_jq_tests(FILE *testdata) {
   int tests = 0, passed = 0, invalid = 0;
   jq_state *jq = NULL;
 
+  jq = jq_init();
+  assert(jq);
+
   while (1) {
     if (!fgets(buf, sizeof(buf), testdata)) break;
     if (skipline(buf)) continue;
@@ -34,15 +45,15 @@ static void run_jq_tests(FILE *testdata) {
     printf("Testing %s\n", buf);
     int pass = 1;
     tests++;
-    struct bytecode* bc = jq_compile(buf);
-    if (!bc) {invalid++; continue;}
+    int compiled = jq_compile(jq, buf);
+    if (!compiled) {invalid++; continue;}
     printf("Disassembly:\n");
-    dump_disassembly(2, bc);
+    jq_dump_disassembly(jq, 2);
     printf("\n");
-    fgets(buf, sizeof(buf), testdata);
+    if (!fgets(buf, sizeof(buf), testdata)) { invalid++; break; }
     jv input = jv_parse(buf);
     if (!jv_is_valid(input)){ invalid++; continue; }
-    jq_init(bc, input, &jq, JQ_DEBUG_TRACE);
+    jq_start(jq, input, JQ_DEBUG_TRACE);
 
     while (fgets(buf, sizeof(buf), testdata)) {
       if (skipline(buf)) break;
@@ -63,7 +74,7 @@ static void run_jq_tests(FILE *testdata) {
         pass = 0;
       }
       jv as_string = jv_dump_string(jv_copy(expected), rand() & ~JV_PRINT_COLOUR);
-      jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length(jv_copy(as_string)));
+      jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length_bytes(jv_copy(as_string)));
       assert(jv_equal(jv_copy(expected), jv_copy(reparsed)));
       jv_free(as_string);
       jv_free(reparsed);
@@ -81,10 +92,9 @@ static void run_jq_tests(FILE *testdata) {
         jv_free(extra);
       }
     }
-    jq_teardown(&jq);
-    bytecode_free(bc);
     passed+=pass;
   }
+  jq_teardown(&jq);
   printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
   if (passed != tests) exit(1);
 }
@@ -115,9 +125,9 @@ static void jv_test() {
     jv_free(a2);
 
 
-    assert(a.val.complex.ptr->count == 1);
+    assert(a.val.nontrivial.ptr->count == 1);
     a = jv_array_append(a, jv_copy(a));
-    assert(a.val.complex.ptr->count == 1);
+    assert(a.val.nontrivial.ptr->count == 1);
 
     assert(jv_array_length(jv_copy(a)) == 2);
     assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
@@ -150,9 +160,9 @@ static void jv_test() {
 
     jv_free(subarray);
 
-    void* before = sub2.val.complex.ptr;
+    void* before = sub2.val.nontrivial.ptr;
     sub2 = jv_array_append(sub2, jv_number(200));
-    void* after = sub2.val.complex.ptr;
+    void* after = sub2.val.nontrivial.ptr;
     assert(before == after);
     jv_free(sub2);
 
@@ -164,6 +174,18 @@ static void jv_test() {
     jv_free(a3);
 
 
+    jv a4 = jv_array();
+    a4 = jv_array_append(a4, jv_number(1));
+    a4 = jv_array_append(a4, jv_number(2));
+    jv a5 = jv_copy(a4);
+    a4 = jv_array_append(a4, jv_number(3));
+    a4 = jv_array_slice(a4, 0, 1);
+    assert(jv_array_length(jv_copy(a4)) == 1);
+    a4 = jv_array_append(a4, jv_number(4));
+    assert(jv_array_length(a4) == 2);
+    assert(jv_array_length(a5) == 2);
+
+
     assert(jv_array_length(jv_copy(a)) == 2);
     assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
     assert(jv_array_length(jv_array_get(jv_copy(a), 1)) == 1);
@@ -179,8 +201,8 @@ static void jv_test() {
     assert(jv_equal(jv_string("foo"), jv_string_sized("foo", 3)));
     char nasty[] = "foo\0";
     jv shortstr = jv_string(nasty), longstr = jv_string_sized(nasty, sizeof(nasty));
-    assert(jv_string_length(shortstr) == (int)strlen(nasty));
-    assert(jv_string_length(longstr) == (int)sizeof(nasty));
+    assert(jv_string_length_bytes(shortstr) == (int)strlen(nasty));
+    assert(jv_string_length_bytes(longstr) == (int)sizeof(nasty));
 
   
     char a1s[] = "hello", a2s[] = "hello", bs[] = "goodbye";
@@ -201,7 +223,7 @@ static void jv_test() {
     for (int i=0; i<(int)sizeof(big); i++) big[i] = 'a';
     big[sizeof(big)-1] = 0;
     jv str = jv_string_fmt("%s", big);
-    assert(jv_string_length(jv_copy(str)) == sizeof(big) - 1);
+    assert(jv_string_length_bytes(jv_copy(str)) == sizeof(big) - 1);
     assert(!strcmp(big, jv_string_value(str)));
     jv_free(str);
   }

jv.c

@@ -8,25 +8,26 @@
 
 #include "jv_alloc.h"
 #include "jv.h"
+#include "jv_unicode.h"
 
 /*
  * Internal refcounting helpers
  */
 
-static void jvp_refcnt_init(jv_complex* c) {
+static void jvp_refcnt_init(jv_nontrivial* c) {
   c->ptr->count = 1;
 }
 
-static void jvp_refcnt_inc(jv_complex* c) {
+static void jvp_refcnt_inc(jv_nontrivial* c) {
   c->ptr->count++;
 }
 
-static int jvp_refcnt_dec(jv_complex* c) {
+static int jvp_refcnt_dec(jv_nontrivial* c) {
   c->ptr->count--;
   return c->ptr->count == 0;
 }
 
-static int jvp_refcnt_unshared(jv_complex* c) {
+static int jvp_refcnt_unshared(jv_nontrivial* c) {
   assert(c->ptr->count > 0);
   return c->ptr->count == 1;
 }
@@ -86,9 +87,9 @@ typedef struct {
 jv jv_invalid_with_msg(jv err) {
   jv x;
   x.kind = JV_KIND_INVALID;
-  x.val.complex.i[0] = x.val.complex.i[1] = 0;
+  x.val.nontrivial.i[0] = x.val.nontrivial.i[1] = 0;
   jvp_invalid* i = jv_mem_alloc(sizeof(jvp_invalid));
-  x.val.complex.ptr = &i->refcnt;
+  x.val.nontrivial.ptr = &i->refcnt;
   i->refcnt.count = 1;
   i->errmsg = err;
   return x;
@@ -99,7 +100,7 @@ jv jv_invalid() {
 }
 
 jv jv_invalid_get_msg(jv inv) {
-  jv x = jv_copy(((jvp_invalid*)inv.val.complex.ptr)->errmsg);
+  jv x = jv_copy(((jvp_invalid*)inv.val.nontrivial.ptr)->errmsg);
   jv_free(inv);
   return x;
 }
@@ -111,7 +112,7 @@ int jv_invalid_has_msg(jv inv) {
   return r;
 }
 
-static void jvp_invalid_free(jv_complex* x) {
+static void jvp_invalid_free(jv_nontrivial* x) {
   if (jvp_refcnt_dec(x)) {
     jv_free(((jvp_invalid*)x->ptr)->errmsg);
     jv_mem_free(x->ptr);
@@ -153,7 +154,7 @@ typedef struct {
   jv elements[];
 } jvp_array;
 
-static jvp_array* jvp_array_ptr(jv_complex* a) {
+static jvp_array* jvp_array_ptr(jv_nontrivial* a) {
   return (jvp_array*)a->ptr;
 }
 
@@ -165,12 +166,12 @@ static jvp_array* jvp_array_alloc(unsigned size) {
   return a;
 }
 
-static jv_complex jvp_array_new(unsigned size) {
-  jv_complex r = {&jvp_array_alloc(size)->refcnt, {0, 0}};
+static jv_nontrivial jvp_array_new(unsigned size) {
+  jv_nontrivial r = {&jvp_array_alloc(size)->refcnt, {0, 0}};
   return r;
 }
 
-static void jvp_array_free(jv_complex* a) {
+static void jvp_array_free(jv_nontrivial* a) {
   if (jvp_refcnt_dec(a)) {
     jvp_array* array = jvp_array_ptr(a);
     for (int i=0; i<array->length; i++) {
@@ -180,11 +181,11 @@ static void jvp_array_free(jv_complex* a) {
   }
 }
 
-static int jvp_array_length(jv_complex* a) {
+static int jvp_array_length(jv_nontrivial* a) {
   return a->i[1] - a->i[0];
 }
 
-static jv* jvp_array_read(jv_complex* a, int i) {
+static jv* jvp_array_read(jv_nontrivial* a, int i) {
   if (i >= 0 && i < jvp_array_length(a)) {
     jvp_array* array = jvp_array_ptr(a);
     assert(i + a->i[0] < array->length);
@@ -194,7 +195,7 @@ static jv* jvp_array_read(jv_complex* a, int i) {
   }
 }
 
-static jv* jvp_array_write(jv_complex* a, int i) {
+static jv* jvp_array_write(jv_nontrivial* a, int i) {
   assert(i >= 0);
   jvp_array* array = jvp_array_ptr(a);
 
@@ -214,7 +215,7 @@ static jv* jvp_array_write(jv_complex* a, int i) {
         array->elements[j] = JV_NULL;
       }
       array->length = imax(pos + 1, array->length);
-      a->i[1] = imax(pos + 1, array->length);
+      a->i[1] = imax(pos + 1, a->i[1]);
       return &array->elements[pos];
     }
   }
@@ -237,7 +238,7 @@ static jv* jvp_array_write(jv_complex* a, int i) {
   return &new_array->elements[i];
 }
 
-static int jvp_array_equal(jv_complex* a, jv_complex* b) {
+static int jvp_array_equal(jv_nontrivial* a, jv_nontrivial* b) {
   if (jvp_array_length(a) != jvp_array_length(b)) 
     return 0;
   if (jvp_array_ptr(a) == jvp_array_ptr(b) &&
@@ -251,11 +252,11 @@ static int jvp_array_equal(jv_complex* a, jv_complex* b) {
   return 1;
 }
 
-static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
+static jv_nontrivial jvp_array_slice(jv_nontrivial* a, int start, int end) {
   // FIXME: maybe slice should reallocate if the slice is small enough
   assert(start <= end);
   assert(a->i[0] + end <= a->i[1]);
-  jv_complex slice = *a;
+  jv_nontrivial slice = *a;
   slice.i[0] += start;
   slice.i[1] = slice.i[0] + (end - start);
   return slice;
@@ -268,7 +269,7 @@ static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
 jv jv_array_sized(int n) {
   jv j;
   j.kind = JV_KIND_ARRAY;
-  j.val.complex = jvp_array_new(n);
+  j.val.nontrivial = jvp_array_new(n);
   return j;
 }
 
@@ -278,14 +279,14 @@ jv jv_array() {
 
 int jv_array_length(jv j) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
-  int len = jvp_array_length(&j.val.complex);
+  int len = jvp_array_length(&j.val.nontrivial);
   jv_free(j);
   return len;
 }
 
 jv jv_array_get(jv j, int idx) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
-  jv* slot = jvp_array_read(&j.val.complex, idx);
+  jv* slot = jvp_array_read(&j.val.nontrivial, idx);
   jv val;
   if (slot) {
     val = jv_copy(*slot);
@@ -299,7 +300,7 @@ jv jv_array_get(jv j, int idx) {
 jv jv_array_set(jv j, int idx, jv val) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
   // copy/free of val,j coalesced
-  jv* slot = jvp_array_write(&j.val.complex, idx);
+  jv* slot = jvp_array_write(&j.val.nontrivial, idx);
   jv_free(*slot);
   *slot = val;
   return j;
@@ -325,7 +326,7 @@ jv jv_array_concat(jv a, jv b) {
 jv jv_array_slice(jv a, int start, int end) {
   assert(jv_get_kind(a) == JV_KIND_ARRAY);
   // copy/free of a coalesced
-  a.val.complex = jvp_array_slice(&a.val.complex, start, end);
+  a.val.nontrivial = jvp_array_slice(&a.val.nontrivial, start, end);
   return a;
 }
 
@@ -365,7 +366,7 @@ typedef struct {
   char data[];
 } jvp_string;
 
-static jvp_string* jvp_string_ptr(jv_complex* a) {
+static jvp_string* jvp_string_ptr(jv_nontrivial* a) {
   return (jvp_string*)a->ptr;
 }
 
@@ -376,16 +377,16 @@ static jvp_string* jvp_string_alloc(uint32_t size) {
   return s;
 }
 
-static jv_complex jvp_string_new(const char* data, uint32_t length) {
+static jv_nontrivial jvp_string_new(const char* data, uint32_t length) {
   jvp_string* s = jvp_string_alloc(length);
   s->length_hashed = length << 1;
   memcpy(s->data, data, length);
   s->data[length] = 0;
-  jv_complex r = {&s->refcnt, {0,0}};
+  jv_nontrivial r = {&s->refcnt, {0,0}};
   return r;
 }
 
-static void jvp_string_free(jv_complex* s) {
+static void jvp_string_free(jv_nontrivial* s) {
   if (jvp_refcnt_dec(s)) {
     jvp_string* str = jvp_string_ptr(s);
     jv_mem_free(str);
@@ -393,12 +394,12 @@ static void jvp_string_free(jv_complex* s) {
 }
 
 static void jvp_string_free_p(jvp_string* s) {
-  jv_complex p = {&s->refcnt,{0,0}};
+  jv_nontrivial p = {&s->refcnt,{0,0}};
   jvp_string_free(&p);
 }
 
 static jvp_string* jvp_string_copy_p(jvp_string* s) {
-  jv_complex p = {&s->refcnt,{0,0}};
+  jv_nontrivial p = {&s->refcnt,{0,0}};
   jvp_refcnt_inc(&p);
   return s;
 }
@@ -413,7 +414,7 @@ static uint32_t jvp_string_remaining_space(jvp_string* s) {
   return r;
 }
 
-static void jvp_string_append(jv_complex* string, const char* data, uint32_t len) {
+static void jvp_string_append(jv_nontrivial* string, const char* data, uint32_t len) {
   jvp_string* s = jvp_string_ptr(string);
   uint32_t currlen = jvp_string_length(s);
     
@@ -433,7 +434,7 @@ static void jvp_string_append(jv_complex* string, const char* data, uint32_t len
     memcpy(news->data + currlen, data, len);
     news->data[currlen + len] = 0;
     jvp_string_free(string);
-    jv_complex r = {&news->refcnt, {0,0}};
+    jv_nontrivial r = {&news->refcnt, {0,0}};
     *string = r;
   }
 }
@@ -508,7 +509,7 @@ static int jvp_string_equal_hashed(jvp_string* a, jvp_string* b) {
   return memcmp(a->data, b->data, jvp_string_length(a)) == 0;
 }
 
-static int jvp_string_equal(jv_complex* a, jv_complex* b) {
+static int jvp_string_equal(jv_nontrivial* a, jv_nontrivial* b) {
   jvp_string* stra = jvp_string_ptr(a);
   jvp_string* strb = jvp_string_ptr(b);
   if (jvp_string_length(stra) != jvp_string_length(strb)) return 0;
@@ -522,7 +523,7 @@ static int jvp_string_equal(jv_complex* a, jv_complex* b) {
 jv jv_string_sized(const char* str, int len) {
   jv j;
   j.kind = JV_KIND_STRING;
-  j.val.complex = jvp_string_new(str, len);
+  j.val.nontrivial = jvp_string_new(str, len);
   return j;
 }
 
@@ -530,34 +531,44 @@ jv jv_string(const char* str) {
   return jv_string_sized(str, strlen(str));
 }
 
-int jv_string_length(jv j) {
+int jv_string_length_bytes(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  int r = jvp_string_length(jvp_string_ptr(&j.val.complex));
+  int r = jvp_string_length(jvp_string_ptr(&j.val.nontrivial));
   jv_free(j);
   return r;
 }
 
+int jv_string_length_codepoints(jv j) {
+  assert(jv_get_kind(j) == JV_KIND_STRING);
+  const char* i = jv_string_value(j);
+  const char* end = i + jv_string_length_bytes(jv_copy(j));
+  int c = 0, len = 0;
+  while ((i = jvp_utf8_next(i, end, &c))) len++;
+  jv_free(j);
+  return len;
+}
+
 uint32_t jv_string_hash(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.complex));
+  uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.nontrivial));
   jv_free(j);
   return hash;
 }
 
 const char* jv_string_value(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  return jvp_string_ptr(&j.val.complex)->data;
+  return jvp_string_ptr(&j.val.nontrivial)->data;
 }
 
 jv jv_string_concat(jv a, jv b) {
-  jvp_string* sb = jvp_string_ptr(&b.val.complex);
-  jvp_string_append(&a.val.complex, sb->data, jvp_string_length(sb));
+  jvp_string* sb = jvp_string_ptr(&b.val.nontrivial);
+  jvp_string_append(&a.val.nontrivial, sb->data, jvp_string_length(sb));
   jv_free(b);
   return a;
 }
 
 jv jv_string_append_buf(jv a, const char* buf, int len) {
-  jvp_string_append(&a.val.complex, buf, len);
+  jvp_string_append(&a.val.nontrivial, buf, len);
   return a;
 }
 
@@ -602,8 +613,8 @@ typedef struct {
 } jvp_object;
 
 
-/* warning: complex justification of alignment */
-static jv_complex jvp_object_new(int size) {
+/* warning: nontrivial justification of alignment */
+static jv_nontrivial jvp_object_new(int size) {
   // Allocates an object of (size) slots and (size*2) hash buckets.
 
   // size must be a power of two
@@ -620,7 +631,7 @@ static jv_complex jvp_object_new(int size) {
   }
   obj->first_free = size - 1;
   int* hashbuckets = (int*)(&obj->elements[size]);
-  jv_complex r = {&obj->refcnt, 
+  jv_nontrivial r = {&obj->refcnt, 
                   {size*2 - 1, (char*)hashbuckets - (char*)obj}};
   for (int i=0; i<size*2; i++) {
     hashbuckets[i] = -1;
@@ -628,39 +639,39 @@ static jv_complex jvp_object_new(int size) {
   return r;
 }
 
-static jvp_object* jvp_object_ptr(jv_complex* o) {
+static jvp_object* jvp_object_ptr(jv_nontrivial* o) {
   return (jvp_object*)o->ptr;
 }
 
-static uint32_t jvp_object_mask(jv_complex* o) {
+static uint32_t jvp_object_mask(jv_nontrivial* o) {
   return o->i[0];
 }
 
-static int jvp_object_size(jv_complex* o) {
+static int jvp_object_size(jv_nontrivial* o) {
   return (o->i[0] + 1) >> 1;
 }
 
-static int* jvp_object_buckets(jv_complex* o) {
+static int* jvp_object_buckets(jv_nontrivial* o) {
   int* buckets = (int*)((char*)o->ptr + o->i[1]);
   assert(buckets == (int*)&jvp_object_ptr(o)->elements[jvp_object_size(o)]);
   return buckets;
 }
 
-static int* jvp_object_find_bucket(jv_complex* object, jvp_string* key) {
+static int* jvp_object_find_bucket(jv_nontrivial* object, jvp_string* key) {
   return jvp_object_buckets(object) + (jvp_object_mask(object) & jvp_string_hash(key));
 }
 
-static struct object_slot* jvp_object_get_slot(jv_complex* object, int slot) {
+static struct object_slot* jvp_object_get_slot(jv_nontrivial* object, int slot) {
   assert(slot == -1 || (slot >= 0 && slot < jvp_object_size(object)));
   if (slot == -1) return 0;
   else return &jvp_object_ptr(object)->elements[slot];
 }
 
-static struct object_slot* jvp_object_next_slot(jv_complex* object, struct object_slot* slot) {
+static struct object_slot* jvp_object_next_slot(jv_nontrivial* object, struct object_slot* slot) {
   return jvp_object_get_slot(object, slot->next);
 }
 
-static struct object_slot* jvp_object_find_slot(jv_complex* object, jvp_string* keystr, int* bucket) {
+static struct object_slot* jvp_object_find_slot(jv_nontrivial* object, jvp_string* keystr, int* bucket) {
   for (struct object_slot* curr = jvp_object_get_slot(object, *bucket); 
        curr; 
        curr = jvp_object_next_slot(object, curr)) {
@@ -671,7 +682,7 @@ static struct object_slot* jvp_object_find_slot(jv_complex* object, jvp_string*
   return 0;
 }
 
-static struct object_slot* jvp_object_add_slot(jv_complex* object, jvp_string* key, int* bucket) {
+static struct object_slot* jvp_object_add_slot(jv_nontrivial* object, jvp_string* key, int* bucket) {
   jvp_object* o = jvp_object_ptr(object);
   int newslot_idx = o->first_free;
   struct object_slot* newslot = jvp_object_get_slot(object, newslot_idx);
@@ -684,7 +695,7 @@ static struct object_slot* jvp_object_add_slot(jv_complex* object, jvp_string* k
   return newslot;
 }
 
-static void jvp_object_free_slot(jv_complex* object, struct object_slot* slot) {
+static void jvp_object_free_slot(jv_nontrivial* object, struct object_slot* slot) {
   jvp_object* o = jvp_object_ptr(object);
   slot->next = o->first_free;
   assert(slot->string);
@@ -694,14 +705,14 @@ static void jvp_object_free_slot(jv_complex* object, struct object_slot* slot) {
   o->first_free = slot - jvp_object_get_slot(object, 0);
 }
 
-static jv* jvp_object_read(jv_complex* object, jvp_string* key) {
+static jv* jvp_object_read(jv_nontrivial* object, jvp_string* key) {
   int* bucket = jvp_object_find_bucket(object, key);
   struct object_slot* slot = jvp_object_find_slot(object, key, bucket);
   if (slot == 0) return 0;
   else return &slot->value;
 }
 
-static void jvp_object_free(jv_complex* o) {
+static void jvp_object_free(jv_nontrivial* o) {
   if (jvp_refcnt_dec(o)) {
     for (int i=0; i<jvp_object_size(o); i++) {
       struct object_slot* slot = jvp_object_get_slot(o, i);
@@ -714,10 +725,10 @@ static void jvp_object_free(jv_complex* o) {
   }
 }
 
-static void jvp_object_rehash(jv_complex* object) {
+static void jvp_object_rehash(jv_nontrivial* object) {
   assert(jvp_refcnt_unshared(object));
   int size = jvp_object_size(object);
-  jv_complex new_object = jvp_object_new(size * 2);
+  jv_nontrivial new_object = jvp_object_new(size * 2);
   for (int i=0; i<size; i++) {
     struct object_slot* slot = jvp_object_get_slot(object, i);
     if (!slot->string) continue;
@@ -733,11 +744,11 @@ static void jvp_object_rehash(jv_complex* object) {
   *object = new_object;
 }
 
-static void jvp_object_unshare(jv_complex* object) {
+static void jvp_object_unshare(jv_nontrivial* object) {
   if (jvp_refcnt_unshared(object))
     return;
 
-  jv_complex new_object = jvp_object_new(jvp_object_size(object));
+  jv_nontrivial new_object = jvp_object_new(jvp_object_size(object));
   jvp_object_ptr(&new_object)->first_free = jvp_object_ptr(object)->first_free;
   for (int i=0; i<jvp_object_size(&new_object); i++) {
     struct object_slot* old_slot = jvp_object_get_slot(object, i);
@@ -758,7 +769,7 @@ static void jvp_object_unshare(jv_complex* object) {
   assert(jvp_refcnt_unshared(object));
 }
 
-static jv* jvp_object_write(jv_complex* object, jvp_string* key) {
+static jv* jvp_object_write(jv_nontrivial* object, jvp_string* key) {
   jvp_object_unshare(object);
   int* bucket = jvp_object_find_bucket(object, key);
   struct object_slot* slot = jvp_object_find_slot(object, key, bucket);
@@ -781,7 +792,7 @@ static jv* jvp_object_write(jv_complex* object, jvp_string* key) {
   return &slot->value;
 }
 
-static int jvp_object_delete(jv_complex* object, jvp_string* key) {
+static int jvp_object_delete(jv_nontrivial* object, jvp_string* key) {
   jvp_object_unshare(object);
   int* bucket = jvp_object_find_bucket(object, key);
   int* prev_ptr = bucket;
@@ -798,7 +809,7 @@ static int jvp_object_delete(jv_complex* object, jvp_string* key) {
   return 0;
 }
 
-static int jvp_object_length(jv_complex* object) {
+static int jvp_object_length(jv_nontrivial* object) {
   int n = 0;
   for (int i=0; i<jvp_object_size(object); i++) {
     struct object_slot* slot = jvp_object_get_slot(object, i);
@@ -807,7 +818,7 @@ static int jvp_object_length(jv_complex* object) {
   return n;
 }
 
-static int jvp_object_equal(jv_complex* o1, jv_complex* o2) {
+static int jvp_object_equal(jv_nontrivial* o1, jv_nontrivial* o2) {
   int len2 = jvp_object_length(o2);
   int len1 = 0;
   for (int i=0; i<jvp_object_size(o1); i++) {
@@ -829,14 +840,14 @@ static int jvp_object_equal(jv_complex* o1, jv_complex* o2) {
 jv jv_object() {
   jv j;
   j.kind = JV_KIND_OBJECT;
-  j.val.complex = jvp_object_new(8);
+  j.val.nontrivial = jvp_object_new(8);
   return j;
 }
 
 jv jv_object_get(jv object, jv key) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
-  jv* slot = jvp_object_read(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jv* slot = jvp_object_read(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv val;
   if (slot) {
     val = jv_copy(*slot);
@@ -852,7 +863,7 @@ jv jv_object_set(jv object, jv key, jv value) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
   // copy/free of object, key, value coalesced
-  jv* slot = jvp_object_write(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jv* slot = jvp_object_write(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv_free(*slot);
   *slot = value;
   return object;
@@ -861,14 +872,14 @@ jv jv_object_set(jv object, jv key, jv value) {
 jv jv_object_delete(jv object, jv key) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
-  jvp_object_delete(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jvp_object_delete(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv_free(key);
   return object;
 }
 
 int jv_object_length(jv object) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
-  int n = jvp_object_length(&object.val.complex);
+  int n = jvp_object_length(&object.val.nontrivial);
   jv_free(object);
   return n;
 }
@@ -919,7 +930,7 @@ int jv_object_iter(jv object) {
 int jv_object_iter_next(jv object, int iter) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(iter != ITER_FINISHED);
-  jv_complex* o = &object.val.complex;
+  jv_nontrivial* o = &object.val.nontrivial;
   struct object_slot* slot;
   do {
     iter++;
@@ -931,18 +942,18 @@ int jv_object_iter_next(jv object, int iter) {
 }
 
 jv jv_object_iter_key(jv object, int iter) {
-  jvp_string* s = jvp_object_get_slot(&object.val.complex, iter)->string;
+  jvp_string* s = jvp_object_get_slot(&object.val.nontrivial, iter)->string;
   assert(s);
   jv j;
   j.kind = JV_KIND_STRING;
-  j.val.complex.ptr = &s->refcnt;
-  j.val.complex.i[0] = 0;
-  j.val.complex.i[1] = 0;
+  j.val.nontrivial.ptr = &s->refcnt;
+  j.val.nontrivial.i[0] = 0;
+  j.val.nontrivial.i[1] = 0;
   return jv_copy(j);
 }
 
 jv jv_object_iter_value(jv object, int iter) {
-  return jv_copy(jvp_object_get_slot(&object.val.complex, iter)->value);
+  return jv_copy(jvp_object_get_slot(&object.val.nontrivial, iter)->value);
 }
 
 /*
@@ -953,20 +964,20 @@ jv jv_copy(jv j) {
       jv_get_kind(j) == JV_KIND_STRING || 
       jv_get_kind(j) == JV_KIND_OBJECT ||
       jv_get_kind(j) == JV_KIND_INVALID) {
-    jvp_refcnt_inc(&j.val.complex);
+    jvp_refcnt_inc(&j.val.nontrivial);
   }
   return j;
 }
 
 void jv_free(jv j) {
   if (jv_get_kind(j) == JV_KIND_ARRAY) {
-    jvp_array_free(&j.val.complex);
+    jvp_array_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_STRING) {
-    jvp_string_free(&j.val.complex);
+    jvp_string_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_OBJECT) {
-    jvp_object_free(&j.val.complex);
+    jvp_object_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_INVALID) {
-    jvp_invalid_free(&j.val.complex);
+    jvp_invalid_free(&j.val.nontrivial);
   }
 }
 
@@ -975,7 +986,7 @@ int jv_get_refcnt(jv j) {
   case JV_KIND_ARRAY:
   case JV_KIND_STRING:
   case JV_KIND_OBJECT:
-    return j.val.complex.ptr->count;
+    return j.val.nontrivial.ptr->count;
   default:
     return 1;
   }
@@ -991,20 +1002,20 @@ int jv_equal(jv a, jv b) {
     r = 0;
   } else if (jv_get_kind(a) == JV_KIND_NUMBER) {
     r = jv_number_value(a) == jv_number_value(b);
-  } else if (a.val.complex.ptr == b.val.complex.ptr &&
-             a.val.complex.i[0] == b.val.complex.i[0] &&
-             a.val.complex.i[1] == b.val.complex.i[1]) {
+  } else if (a.val.nontrivial.ptr == b.val.nontrivial.ptr &&
+             a.val.nontrivial.i[0] == b.val.nontrivial.i[0] &&
+             a.val.nontrivial.i[1] == b.val.nontrivial.i[1]) {
     r = 1;
   } else {
     switch (jv_get_kind(a)) {
     case JV_KIND_ARRAY:
-      r = jvp_array_equal(&a.val.complex, &b.val.complex);
+      r = jvp_array_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     case JV_KIND_STRING:
-      r = jvp_string_equal(&a.val.complex, &b.val.complex);
+      r = jvp_string_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     case JV_KIND_OBJECT:
-      r = jvp_object_equal(&a.val.complex, &b.val.complex);
+      r = jvp_object_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     default:
       r = 1;

jv.h

@@ -25,13 +25,13 @@ typedef struct {
 typedef struct{
   jv_refcnt* ptr;
   int i[2];
-} jv_complex;
+} jv_nontrivial;
 
 typedef struct {
   jv_kind kind;
   union {
     double number;
-    jv_complex complex;
+    jv_nontrivial nontrivial;
   } val;
 } jv;
 
@@ -82,7 +82,8 @@ jv jv_array_slice(jv, int, int);
 
 jv jv_string(const char*);
 jv jv_string_sized(const char*, int);
-int jv_string_length(jv);
+int jv_string_length_bytes(jv);
+int jv_string_length_codepoints(jv);
 uint32_t jv_string_hash(jv);
 const char* jv_string_value(jv);
 jv jv_string_concat(jv, jv);
@@ -116,7 +117,7 @@ jv jv_object_iter_value(jv, int);
 
 int jv_get_refcnt(jv);
 
-enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4 };
+enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4, JV_PRINT_SORTED = 8 };
 void jv_dump(jv, int flags);
 jv jv_dump_string(jv, int flags);
 

jv_alloc.c

@@ -2,11 +2,89 @@
 #include <stdio.h>
 #include "jv_alloc.h"
 
+struct nomem_handler {
+    jv_nomem_handler_f handler;
+    void *data;
+};
+
+#ifndef USE_PTHREAD_KEY
+#ifdef _MSC_VER
+static __declspec(thread) struct nomem_handler nomem_handler;
+#else
+static __thread struct nomem_handler nomem_handler;
+#endif
+
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  nomem_handler.handler = handler;
+}
+
+static void memory_exhausted() {
+  if (nomem_handler.handler)
+    nomem_handler.handler(nomem_handler.data); // Maybe handler() will longjmp() to safety
+  // Or not
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+#else
+#ifdef HAVE_PTHREAD_KEY_CREATE
+#include <pthread.h>
+
+pthread_key_t nomem_handler_key;
+pthread_once_t mem_once = PTHREAD_ONCE_INIT;
+
+static void tsd_init(void) {
+  if (pthread_key_create(&nomem_handler_key, NULL) != 0) {
+    fprintf(stderr, "error: cannot create thread specific key");
+    abort();
+  }
+}
+
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  pthread_once(&mem_once, tsd_init); // cannot fail
+  struct nomem_handler *nomem_handler = calloc(1, sizeof (nomem_handler));
+  if (nomem_handler == NULL) {
+    handler(data);
+    fprintf(stderr, "error: cannot allocate memory\n");
+    abort();
+  }
+  nomem_handler.handler = handler;
+  nomem_handler.data = data;
+  if (pthread_setspecific(nomem_handler_key, nomem_handler) != 0) {
+    handler(data);
+    fprintf(stderr, "error: cannot set thread specific data");
+    abort();
+  }
+}
+
+static void memory_exhausted() {
+  jv_nomem_handler_f handler;
+
+  pthread_once(&mem_once, tsd_init);
+  handler = pthread_getspecific(nomem_handler_key);
+  if (handler)
+    handler(); // Maybe handler() will longjmp() to safety
+  // Or not
+  fprintf(stderr, "error: cannot allocate memory\n");
+  abort();
+}
+
+#else
+
+static struct nomem_handler nomem_handler;
+void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
+  nomem_handler.handler = handler;
+  nomem_handler.data = data;
+}
+
 static void memory_exhausted() {
   fprintf(stderr, "error: cannot allocate memory\n");
   abort();
 }
 
+#endif /* HAVE_PTHREAD_KEY_CREATE */
+#endif /* !USE_PTHREAD_KEY */
+
+
 void* jv_mem_alloc(size_t sz) {
   void* p = malloc(sz);
   if (!p) {
@@ -15,6 +93,10 @@ void* jv_mem_alloc(size_t sz) {
   return p;
 }
 
+void* jv_mem_alloc_unguarded(size_t sz) {
+  return malloc(sz);
+}
+
 void jv_mem_free(void* p) {
   free(p);
 }

jv_alloc.h

@@ -14,7 +14,10 @@ static void jv_mem_invalidate(void* mem, size_t n) {
 #endif
 }
 
+typedef void (*jv_nomem_handler_f)(void *);
+void jv_nomem_handler(jv_nomem_handler_f, void *);
 void* jv_mem_alloc(size_t);
+void* jv_mem_alloc_unguarded(size_t);
 void jv_mem_free(void*);
 __attribute__((warn_unused_result)) void* jv_mem_realloc(void*, size_t);
 

jv_aux.c

@@ -3,6 +3,41 @@
 #include <stdlib.h>
 #include "jv_alloc.h"
 
+static int parse_slice(jv array, jv slice, int* pstart, int* pend) {
+  // Array slices
+  int len = jv_array_length(jv_copy(array));
+  jv start_jv = jv_object_get(jv_copy(slice), jv_string("start"));
+  jv end_jv = jv_object_get(slice, jv_string("end"));
+  if (jv_get_kind(start_jv) == JV_KIND_NULL) {
+    jv_free(start_jv);
+    start_jv = jv_number(0);
+  }
+  if (jv_get_kind(end_jv) == JV_KIND_NULL) {
+    jv_free(end_jv);
+    end_jv = jv_number(len);
+  }
+  if (jv_get_kind(start_jv) != JV_KIND_NUMBER ||
+      jv_get_kind(end_jv) != JV_KIND_NUMBER) {
+    jv_free(start_jv);
+    jv_free(end_jv);
+    return 0;
+  } else {
+    int start = (int)jv_number_value(start_jv);
+    int end = (int)jv_number_value(end_jv);
+    if (start < 0) start = len + start;
+    if (end < 0) end = len + end;
+
+    if (start < 0) start = 0;
+    if (start > len) start = len;
+    if (end > len) end = len;
+    if (end < start) end = start;
+    assert(0 <= start && start <= end && end <= len);
+    *pstart = start;
+    *pend = end;
+    return 1;
+  }
+}
+
 jv jv_get(jv t, jv k) {
   jv v;
   if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) {
@@ -18,8 +53,18 @@ jv jv_get(jv t, jv k) {
       jv_free(v);
       v = jv_null();
     }
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_OBJECT) {
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      v = jv_array_slice(t, start, end);
+    } else {
+      v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+      jv_free(t);
+    }
   } else if (jv_get_kind(t) == JV_KIND_NULL && 
-             (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER)) {
+             (jv_get_kind(k) == JV_KIND_STRING || 
+              jv_get_kind(k) == JV_KIND_NUMBER || 
+              jv_get_kind(k) == JV_KIND_OBJECT)) {
     jv_free(t);
     jv_free(k);
     v = jv_null();
@@ -48,10 +93,49 @@ jv jv_set(jv t, jv k, jv v) {
              (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
     if (isnull) t = jv_array();
     t = jv_array_set(t, (int)jv_number_value(k), v);
+  } else if (jv_get_kind(k) == JV_KIND_OBJECT &&
+             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
+    if (isnull) t = jv_array();
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      if (jv_get_kind(v) == JV_KIND_ARRAY) {
+        int array_len = jv_array_length(jv_copy(t));
+        assert(0 <= start && start <= end && end <= array_len);
+        int slice_len = end - start;
+        int insert_len = jv_array_length(jv_copy(v));
+        if (slice_len < insert_len) {
+          // array is growing
+          int shift = insert_len - slice_len;
+          for (int i = array_len - 1; i >= end; i--) {
+            t = jv_array_set(t, i + shift, jv_array_get(jv_copy(t), i));
+          }
+        } else if (slice_len > insert_len) {
+          // array is shrinking
+          int shift = slice_len - insert_len;
+          for (int i = end; i < array_len; i++) {
+            t = jv_array_set(t, i - shift, jv_array_get(jv_copy(t), i));
+          }
+          t = jv_array_slice(t, 0, array_len - shift);
+        }
+        for (int i=0; i < insert_len; i++) {
+          t = jv_array_set(t, start + i, jv_array_get(jv_copy(v), i));
+        }
+        jv_free(v);
+      } else {
+        jv_free(t);
+        jv_free(v);
+        t = jv_invalid_with_msg(jv_string_fmt("A slice of an array can only be assigned another array"));
+      }
+    } else {
+      jv_free(t);
+      jv_free(k);
+      jv_free(v);
+      t = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+    }
   } else {
     jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s",
-                                               jv_kind_name(jv_get_kind(t)),
-                                               jv_kind_name(jv_get_kind(v))));
+                                               jv_kind_name(jv_get_kind(k)),
+                                               jv_kind_name(jv_get_kind(t))));
     jv_free(t);
     jv_free(k);
     jv_free(v);
@@ -96,23 +180,39 @@ jv jv_dels(jv t, jv keys) {
   if (jv_get_kind(t) == JV_KIND_NULL || jv_array_length(jv_copy(keys)) == 0) {
     // no change
   } else if (jv_get_kind(t) == JV_KIND_ARRAY) {
+    // extract slices, they must be handled differently
+    jv orig_keys = keys;
+    keys = jv_array();
     jv new_array = jv_array();
+    jv starts = jv_array(), ends = jv_array();
+    jv_array_foreach(orig_keys, i, key) {
+      if (jv_get_kind(key) == JV_KIND_NUMBER) {
+        keys = jv_array_append(keys, key);
+      } else if (jv_get_kind(key) == JV_KIND_OBJECT) {
+        int start, end;
+        if (parse_slice(t, key, &start, &end)) {
+          starts = jv_array_append(starts, jv_number(start));
+          ends = jv_array_append(ends, jv_number(end));
+        } else {
+          jv_free(new_array);
+          jv_free(key);
+          new_array = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+          goto arr_out;
+        }
+      } else {
+        jv_free(new_array);
+        new_array = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array",
+                                                      jv_kind_name(jv_get_kind(key))));
+        jv_free(key);
+        goto arr_out;
+      }
+    }
+
     int kidx = 0;
     jv_array_foreach(t, i, elem) {
       int del = 0;
       while (kidx < jv_array_length(jv_copy(keys))) {
-        jv nextdel = jv_array_get(jv_copy(keys), kidx);
-        if (jv_get_kind(nextdel) != JV_KIND_NUMBER) {
-          jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array",
-                                                     jv_kind_name(jv_get_kind(nextdel))));
-          jv_free(nextdel);
-          jv_free(new_array);
-          jv_free(elem);
-          new_array = err;
-          goto arr_out; // break twice
-        }
-        int delidx = (int)jv_number_value(nextdel);
-        jv_free(nextdel);
+        int delidx = (int)jv_number_value(jv_array_get(jv_copy(keys), kidx));
         if (i == delidx) {
           del = 1;
         }
@@ -121,12 +221,21 @@ jv jv_dels(jv t, jv keys) {
         }
         kidx++;
       }
+      for (int sidx=0; !del && sidx<jv_array_length(jv_copy(starts)); sidx++) {
+        if ((int)jv_number_value(jv_array_get(jv_copy(starts), sidx)) <= i &&
+            i < (int)jv_number_value(jv_array_get(jv_copy(ends), sidx))) {
+          del = 1;
+        }
+      }
       if (!del)
         new_array = jv_array_append(new_array, elem);
       else
         jv_free(elem);
     }
   arr_out:
+    jv_free(starts);
+    jv_free(ends);
+    jv_free(orig_keys);
     jv_free(t);
     t = new_array;
   } else if (jv_get_kind(t) == JV_KIND_OBJECT) {
@@ -271,8 +380,8 @@ jv jv_delpaths(jv object, jv paths) {
 static int string_cmp(const void* pa, const void* pb){
   const jv* a = pa;
   const jv* b = pb;
-  int lena = jv_string_length(jv_copy(*a));
-  int lenb = jv_string_length(jv_copy(*b));
+  int lena = jv_string_length_bytes(jv_copy(*a));
+  int lenb = jv_string_length_bytes(jv_copy(*b));
   int minlen = lena < lenb ? lena : lenb;
   int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen);
   if (r == 0) r = lena - lenb;

jv_file.c

@@ -0,0 +1,45 @@
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "jv.h"
+#include "jv_aux.h"
+#include "jv_parse.h"
+
+jv jv_load_file(const char* filename, int raw) {
+  FILE* file = fopen(filename, "r");
+  struct jv_parser parser;
+  jv data;
+  if (!file) {
+    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
+                                             filename,
+                                             strerror(errno)));
+  }
+  if (raw) {
+    data = jv_string("");
+  } else {
+    data = jv_array();
+    jv_parser_init(&parser);
+  }
+  while (!feof(file) && !ferror(file)) {
+    char buf[4096];
+    size_t n = fread(buf, 1, sizeof(buf), file);
+    if (raw) {
+      data = jv_string_concat(data, jv_string_sized(buf, (int)n));
+    } else {
+      jv_parser_set_buf(&parser, buf, strlen(buf), !feof(file));
+      jv value;
+      while (jv_is_valid((value = jv_parser_next(&parser))))
+        data = jv_array_append(data, value);
+    }
+  }
+  int badread = ferror(file);
+  fclose(file);
+  if (badread) {
+    jv_free(data);
+    return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
+                                             filename));
+  }
+  return data;
+}

jv_file.h

@@ -0,0 +1,8 @@
+#ifndef JV_FILE_H
+#define JV_FILE_H
+
+#include "jv.h"
+
+jv jv_load_file(const char *, int);
+
+#endif

jv_print.c

@@ -5,6 +5,7 @@
 
 #include "jv_dtoa.h"
 #include "jv_unicode.h"
+#include "jv_aux.h"
 
 #define ESC "\033"
 #define COL(c) (ESC "[" c "m")
@@ -45,7 +46,7 @@ static void put_space(int n, FILE* fout, jv* strout) {
 static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) {
   assert(jv_get_kind(str) == JV_KIND_STRING);
   const char* i = jv_string_value(str);
-  const char* end = i + jv_string_length(jv_copy(str));
+  const char* end = i + jv_string_length_bytes(jv_copy(str));
   const char* cstart;
   int c = 0;
   char buf[32];
@@ -124,6 +125,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
     }
   }
   switch (jv_get_kind(x)) {
+  default:
   case JV_KIND_INVALID:
     assert(0 && "Invalid value");
     break;
@@ -193,7 +195,34 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
       put_space(indent+INDENT, F, S);
     }
     int first = 1;
-    jv_object_foreach(x, key, value) {
+    int i = 0;
+    jv keyset = jv_null();
+    while (1) {
+      jv key, value;
+      if (flags & JV_PRINT_SORTED) {
+        if (first) {
+          keyset = jv_keys(jv_copy(x));
+          i = 0;
+        } else {
+          i++;
+        }
+        if (i >= jv_array_length(jv_copy(keyset))) {
+          jv_free(keyset);
+          break;
+        }
+        key = jv_array_get(jv_copy(keyset), i);
+        value = jv_object_get(jv_copy(x), jv_copy(key));
+      } else {
+        if (first) {
+          i = jv_object_iter(x);
+        } else {
+          i = jv_object_iter_next(x, i);
+        }
+        if (!jv_object_iter_valid(x, i)) break;
+        key = jv_object_iter_key(x, i);
+        value = jv_object_iter_value(x, i);
+      }
+
       if (!first) {
         if (flags & JV_PRINT_PRETTY){
           put_str(",\n", F, S);

lexer.l

@@ -49,7 +49,7 @@ struct lexer_param;
 "and" { return AND; }
 "or" { return OR; }
 "end" { return END; }
-"fold" { return FOLD; }
+"reduce" { return REDUCE; }
 "//" { return DEFINEDOR; }
 "|=" { return SETPIPE; }
 "+=" { return SETPLUS; }
@@ -93,7 +93,7 @@ struct lexer_param;
   (\\[^u(]|\\u[a-zA-Z0-9]{0,4})+ {
     /* pass escapes to the json parser */
     jv escapes = jv_string_fmt("\"%.*s\"", yyleng, yytext);
-    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length(jv_copy(escapes)));
+    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length_bytes(jv_copy(escapes)));
     jv_free(escapes);
     return QQSTRING_TEXT;
   }
@@ -108,6 +108,7 @@ struct lexer_param;
   
 
 [a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext); return IDENT;}
+\.[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext+1); return FIELD;}
 
 [ \n\t]+  {}
 

locfile.c

@@ -0,0 +1,65 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <stdarg.h>
+#include "jv_alloc.h"
+#include "locfile.h"
+
+
+void locfile_init(struct locfile* l, const char* data, int length) {
+  l->data = data;
+  l->length = length;
+  l->nlines = 1;
+  for (int i=0; i<length; i++) {
+    if (data[i] == '\n') l->nlines++;
+  }
+  l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
+  l->linemap[0] = 0;
+  int line = 1;
+  for (int i=0; i<length; i++) {
+    if (data[i] == '\n') {
+      l->linemap[line] = i+1;   // at start of line, not of \n
+      line++;
+    }
+  }
+  l->linemap[l->nlines] = length+1;   // virtual last \n
+}
+
+void locfile_free(struct locfile* l) {
+  jv_mem_free(l->linemap);
+}
+
+static int locfile_get_line(struct locfile* l, int pos) {
+  assert(pos < l->length);
+  int line = 1;
+  while (l->linemap[line] <= pos) line++;   // == if pos at start (before, never ==, because pos never on \n)
+  assert(line-1 < l->nlines);
+  return line-1;
+}
+
+static int locfile_line_length(struct locfile* l, int line) {
+  assert(line < l->nlines);
+  return l->linemap[line+1] - l->linemap[line] -1;   // -1 to omit \n
+}
+
+void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
+  va_list fmtargs;
+  va_start(fmtargs, fmt);
+  vfprintf(stderr, fmt, fmtargs);
+  va_end(fmtargs);
+  fprintf(stderr, "\n");
+  if (loc.start == -1) {
+    fprintf(stderr, "<unknown location>\n");
+    return;
+  }
+  int startline = locfile_get_line(l, loc.start);
+  int offset = l->linemap[startline];
+  fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset);
+  fprintf(stderr, "%*s", loc.start - offset, "");
+  for (int i = loc.start; 
+       i < loc.end && i < offset + locfile_line_length(l, startline);
+       i++){
+    fprintf(stderr, "^");
+  }
+  fprintf(stderr, "\n");
+}

locfile.h

@@ -1,10 +1,6 @@
 #ifndef _LOCFILE_H
 #define _LOCFILE_H
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include <stdarg.h>
-#include "jv_alloc.h"
+
 typedef struct {
   int start, end;
 } location;
@@ -18,62 +14,10 @@ struct locfile {
   int nlines;
 };
 
-static void locfile_init(struct locfile* l, const char* data, int length) {
-  l->data = data;
-  l->length = length;
-  l->nlines = 1;
-  for (int i=0; i<length; i++) {
-    if (data[i] == '\n') l->nlines++;
-  }
-  l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
-  l->linemap[0] = 0;
-  int line = 1;
-  for (int i=0; i<length; i++) {
-    if (data[i] == '\n') {
-      l->linemap[line] = i;
-      line++;
-    }
-  }
-  l->linemap[l->nlines] = length;
-}
+void locfile_init(struct locfile* l, const char* data, int length);
 
-static void locfile_free(struct locfile* l) {
-  jv_mem_free(l->linemap);
-}
+void locfile_free(struct locfile* l);
 
-static int locfile_get_line(struct locfile* l, int pos) {
-  assert(pos < l->length);
-  int line = 0;
-  while (l->linemap[line+1] < pos) line++;
-  assert(line < l->nlines);
-  return line;
-}
-
-static int locfile_line_length(struct locfile* l, int line) {
-  assert(line < l->nlines);
-  return l->linemap[line+1] - l->linemap[line];
-}
-
-static void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
-  va_list fmtargs;
-  va_start(fmtargs, fmt);
-  vfprintf(stderr, fmt, fmtargs);
-  va_end(fmtargs);
-  fprintf(stderr, "\n");
-  if (loc.start == -1) {
-    fprintf(stderr, "<unknown location>\n");
-    return;
-  }
-  int startline = locfile_get_line(l, loc.start);
-  int offset = l->linemap[startline];
-  fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset);
-  fprintf(stderr, "%*s", loc.start - offset, "");
-  for (int i = loc.start; 
-       i < loc.end && i < offset + locfile_line_length(l, startline);
-       i++){
-    fprintf(stderr, "^");
-  }
-  fprintf(stderr, "\n");
-}
+void locfile_locate(struct locfile* l, location loc, const char* fmt, ...);
 
 #endif

main.c

@@ -1,4 +1,4 @@
-#define _POSIX_SOURCE
+#include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
@@ -6,17 +6,19 @@
 #include <unistd.h>
 #include "compile.h"
 #include "jv.h"
+#include "jv_file.h"
 #include "jv_parse.h"
 #include "execute.h"
 #include "config.h"  /* Autoconf generated header file */
 #include "jv_alloc.h"
+#include "version.h"
 
 int jq_testsuite(int argc, char* argv[]);
 
 static const char* progname;
 
 static void usage() {
-  fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", PACKAGE_VERSION);
+  fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", JQ_VERSION);
   fprintf(stderr, "Usage: %s [options] <jq filter> [file...]\n\n", progname);
   fprintf(stderr, "For a description of the command line options and\n");
   fprintf(stderr, "how to write jq filters (and why you might want to)\n");
@@ -55,22 +57,21 @@ enum {
   ASCII_OUTPUT = 32,
   COLOUR_OUTPUT = 64,
   NO_COLOUR_OUTPUT = 128,
+  SORTED_OUTPUT = 256,
 
-  FROM_FILE = 256,
+  FROM_FILE = 512,
 
   /* debugging only */
   DUMP_DISASM = 2048,
 };
 static int options = 0;
-static struct bytecode* bc;
 
-static void process(jv value, int flags) {
-  jq_state *jq = NULL;
-  jq_init(bc, value, &jq, flags);
+static void process(jq_state *jq, jv value, int flags) {
+  jq_start(jq, value, flags);
   jv result;
   while (jv_is_valid(result = jq_next(jq))) {
     if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
-      fwrite(jv_string_value(result), 1, jv_string_length(jv_copy(result)), stdout);
+      fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
       jv_free(result);
     } else {
       int dumpopts;
@@ -81,6 +82,7 @@ static void process(jv value, int flags) {
 #else
       dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0;
 #endif
+      if (options & SORTED_OUTPUT) dumpopts |= JV_PRINT_SORTED;
       if (!(options & COMPACT_OUTPUT)) dumpopts |= JV_PRINT_PRETTY;
       if (options & ASCII_OUTPUT) dumpopts |= JV_PRINT_ASCII;
       if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR;
@@ -90,30 +92,6 @@ static void process(jv value, int flags) {
     printf("\n");
   }
   jv_free(result);
-  jq_teardown(&jq);
-}
-
-static jv slurp_file(const char* filename) {
-  FILE* file = fopen(filename, "r");
-  if (!file) {
-    return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
-                                             filename,
-                                             strerror(errno)));
-  }
-  jv data = jv_string("");
-  while (!feof(file) && !ferror(file)) {
-    char buf[4096];
-    size_t n = fread(buf, 1, sizeof(buf), file);
-    data = jv_string_concat(data, jv_string_sized(buf, (int)n));
-  }
-  int badread = ferror(file);
-  fclose(file);
-  if (badread) {
-    jv_free(data);
-    return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
-                                             filename));
-  }
-  return data;
 }
 
 FILE* current_input;
@@ -145,11 +123,20 @@ static int read_more(char* buf, size_t size) {
 }
 
 int main(int argc, char* argv[]) {
+  jq_state *jq;
   int ret = 0;
+  int compiled = 0;
+
   if (argc) progname = argv[0];
 
   if (argc > 1 && !strcmp(argv[1], "--run-tests")) {
-    return jq_testsuite(argc - 1, argv + 1);
+    return jq_testsuite(argc, argv);
+  }
+
+  jq = jq_init();
+  if (jq == NULL) {
+    perror("malloc");
+    return 1;
   }
 
   const char* program = 0;
@@ -182,6 +169,8 @@ int main(int argc, char* argv[]) {
       options |= NO_COLOUR_OUTPUT;
     } else if (isoption(argv[i], 'a', "ascii-output")) {
       options |= ASCII_OUTPUT;
+    } else if (isoption(argv[i], 'S', "sort-keys")) {
+      options |= SORTED_OUTPUT;
     } else if (isoption(argv[i], 'R', "raw-input")) {
       options |= RAW_INPUT;
     } else if (isoption(argv[i], 'n', "null-input")) {
@@ -198,6 +187,26 @@ int main(int argc, char* argv[]) {
       arg = jv_object_set(arg, jv_string("value"), jv_string(argv[i+2]));
       program_arguments = jv_array_append(program_arguments, arg);
       i += 2; // skip the next two arguments
+    } else if (isoption(argv[i], 0, "argfile")) {
+      if (i >= argc - 2) {
+        fprintf(stderr, "%s: --argfile takes two parameters (e.g. -a varname filename)\n", progname);
+        die();
+      }
+      jv arg = jv_object();
+      arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
+      jv data = jv_load_file(argv[i+2], 0);
+      if (!jv_is_valid(data)) {
+        data = jv_invalid_get_msg(data);
+        fprintf(stderr, "%s: Bad JSON in --argfile %s %s: %s\n", progname,
+                argv[i+1], argv[i+2], jv_string_value(data));
+        jv_free(data);
+        return 1;
+      }
+      if (jv_array_length(data) == 1)
+          data = jv_array_get(data, 0);
+      arg = jv_object_set(arg, jv_string("value"), data);
+      program_arguments = jv_array_append(program_arguments, arg);
+      i += 2; // skip the next two arguments
     } else if (isoption(argv[i],  0,  "debug-dump-disasm")) {
       options |= DUMP_DISASM;
     } else if (isoption(argv[i],  0,  "debug-trace")) {
@@ -205,7 +214,7 @@ int main(int argc, char* argv[]) {
     } else if (isoption(argv[i], 'h', "help")) {
       usage();
     } else if (isoption(argv[i], 'V', "version")) {
-      fprintf(stderr, "jq version %s\n", PACKAGE_VERSION);
+      printf("jq-%s\n", JQ_VERSION);
       return 0;
     } else {
       fprintf(stderr, "%s: Unknown option %s\n", progname, argv[i]);
@@ -221,27 +230,27 @@ int main(int argc, char* argv[]) {
   }
   
   if (options & FROM_FILE) {
-    jv data = slurp_file(program);
+    jv data = jv_load_file(program, 1);
     if (!jv_is_valid(data)) {
       data = jv_invalid_get_msg(data);
       fprintf(stderr, "%s: %s\n", progname, jv_string_value(data));
       jv_free(data);
       return 1;
     }
-    bc = jq_compile_args(jv_string_value(data), program_arguments);
+    compiled = jq_compile_args(jq, jv_string_value(data), program_arguments);
     jv_free(data);
   } else {
-    bc = jq_compile_args(program, program_arguments);
+    compiled = jq_compile_args(jq, program, program_arguments);
   }
-  if (!bc) return 1;
+  if (!compiled) return 1;
 
   if (options & DUMP_DISASM) {
-    dump_disassembly(0, bc);
+    jq_dump_disassembly(jq, 0);
     printf("\n");
   }
 
   if (options & PROVIDE_NULL) {
-    process(jv_null(), jq_flags);
+    process(jq, jv_null(), jq_flags);
   } else {
     jv slurped;
     if (options & SLURP) {
@@ -262,7 +271,7 @@ int main(int argc, char* argv[]) {
             slurped = jv_string_concat(slurped, jv_string(buf));
           } else {
             if (buf[len-1] == '\n') buf[len-1] = 0;
-            process(jv_string(buf), jq_flags);
+            process(jq, jv_string(buf), jq_flags);
           }
         }
       } else {
@@ -272,7 +281,7 @@ int main(int argc, char* argv[]) {
           if (options & SLURP) {
             slurped = jv_array_append(slurped, value);
           } else {
-            process(value, jq_flags);
+            process(jq, value, jq_flags);
           }
         }
         if (jv_invalid_has_msg(jv_copy(value))) {
@@ -290,11 +299,11 @@ int main(int argc, char* argv[]) {
     if (ret != 0)
       goto out;
     if (options & SLURP) {
-      process(slurped, jq_flags);
+      process(jq, slurped, jq_flags);
     }
   }
 out:
   jv_mem_free(input_filenames);
-  bytecode_free(bc);
+  jq_teardown(&jq);
   return ret;
 }

opcode.c

@@ -1,31 +0,0 @@
-#include "opcode.h"
-
-// flags, length
-#define NONE 0, 1
-#define CONSTANT OP_HAS_CONSTANT, 2
-#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
-#define BRANCH OP_HAS_BRANCH, 2
-#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
-#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
-#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
-#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
-
-#define OP(name, imm, in, out) \
-  {name, #name, imm, in, out},
-
-static const struct opcode_description opcode_descriptions[] = {
-#include "opcode_list.h"
-};
-
-static const struct opcode_description invalid_opcode_description = {
-  -1, "#INVALID", 0, 0, 0, 0
-};
-
-
-const struct opcode_description* opcode_describe(opcode op) {
-  if ((int)op >= 0 && (int)op < NUM_OPCODES) {
-    return &opcode_descriptions[op];
-  } else {
-    return &invalid_opcode_description;
-  }
-}

opcode.h

@@ -1,41 +0,0 @@
-#ifndef OPCODE_H
-#define OPCODE_H
-#include <assert.h>
-
-typedef enum {
-#define OP(name, imm, in, out) name,
-#include "opcode_list.h"
-#undef OP
-} opcode;
-
-enum {
-  NUM_OPCODES = 
-#define OP(name, imm, in, out) +1
-#include "opcode_list.h"
-#undef OP
-};
-
-enum {
-  OP_HAS_CONSTANT = 2,
-  OP_HAS_VARIABLE = 4,
-  OP_HAS_BRANCH = 8,
-  OP_HAS_CFUNC = 32,
-  OP_HAS_UFUNC = 64,
-  OP_IS_CALL_PSEUDO = 128,
-  OP_HAS_BINDING = 1024,
-};
-struct opcode_description {
-  opcode op;
-  const char* name;
-
-  int flags;
-
-  // length in 16-bit units
-  int length;
-
-  int stack_in, stack_out;
-};
-
-const struct opcode_description* opcode_describe(opcode op);
-
-#endif

opcode_list.h

@@ -1,12 +1,11 @@
 OP(LOADK, CONSTANT, 1, 1)
 OP(DUP,   NONE,     1, 2)
 OP(DUP2,  NONE,     2, 3)
-OP(SWAP,  NONE,     2, 2)
 OP(POP,   NONE,     1, 0)
 OP(LOADV, VARIABLE, 1, 1)
+OP(LOADVN, VARIABLE, 1, 1)
 OP(STOREV, VARIABLE, 1, 0)
 OP(INDEX, NONE,     2, 1)
-OP(YIELD, NONE, 1, 0)
 OP(EACH,  NONE,     1, 1)
 OP(FORK,  BRANCH,   0, 0)
 OP(JUMP,  BRANCH,   0, 0)
@@ -14,8 +13,13 @@ OP(JUMP_F,BRANCH,   1, 0)
 OP(BACKTRACK, NONE, 0, 0)
 OP(APPEND, VARIABLE,1, 0)
 OP(INSERT, NONE,    4, 2)
+OP(RANGE, VARIABLE, 1, 1)
 
-OP(GETPATH, NONE,   2, 1)
+OP(SUBEXP_BEGIN,  NONE,     1, 2)
+OP(SUBEXP_END,    NONE,     2, 2)
+
+OP(PATH_BEGIN, NONE, 1, 2)
+OP(PATH_END,   NONE, 2, 1)
 
 OP(CALL_BUILTIN, CFUNC, -1, 1)
 

parser.y

@@ -5,12 +5,11 @@
 #include "jv_alloc.h"
 #define YYMALLOC jv_mem_alloc
 #define YYFREE jv_mem_free
-
-struct lexer_param;
-
 %}
 %code requires {
 #include "locfile.h"
+struct lexer_param;
+
 #define YYLTYPE location
 #define YYLLOC_DEFAULT(Loc, Rhs, N)             \
   do {                                          \
@@ -47,6 +46,7 @@ struct lexer_param;
 
 %token INVALID_CHARACTER
 %token <literal> IDENT
+%token <literal> FIELD
 %token <literal> LITERAL
 %token <literal> FORMAT
 %token EQ "=="
@@ -58,7 +58,7 @@ struct lexer_param;
 %token THEN "then"
 %token ELSE "else"
 %token ELSE_IF "elif"
-%token FOLD "fold"
+%token REDUCE "reduce"
 %token END "end"
 %token AND "and"
 %token OR "or"
@@ -112,25 +112,19 @@ void yyerror(YYLTYPE* loc, block* answer, int* errors,
 int yylex(YYSTYPE* yylval, YYLTYPE* yylloc, block* answer, int* errors, 
           struct locfile* locations, struct lexer_param* lexer_param_ptr) {
   yyscan_t lexer = lexer_param_ptr->lexer;
-  while (1) {
-    int tok = jq_yylex(yylval, yylloc, lexer);
-    if (tok == INVALID_CHARACTER) {
-      FAIL(*yylloc, "Invalid character");
+  int tok = jq_yylex(yylval, yylloc, lexer);
+  if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
+    jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
+    if (jv_get_kind(msg) == JV_KIND_STRING) {
+      FAIL(*yylloc, jv_string_value(msg));
     } else {
-      if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
-        jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
-        if (jv_get_kind(msg) == JV_KIND_STRING) {
-          FAIL(*yylloc, jv_string_value(msg));
-        } else {
-          FAIL(*yylloc, "Invalid literal");
-        }
-        jv_free(msg);
-        jv_free(yylval->literal);
-        yylval->literal = jv_null();
-      }
-      return tok;
+      FAIL(*yylloc, "Invalid literal");
     }
+    jv_free(msg);
+    jv_free(yylval->literal);
+    yylval->literal = jv_null();
   }
+  return tok;
 }
 
 static block gen_dictpair(block k, block v) {
@@ -141,6 +135,17 @@ static block gen_index(block obj, block key) {
   return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX));
 }
 
+static block gen_slice_index(block obj, block start, block end) {
+  block key = BLOCK(gen_subexp(gen_const(jv_object())),
+                    gen_subexp(gen_const(jv_string("start"))),
+                    gen_subexp(start),
+                    gen_op_simple(INSERT),
+                    gen_subexp(gen_const(jv_string("end"))),
+                    gen_subexp(end),
+                    gen_op_simple(INSERT));
+  return BLOCK(key, obj, gen_op_simple(INDEX));
+}
+
 static block gen_binop(block a, block b, int op) {
   const char* funcname = 0;
   switch (op) {
@@ -163,16 +168,24 @@ static block gen_binop(block a, block b, int op) {
 static block gen_format(block a, jv fmt) {
   return BLOCK(a, gen_call("format", BLOCK(gen_lambda(gen_const(fmt)))));
 }
+
+static block gen_definedor_assign(block object, block val) {
+  block tmp = gen_op_var_fresh(STOREV, "tmp");
+  return BLOCK(gen_op_simple(DUP),
+               val, tmp,
+               gen_call("_modify", BLOCK(gen_lambda(object),
+                                         gen_lambda(gen_definedor(gen_noop(), 
+                                                                  gen_op_bound(LOADV, tmp))))));
+}
  
 static block gen_update(block object, block val, int optype) {
-  block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
-                         gen_noop(), OP_HAS_VARIABLE);
+  block tmp = gen_op_var_fresh(STOREV, "tmp");
   return BLOCK(gen_op_simple(DUP),
                val,
                tmp,
                gen_call("_modify", BLOCK(gen_lambda(object), 
                                          gen_lambda(gen_binop(gen_noop(),
-                                                              gen_op_var_bound(LOADV, tmp),
+                                                              gen_op_bound(LOADV, tmp),
                                                               optype)))));
 }
 
@@ -205,15 +218,15 @@ Term "as" '$' IDENT '|' Exp {
   jv_free($4);
 } |
 
-"fold" Term "as" '$' IDENT '(' Exp ')' {
-  $$ = gen_fold(jv_string_value($5), $2, $7);
+"reduce" Term "as" '$' IDENT '(' Exp ';' Exp ')' {
+  $$ = gen_reduce(jv_string_value($5), $2, $7, $9);
   jv_free($5);
 } |
 
 "if" Exp "then" Exp ElseBody {
   $$ = gen_cond($2, $4, $5);
 } |
-"if" Exp error {
+"if" Exp "then" error {
   FAIL(@$, "Possibly unterminated 'if' statment");
   $$ = $2;
 } |
@@ -235,7 +248,7 @@ Exp "//" Exp {
 } |
 
 Exp "//=" Exp {
-  $$ = gen_update($1, gen_definedor(gen_noop(), $3), 0);
+  $$ = gen_definedor_assign($1, $3);
 } |
 
 Exp "|=" Exp {
@@ -322,7 +335,7 @@ FuncDef:
 
 "def" IDENT '(' IDENT ')' ':' Exp ';' {
   $$ = gen_function(jv_string_value($2), 
-                    gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), 
+                    gen_param(jv_string_value($4)), 
                     $7);
   jv_free($2);
   jv_free($4);
@@ -330,8 +343,8 @@ FuncDef:
 
 "def" IDENT '(' IDENT ';' IDENT ')' ':' Exp ';' {
   $$ = gen_function(jv_string_value($2), 
-                    BLOCK(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), 
-                          gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($6))),
+                    BLOCK(gen_param(jv_string_value($4)), 
+                          gen_param(jv_string_value($6))),
                     $9);
   jv_free($2);
   jv_free($4);
@@ -385,12 +398,27 @@ Term:
 '.' {
   $$ = gen_noop(); 
 } |
-Term '.' IDENT {
-  $$ = gen_index($1, gen_const($3)); 
+Term FIELD {
+  $$ = gen_index($1, gen_const($2));
 } |
-'.' IDENT { 
-  $$ = gen_index(gen_noop(), gen_const($2)); 
+FIELD { 
+  $$ = gen_index(gen_noop(), gen_const($1)); 
 } |
+Term '.' String {
+  $$ = gen_index($1, $3);
+} |
+'.' String {
+  $$ = gen_index(gen_noop(), $2);
+} |
+'.' error {
+  FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
+  $$ = gen_noop();
+} |
+'.' IDENT error {
+  jv_free($2);
+  FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
+  $$ = gen_noop();
+} | 
 /* FIXME: string literals */
 Term '[' Exp ']' {
   $$ = gen_index($1, $3); 
@@ -398,6 +426,15 @@ Term '[' Exp ']' {
 Term '[' ']' {
   $$ = block_join($1, gen_op_simple(EACH)); 
 } |
+Term '[' Exp ':' Exp ']' {
+  $$ = gen_slice_index($1, $3, $5);
+} |
+Term '[' Exp ':' ']' {
+  $$ = gen_slice_index($1, $3, gen_const(jv_null()));
+} |
+Term '[' ':' Exp ']' {
+  $$ = gen_slice_index($1, gen_const(jv_null()), $4);
+} |
 LITERAL {
   $$ = gen_const($1); 
 } |
@@ -420,7 +457,7 @@ FORMAT {
   $$ = BLOCK(gen_subexp(gen_const(jv_object())), $2, gen_op_simple(POP));
 } |
 '$' IDENT {
-  $$ = gen_location(@$, gen_op_var_unbound(LOADV, jv_string_value($2)));
+  $$ = gen_location(@$, gen_op_unbound(LOADV, jv_string_value($2)));
   jv_free($2);
 } | 
 IDENT {

scripts/version

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+cd `dirname "$0"`
+git rev-parse --verify -q jq-1.0 > /dev/null 2>&1
+git describe --always --match 'jq-*' --dirty | sed 's/^jq-//'

tests/all.test

@@ -115,6 +115,10 @@ null
 {"foo": {"bar": 42}, "bar": "badvalue"}
 42
 
+."foo"."bar"
+{"foo": {"bar": 20}}
+20
+
 
 #
 # Multiple outputs, iteration
@@ -166,6 +170,25 @@ null
 2
 3
 
+#
+# Slices
+#
+
+[.[3:2], .[-5:4], .[:-2], .[-2:], .[3:3][1:], .[10:]]
+[0,1,2,3,4,5,6]
+[[], [2,3], [0,1,2,3,4], [5,6], [], []]
+
+del(.[2:4],.[0],.[-2:])
+[0,1,2,3,4,5,6,7]
+[1,4,5]
+
+.[2:4] = ([], ["a","b"], ["a","b","c"])
+[0,1,2,3,4,5,6,7]
+[0,1,4,5,6,7]
+[0,1,"a","b",4,5,6,7]
+[0,1,"a","b","c",4,5,6,7]
+
+
 #
 # Variables
 #
@@ -283,8 +306,8 @@ null
 [false, false, false, false, false, false, false, false, true ]
 
 [.[] | length]
-[[], {}, [1,2], {"a":42}, "asdf"]
-[0, 0, 2, 1, 4]
+[[], {}, [1,2], {"a":42}, "asdf", "\u03bc"]
+[0, 0, 2, 1, 4, 1]
 
 map(keys)
 [{}, {"abcd":1,"abc":2,"abcde":3}, {"x":1, "z": 3, "y":2}]
@@ -360,7 +383,7 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
 # []
 # 1001
 
-fold 0 as $s (.[] | $s + .)
+reduce .[] as $x (0; . + $x)
 [1,2,4]
 7
 
@@ -475,10 +498,9 @@ def inc(x): x |= .+1; inc(.[].a)
 [{"foo":[1,2], "bar": 42}, {"foo":[1], "bar": null}, {"foo":[null,false,3], "bar": 18}, {"foo":[], "bar":42}, {"foo": [null,false,null], "bar": 41}]
 [[1,2], [1], [3], [42], [41]]
 
-# FIXME: behaviour of update operators
-# .[] //= .[0]
-# ["hello",true,false,[false],null]
-# ["hello",true,"hello",[false],"hello"]
+.[] //= .[0]
+["hello",true,false,[false],null]
+["hello",true,"hello",[false],"hello"]
 
 .[] | [.[0] and .[1], .[0] or .[1]]
 [[true,[]], [false,1], [42,null], [null,false]]
@@ -574,3 +596,11 @@ unique
 .foo[.baz]
 {"foo":{"bar":4},"baz":"bar"}
 4
+
+.[] | .error = "no, it's OK"
+[{"error":true}]
+{"error": "no, it's OK"}
+
+[{a:1}] | .[] | .a=999
+null
+{"a": 999}