Merge pull request #132 from dolmen/fix/github-urls

Fix Github URLs: stedolan.github.com -> stedolan.github.io

Makefile.am

@@ -77,7 +77,7 @@ endif
 ### Packaging
 
 docs/site.yml: configure.ac
-	sed 's/^jq_version: .*/jq_version: $(VERSION)/' $@ > $@.new
+	sed 's/^jq_version: .*/jq_version: "$(VERSION)"/' $@ > $@.new
 	mv $@.new $@
 
 install-binaries: $(BUILT_SOURCES)

README.md

@@ -4,12 +4,12 @@ jq
 jq is a command-line JSON processor.
 
 If you want to learn to use jq, read the documentation at
-[http://stedolan.github.com/jq](http://stedolan.github.com/jq). This
+[http://stedolan.github.io/jq](http://stedolan.github.io/jq). This
 documentation is generated from the docs/ folder of this repository.
 
 If you want to hack on jq, feel free, but be warned that its internals
 are not well-documented at the moment. Bring a hard hat and a
-shovel. Also, read the wiki: http://github.com/stedolan/jq/wiki
+shovel. Also, read the wiki: https://github.com/stedolan/jq/wiki
 
 To build jq, run
 

builtin.c

@@ -187,7 +187,7 @@ static jv f_length(jv input) {
   } else if (jv_get_kind(input) == JV_KIND_OBJECT) {
     return jv_number(jv_object_length(input));
   } else if (jv_get_kind(input) == JV_KIND_STRING) {
-    return jv_number(jv_string_length(input));
+    return jv_number(jv_string_length_codepoints(input));
   } else if (jv_get_kind(input) == JV_KIND_NULL) {
     jv_free(input);
     return jv_number(0);
@@ -220,7 +220,7 @@ static jv escape_string(jv input, const char* escapings) {
 
   jv ret = jv_string("");
   const char* i = jv_string_value(input);
-  const char* end = i + jv_string_length(jv_copy(input));
+  const char* end = i + jv_string_length_bytes(jv_copy(input));
   const char* cstart;
   int c = 0;
   while ((i = jvp_utf8_next((cstart = i), end, &c))) {
@@ -299,7 +299,7 @@ static jv f_format(jv input, jv fmt) {
 
     jv line = jv_string("");
     const char* s = jv_string_value(input);
-    for (int i=0; i<jv_string_length(jv_copy(input)); i++) {
+    for (int i=0; i<jv_string_length_bytes(jv_copy(input)); i++) {
       unsigned ch = (unsigned)(unsigned char)*s;
       if (ch < 128 && unreserved[ch]) {
         line = jv_string_append_buf(line, s, 1);
@@ -346,7 +346,7 @@ static jv f_format(jv input, jv fmt) {
     jv line = jv_string("");
     const char b64[64 + 1] = CHARS_ALPHANUM "+/";
     const char* data = jv_string_value(input);
-    int len = jv_string_length(jv_copy(input));
+    int len = jv_string_length_bytes(jv_copy(input));
     for (int i=0; i<len; i+=3) {
       uint32_t code = 0;
       int n = len - i >= 3 ? 3 : len-i;
@@ -520,9 +520,9 @@ static block bind_bytecoded_builtins(block b) {
   }
   {
     struct bytecoded_builtin builtin_def_1arg[] = {
-      {"path", BLOCK(gen_op_simple(DUP), 
+      {"path", BLOCK(gen_op_simple(PATH_BEGIN), 
                      gen_call("arg", gen_noop()),
-                     gen_op_simple(GETPATH))},
+                     gen_op_simple(PATH_END))},
     };
     for (unsigned i=0; i<sizeof(builtin_def_1arg)/sizeof(builtin_def_1arg[0]); i++) {
       builtins = BLOCK(builtins, gen_function(builtin_def_1arg[i].name,
@@ -530,8 +530,20 @@ static block bind_bytecoded_builtins(block b) {
                                               builtin_def_1arg[i].code));
     }
   }
+  {
+    block rangevar = block_bind(gen_op_var_unbound(STOREV, "rangevar"),
+                                gen_noop(), OP_HAS_VARIABLE);
+    block init = BLOCK(gen_op_simple(DUP), gen_call("start", gen_noop()), rangevar);
+    block range = BLOCK(init, 
+                        gen_call("end", gen_noop()),
+                        gen_op_var_bound(RANGE, rangevar));
+    builtins = BLOCK(builtins, gen_function("range",
+                                            BLOCK(gen_op_block_unbound(CLOSURE_PARAM, "start"),
+                                                  gen_op_block_unbound(CLOSURE_PARAM, "end")),
+                                            range));
+  }
   
-  return block_bind(builtins, b, OP_IS_CALL_PSEUDO);
+  return block_bind_referenced(builtins, b, OP_IS_CALL_PSEUDO);
 }
 
 static const char* const jq_builtins[] = {
@@ -542,11 +554,15 @@ static const char* const jq_builtins[] = {
   "def unique: group_by(.) | map(.[0]);",
   "def max_by(f): _max_by_impl(map([f]));",
   "def min_by(f): _min_by_impl(map([f]));",
-  "def add: fold null as $sum (.[] | $sum + .);",
+  "def add: reduce .[] as $x (null; . + $x);",
   "def del(f): delpaths([path(f)]);",
-  "def _assign(paths; value): value as $v | fold . as $obj (path(paths) as $p | $obj | setpath($p; $v));",
-  "def _modify(paths; update): fold . as $obj (path(paths) as $p | $obj | setpath($p; getpath($p) | update));",
+  "def _assign(paths; value): value as $v | reduce path(paths) as $p (.; setpath($p; $v));",
+  "def _modify(paths; update): reduce path(paths) as $p (.; setpath($p; getpath($p) | update));",
   "def recurse(f): ., (f | select(. != null) | recurse(f));",
+  "def to_entries: [keys[] as $k | {key: $k, value: .[$k]}];",
+  "def from_entries: map({(.key): .value}) | add;",
+  "def with_entries(f): to_entries | map(f) | from_entries;",
+  "def reverse: [.[length - 1 - range(0;length)]];",
 };
 
 
@@ -557,7 +573,7 @@ block builtins_bind(block b) {
     block funcs;
     int nerrors = jq_parse_library(&src, &funcs);
     assert(!nerrors);
-    b = block_bind(funcs, b, OP_IS_CALL_PSEUDO);
+    b = block_bind_referenced(funcs, b, OP_IS_CALL_PSEUDO);
     locfile_free(&src);
   }
   b = bind_bytecoded_builtins(b);

bytecode.c

@@ -16,12 +16,24 @@ static int bytecode_operation_length(uint16_t* codeptr) {
 
 void dump_disassembly(int indent, struct bytecode* bc) {
   if (bc->nclosures > 0) {
-    printf("%*s[params: %d]\n", indent, "", bc->nclosures);
+    printf("%*s[params: ", indent, "");
+    jv params = jv_object_get(jv_copy(bc->debuginfo), jv_string("params"));
+    for (int i=0; i<bc->nclosures; i++) {
+      if (i) printf(", ");
+      jv name = jv_array_get(jv_copy(params), i);
+      printf("%s", jv_string_value(name));
+      jv_free(name);
+    }
+    jv_free(params);
+    printf("]\n");
   }
   dump_code(indent, bc);
   for (int i=0; i<bc->nsubfunctions; i++) {
-    printf("%*ssubfn[%d]:\n", indent, "", i);
-    dump_disassembly(indent+2, bc->subfunctions[i]);
+    struct bytecode* subfn = bc->subfunctions[i];
+    jv name = jv_object_get(jv_copy(subfn->debuginfo), jv_string("name"));
+    printf("%*s%s:%d:\n", indent, "", jv_string_value(name), i);
+    jv_free(name);
+    dump_disassembly(indent+2, subfn);
   }
 }
 
@@ -35,6 +47,14 @@ void dump_code(int indent, struct bytecode* bc) {
   }
 }
 
+static struct bytecode* getlevel(struct bytecode* bc, int level) {
+  while (level > 0) {
+    bc = bc->parent;
+    level--;
+  }
+  return bc;
+}
+
 void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
   int pc = codeptr - bc->code;
   printf("%04d ", pc);
@@ -46,15 +66,28 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
       for (int i=0; i<imm+1; i++) {
         uint16_t level = bc->code[pc++];
         uint16_t idx = bc->code[pc++];
+        jv name;
         if (idx & ARG_NEWCLOSURE) {
-          printf(" subfn[%d]", idx & ~ARG_NEWCLOSURE);
+          idx &= ~ARG_NEWCLOSURE;
+          name = jv_object_get(jv_copy(getlevel(bc,level)->subfunctions[idx]->debuginfo), 
+                               jv_string("name"));
         } else {
-          printf(" param[%d]", idx);
+          name = jv_array_get(jv_object_get(jv_copy(getlevel(bc,level)->debuginfo), 
+                                            jv_string("params")), idx);
         }
+        printf(" %s:%d",
+               jv_string_value(name),
+               idx);
+        jv_free(name);
         if (level) {
           printf("^%d", level);
         }
       }
+    } else if (op->op == CALL_BUILTIN) {
+      int func = bc->code[pc++];
+      jv name = jv_array_get(jv_copy(bc->globals->cfunc_names), func);
+      printf(" %s", jv_string_value(name));
+      jv_free(name);
     } else if (op->flags & OP_HAS_BRANCH) {
       printf(" %04d", pc + imm);
     } else if (op->flags & OP_HAS_CONSTANT) {
@@ -62,7 +95,11 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
       jv_dump(jv_array_get(jv_copy(bc->constants), imm), 0);
     } else if (op->flags & OP_HAS_VARIABLE) {
       uint16_t v = bc->code[pc++];
-      printf(" v%d", v);
+      jv name = jv_array_get(jv_object_get(jv_copy(getlevel(bc,imm)->debuginfo), jv_string("locals")), v);
+      printf(" $%s:%d", 
+             jv_string_value(name), 
+             v);
+      jv_free(name);
       if (imm) {
         printf("^%d", imm);
       }
@@ -74,6 +111,7 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
 
 void symbol_table_free(struct symbol_table* syms) {
   jv_mem_free(syms->cfunctions);
+  jv_free(syms->cfunc_names);
   jv_mem_free(syms);
 }
 
@@ -85,5 +123,6 @@ void bytecode_free(struct bytecode* bc) {
   if (!bc->parent)
     symbol_table_free(bc->globals);
   jv_mem_free(bc->subfunctions);
+  jv_free(bc->debuginfo);
   jv_mem_free(bc);
 }

bytecode.h

@@ -10,6 +10,7 @@
 struct symbol_table {
   struct cfunction* cfunctions;
   int ncfunctions;
+  jv cfunc_names;
 };
 
 // The bytecode format matters in:
@@ -33,6 +34,8 @@ struct bytecode {
   int nsubfunctions;
 
   struct bytecode* parent;
+
+  jv debuginfo;
 };
 
 void dump_disassembly(int, struct bytecode* code);

compile.c

@@ -216,13 +216,14 @@ int block_has_only_binders(block binders, int bindflags) {
   return 1;
 }
 
-static void block_bind_subblock(block binder, block body, int bindflags) {
+static int block_bind_subblock(block binder, block body, int bindflags) {
   assert(block_is_single(binder));
   assert((opcode_describe(binder.first->op)->flags & bindflags) == bindflags);
   assert(binder.first->symbol);
   assert(binder.first->bound_by == 0 || binder.first->bound_by == binder.first);
 
   binder.first->bound_by = binder.first;
+  int nrefs = 0;
   for (inst* i = body.first; i; i = i->next) {
     int flags = opcode_describe(i->op)->flags;
     if ((flags & bindflags) == bindflags &&
@@ -230,12 +231,14 @@ static void block_bind_subblock(block binder, block body, int bindflags) {
         !strcmp(i->symbol, binder.first->symbol)) {
       // bind this instruction
       i->bound_by = binder.first;
+      nrefs++;
     }
     // binding recurses into closures
-    block_bind_subblock(binder, i->subfn, bindflags);
+    nrefs += block_bind_subblock(binder, i->subfn, bindflags);
     // binding recurses into argument list
-    block_bind_subblock(binder, i->arglist, bindflags);
+    nrefs += block_bind_subblock(binder, i->arglist, bindflags);
   }
+  return nrefs;
 }
 
 static void block_bind_each(block binder, block body, int bindflags) {
@@ -251,6 +254,21 @@ block block_bind(block binder, block body, int bindflags) {
   return block_join(binder, body);
 }
 
+block block_bind_referenced(block binder, block body, int bindflags) {
+  assert(block_has_only_binders(binder, bindflags));
+  bindflags |= OP_HAS_BINDING;
+  block refd = gen_noop();
+  for (inst* curr; (curr = block_take(&binder));) {
+    block b = inst_block(curr);
+    if (block_bind_subblock(b, body, bindflags)) {
+      refd = BLOCK(refd, b);
+    } else {
+      block_free(b);
+    }
+  }
+  return block_join(refd, body);
+}
+
 block gen_function(const char* name, block formals, block body) {
   block_bind_each(formals, body, OP_IS_CALL_PSEUDO);
   inst* i = inst_new(CLOSURE_CREATE);
@@ -275,7 +293,7 @@ block gen_call(const char* name, block args) {
 
 
 block gen_subexp(block a) {
-  return BLOCK(gen_op_simple(DUP), a, gen_op_simple(SWAP));
+  return BLOCK(gen_op_simple(SUBEXP_BEGIN), a, gen_op_simple(SUBEXP_END));
 }
 
 block gen_both(block a, block b) {
@@ -299,18 +317,27 @@ block gen_collect(block expr) {
                gen_op_target(FORK, tail),
                expr, 
                tail,
-               gen_op_var_bound(LOADV, array_var));
+               gen_op_var_bound(LOADVN, array_var));
 }
 
-block gen_fold(const char* varname, block init, block fold) {
-  block loop = BLOCK(fold, gen_op_var_unbound(STOREV, varname), gen_op_simple(BACKTRACK));
+block gen_reduce(const char* varname, block source, block init, block body) {
+  block res_var = block_bind(gen_op_var_unbound(STOREV, "reduce"),
+                             gen_noop(), OP_HAS_VARIABLE);
+
+  block loop = BLOCK(gen_op_simple(DUP),
+                     source,
+                     block_bind(gen_op_var_unbound(STOREV, varname),
+                                BLOCK(gen_op_var_bound(LOADVN, res_var),
+                                      body,
+                                      gen_op_var_bound(STOREV, res_var)),
+                                OP_HAS_VARIABLE),
+                     gen_op_simple(BACKTRACK));
   return BLOCK(gen_op_simple(DUP),
                init,
-               block_bind(gen_op_var_unbound(STOREV, varname),
-                          BLOCK(gen_op_target(FORK, loop),
-                                loop,
-                                gen_op_var_unbound(LOADV, varname)),
-                          OP_HAS_VARIABLE));
+               res_var,
+               gen_op_target(FORK, loop),
+               loop,
+               gen_op_var_bound(LOADVN, res_var));
 }
 
 block gen_definedor(block a, block b) {
@@ -507,13 +534,8 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
   int var_frame_idx = 0;
   bc->nsubfunctions = 0;
   errors += expand_call_arglist(locations, &b);
-  if (bc->parent) {
-    // functions should end in a return
-    b = BLOCK(b, gen_op_simple(RET));
-  } else {
-    // the toplevel should YIELD;BACKTRACK; when it finds an answer
-    b = BLOCK(b, gen_op_simple(YIELD), gen_op_simple(BACKTRACK));
-  }
+  b = BLOCK(b, gen_op_simple(RET));
+  jv localnames = jv_array();
   for (inst* curr = b.first; curr; curr = curr->next) {
     if (!curr->next) assert(curr == b.last);
     int length = opcode_describe(curr->op)->length;
@@ -531,6 +553,7 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
     if ((opcode_describe(curr->op)->flags & OP_HAS_VARIABLE) &&
         curr->bound_by == curr) {
       curr->imm.intval = var_frame_idx++;
+      localnames = jv_array_append(localnames, jv_string(curr->symbol));
     }
 
     if (curr->op == CLOSURE_CREATE) {
@@ -540,10 +563,13 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
     if (curr->op == CLOSURE_CREATE_C) {
       assert(curr->bound_by == curr);
       int idx = bc->globals->ncfunctions++;
+      bc->globals->cfunc_names = jv_array_append(bc->globals->cfunc_names,
+                                                 jv_string(curr->symbol));
       bc->globals->cfunctions[idx] = *curr->imm.cfunc;
       curr->imm.intval = idx;
     }
   }
+  bc->debuginfo = jv_object_set(bc->debuginfo, jv_string("locals"), localnames);
   if (bc->nsubfunctions) {
     bc->subfunctions = jv_mem_alloc(sizeof(struct bytecode*) * bc->nsubfunctions);
     for (inst* curr = b.first; curr; curr = curr->next) {
@@ -553,12 +579,16 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
         subfn->globals = bc->globals;
         subfn->parent = bc;
         subfn->nclosures = 0;
+        subfn->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_string(curr->symbol));
+        jv params = jv_array();
         for (inst* param = curr->arglist.first; param; param = param->next) {
           assert(param->op == CLOSURE_PARAM);
           assert(param->bound_by == param);
           param->imm.intval = subfn->nclosures++;
           param->compiled = subfn;
+          params = jv_array_append(params, jv_string(param->symbol));
         }
+        subfn->debuginfo = jv_object_set(subfn->debuginfo, jv_string("params"), params);
         errors += compile(locations, subfn, curr->subfn);
         curr->subfn = gen_noop();
       }
@@ -626,6 +656,8 @@ int block_compile(block b, struct locfile* locations, struct bytecode** out) {
   int ncfunc = count_cfunctions(b);
   bc->globals->ncfunctions = 0;
   bc->globals->cfunctions = jv_mem_alloc(sizeof(struct cfunction) * ncfunc);
+  bc->globals->cfunc_names = jv_array();
+  bc->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_null());
   int nerrors = compile(locations, bc, b);
   assert(bc->globals->ncfunctions == ncfunc);
   if (nerrors > 0) {

compile.h

@@ -34,7 +34,7 @@ block gen_call(const char* name, block body);
 block gen_subexp(block a);
 block gen_both(block a, block b);
 block gen_collect(block expr);
-block gen_fold(const char* varname, block init, block body);
+block gen_reduce(const char* varname, block source, block init, block body);
 block gen_definedor(block a, block b);
 block gen_condbranch(block iftrue, block iffalse);
 block gen_and(block a, block b);
@@ -50,6 +50,7 @@ void block_append(block* b, block b2);
 block block_join(block a, block b);
 int block_has_only_binders(block, int bindflags);
 block block_bind(block binder, block body, int bindflags);
+block block_bind_referenced(block binder, block body, int bindflags);
 
 int block_compile(block, struct locfile*, struct bytecode**);
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([jq], [1.2], [mu@netsoc.tcd.ie],
+AC_INIT([jq], [1.3], [mu@netsoc.tcd.ie],
              [jq], [http://stedolan.github.com/jq/])
 
 dnl Created autoconf implementation thompson@dtosolutions, 26NOV12

docs/Rakefile

@@ -91,6 +91,10 @@ def load_manual
   YAML::load(File.open("content/3.manual/manual.yml"))
 end
 
+def jq_version
+  YAML::load(File.open("site.yml"))["jq_version"]
+end
+
 task :manpage do
   Tempfile.open "manpage" do |f|
     manual = load_manual
@@ -137,10 +141,14 @@ task :mantests do
   end
 end
 
+$TARBALL = "../jq-#{jq_version}.tar.gz"
+file $TARBALL do
+  sh "cd ..; ./configure && make dist && make distclean"  
+end
+
 directory "output/download/source"
-task :tarball => ["output/download/source"] do
-  sh "cd ..; ./configure && make dist && make distclean"
-  sh "cp ../jq-*.tar.gz output/download/source"
+task :tarball => ["output/download/source", $TARBALL] do
+  sh "cp #{$TARBALL} output/download/source"
 end
 
 task :dist => [:build, :binaries, :tarball]

docs/content/2.download/default.yml

@@ -27,11 +27,11 @@ body:
          manually by following instructions on [Arch Linux's
          Wiki](https://wiki.archlinux.org/index.php/Arch_UseRepository).
 
-       * Debian/Ubuntu packages are coming Real Soon Now.
+       * jq 1.2 is [in Debian](http://packages.debian.org/jq)
 
       ### OS X
 
-       * `brew install jq` using [homebrew](http://mxcl.github.com/homebrew/)
+       * `brew install jq` using [homebrew](http://mxcl.github.io/homebrew/)
 
        * Or, grab prebuilt [64-bit binaries](osx64/jq) or [32-bit
          binaries](osx32/jq)
@@ -42,18 +42,28 @@ body:
 
       ### From source on Linux or OS X
 
+       * [Source tarball for jq $JQ_VERSION](source/jq-$JQ_VERSION.tar.gz)
+
+      You can build it using the usual `./configure && make && sudo
+      make install` rigmarole.
+
+      If you're interested in using the latest development version, try:
+
           git clone https://github.com/stedolan/jq.git
           cd jq
-          make && sudo make install
+          autoreconf
+          ./configure
+          make
+          sudo make install
 
       To build it from a git clone, you'll need to install a few
       packages first:
 
        * [Flex](http://www.gnu.org/software/flex/)
        * [Bison](http://www.gnu.org/software/bison/)
-       * [Python](http://www.python.org)
        * [GCC](http://gcc.gnu.org)
        * [Make](http://www.gnu.org/software/make)
+       * [Autotools](http://www.gnu.org/software/automake)
 
       For Linux systems, these will all be in your system's package
       manager, and if you do development on the machine they're most
@@ -62,7 +72,7 @@ body:
       On OS X, these are all included in Apple's command line tools, which
       can be installed from [Xcode](http://developer.apple.com/technologies/tools/).
       However, you may find that you need a newer version of Bison than the one provided
-      by Apple. This can be found in [Homebrew](http://mxcl.github.com/homebrew/)
+      by Apple. This can be found in [Homebrew](http://mxcl.github.io/homebrew/)
       or [MacPorts](http://macports.org/).
 
       `flex` and `bison` are used to generate the lexer and parser for
@@ -73,6 +83,8 @@ body:
 
       jq's documentation is compiled into static HTML using
       [Bonsai](http://www.tinytree.info). To view the documentation
-      locally, run `rake serve` from the docs/ subdirectory.
+      locally, run `rake serve` from the docs/ subdirectory. You'll
+      need a few Ruby dependencies, which can be installed by
+      following the instructions in `docs/README.md`.
 
 

docs/content/3.manual/manual.yml

@@ -160,7 +160,7 @@ sections:
             input: '{"notfoo": true, "alsonotfoo": false}'
             output: ['null']
 
-      - title: "`.[foo]`"
+      - title: "`.[foo]`, `.[2]`, `.[10:15]`"
         body: |
           
           You can also look up fields of an object using syntax like
@@ -169,6 +169,13 @@ sections:
           integer. Arrays are zero-based (like javascript), so `.[2]`
           returns the third element of the array.
 
+          The `.[10:15]` syntax can be used to return a subarray of an
+          array. The array returned by `.[10:15]` will be of length 5,
+          containing the elements from index 10 (inclusive) to index
+          15 (exclusive). Either index may be negative (in which case
+          it counts backwards from the end of the array), or omitted
+          (in which case it refers to the start or end of the array).
+
         examples:
           - program: '.[0]'
             input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
@@ -178,6 +185,18 @@ sections:
             input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
             output: ['null']
 
+          - program: '.[2:4]'
+            input: '["a","b","c","d","e"]'
+            output: ['["c", "d"]']
+          
+          - program: '.[:3]'
+            input: '["a","b","c","d","e"]'
+            output: ['["a", "b", "c"]']
+
+          - program: '.[-2:]'
+            input: '["a","b","c","d","e"]'
+            output: ['["d", "e"]']
+
       - title: "`.[]`"
         body: |
           
@@ -202,8 +221,8 @@ sections:
             output: []
 
           - program: '.[]'
-            input: '{"a": 1, "b": 2}'
-            output: ['[1, 2]']
+            input: '{"a": 1, "b": 1}'
+            output: ['1', '1']
 
       - title: "`,`"
         body: |
@@ -478,6 +497,31 @@ sections:
             input: '[[0,1], ["a","b","c"]]'
             output: ['[false, true]']
 
+      - title: `to_entries`, `from_entries`, `with_entries`
+        body: |
+          
+          These functions convert between an object and an array of
+          key-value pairs. If `to_entries` is passed an object, then
+          for each `k: v` entry in the input, the output array
+          includes `{"key": k, "value": v}`. 
+
+          `from_entries` does the opposite conversion, and
+          `with_entries(foo)` is a shorthand for `to_entries |
+          map(foo) | from_entries`, useful for doing some operation to
+          all keys and values of an object.
+
+        examples:
+          - program: 'to_entries'
+            input: '{"a": 1, "b": 2}'
+            output: ['[{"key":"a", "value":1}, {"key":"b", "value":2}]']
+          - program: 'from_entries'
+            input: '[{"key":"a", "value":1}, {"key":"b", "value":2}]'
+            output: ['{"a": 1, "b": 2}']
+          - program: 'with_entries(.key |= "KEY_" + .)'
+            input: '{"a": 1, "b": 2}'
+            output: ['{"KEY_a": 1, "KEY_b": 2}']
+
+
       - title: `select`
         body: |
           
@@ -544,6 +588,22 @@ sections:
           - program: add
             input: '[]'
             output: ["null"]
+
+      - title: `range`
+        body: |
+          
+          The `range` function produces a range of numbers. `range(4;10)`
+          produces 6 numbers, from 4 (inclusive) to 10 (exclusive). The numbers
+          are produced as separate outputs. Use `[range(4;10)]` to get a range as 
+          an array.
+
+        examples:
+          - program: 'range(2;4)'
+            input: 'null'
+            output: ['2', '3']
+          - program: '[range(2;4)]'
+            input: 'null'
+            output: ['[2,3]']
             
       - title: `tonumber`
         body: |
@@ -658,6 +718,16 @@ sections:
             input: '[1,2,5,3,5,3,1,3]'
             output: ['[1,2,3,5]']
 
+      - title: `reverse`
+        body: |
+          
+          This function reverses an array.
+
+        examples:
+          - program: 'reverse'
+            input: '[1,2,3,4]'
+            output: ['[4,3,2,1]']
+
       - title: `contains`
         body: |
 
@@ -801,7 +871,7 @@ sections:
 
           - program: '@sh "echo \(.)"'
             input: "\"O'Hara's Ale\""
-            output: ["\"echo 'O'\\''Hara'\\''s Ale\""]
+            output: ["\"echo 'O'\\\\''Hara'\\\\''s Ale'\""]
         
   - title: Conditionals and Comparisons
     entries:
@@ -898,9 +968,12 @@ sections:
           - program: '(true, false) or false'
             input: 'null'
             output: ['true', 'false']
-          - program: '(true, false) and (true, false)'
+#          - program: '(true, false) and (true, false)'
+#            input: 'null'
+#            output: ['true', 'false', 'false', 'false']
+          - program: '(true, true) and (true, false)'
             input: 'null'
-            output: ['true', 'false', 'false', 'false']
+            output: ['true', 'false', 'true', 'false']
           - program: '[true, false | not]'
             input: 'null'
             output: ['[false, true]']
@@ -943,7 +1016,7 @@ sections:
       (many jq functions such as `map` and `find` are in fact written
       in jq).
 
-      Finally, jq has a `fold` operation, which is very powerful but a
+      Finally, jq has a `reduce` operation, which is very powerful but a
       bit tricky. Again, it's mostly used internally, to define some
       useful bits of jq's standard library.
 
@@ -1040,7 +1113,7 @@ sections:
           If you want the value-argument behaviour for defining simple
           functions, you can just use a variable:
 
-              def addvalue(f): f as $value | map(, + $value);
+              def addvalue(f): f as $value | map(. + $value);
 
           With that definition, `addvalue(.foo)` will add the current
           input's `.foo` field to each element of the array.
@@ -1053,36 +1126,26 @@ sections:
             input: '[[1,2],[10,20]]'
             output: ['[[1,2,1,2], [10,20,1,2]]']
 
-      - title: Fold
+      - title: Reduce
         body: |
         
-          The `fold` syntax in jq allows you to combine all of the
+          The `reduce` syntax in jq allows you to combine all of the
           results of an expression by accumulating them into a single
           answer. As an example, we'll pass `[3,2,1]` to this expression:
 
-              fold 0 as $sum (.[] | $sum + .)
+              reduce .[] as $item (0; . + $item)
           
-          The variable `$sum` is first given the value `0`. The body
-          of the fold (i.e. `.[] | $sum + .`) is evaluated. `.[]`
-          produces three results, `3`, `2`, and `1`. For the first
-          one, `$sum + .` gives `3`.
+          For each result that `.[]` produces, `. + $item` is run to
+          accumulate a running total, starting from 0. In this
+          example, `.[]` produces the results 3, 2, and 1, so the
+          effect is similar to running something like this:
 
-          Having produced this answer, jq backtracks to find the next
-          result as per usual. However, this time, `$sum` is set to
-          the previous value of the body, so `$sum + .` gives
-          `5`. After the final backtracking, `$sum + .` gives
-          `6`. This final value is used as the value of the entire
-          `fold` expression, so the above filter returns `6`.
-
-          More formally, in order to evaluate `fold INIT as $VAR
-          (BODY)`, jq first sets `$VAR` to the value of `INIT`. It
-          then runs through `BODY`. Each time `BODY` produces a value,
-          `$VAR` is set to that value and jq backtracks to find the
-          next one. When `BODY` stops producing values, the final
-          value of `$VAR` is the result of the entire expression.
+              0 | (3 as $item | . + $item) | 
+                  (2 as $item | . + $item) |
+                  (1 as $item | . + $item)
 
         examples:
-          - program: 'fold 0 as $sum (.[] | $sum + .)'
+          - program: 'reduce .[] as $item (0; . + $item)'
             input: '[10,2,5,3]'
             output: ['20']
       

docs/content/index/index.yml

@@ -24,4 +24,11 @@ body3: |
   
 tail: |
   Go read the [tutorial](tutorial/) for more, or the [manual](manual/)
-  for *way* more.
+  for *way* more.
+
+news:
+  - date: 2013-05-19
+    body: |
+      
+      jq 1.3 (finally) released! Get it on the [download](download/) page.
+

docs/public/css/base.scss

@@ -89,4 +89,11 @@ h3 code {
 
 footer {
     text-align: center;
+}
+
+#news {
+    font-size: 12pt;
+    .date {
+          font-style: italic;
+    }
 }

docs/site.yml

@@ -4,7 +4,7 @@
 
 # This line is modified by the Makefile. To change the version number,
 # edit the Autoconf version number at the top of configure.ac
-jq_version: 1.2
+jq_version: "1.3"
 
 root: '/jq'
 footer: |

docs/templates/default.liquid

@@ -10,7 +10,7 @@
         <h1>{{headline}}</h1>
         {% for item in body %}
           {% if item.text %}
-            {{ item.text | markdownify }}
+            {{ item.text | replace: '$JQ_VERSION', jq_version | markdownify }}
           {% endif %}
         
           {% if item.command %}

docs/templates/index.liquid

@@ -35,6 +35,16 @@
       <div class="row" style="text-align:center; margin-top: 30px">
         {{tail | markdownify}}
       </div>
+      <div class="row">
+        <h2>News</h2>
+        <div id="news">
+          <ul>
+            {% for item in news %}
+            <li><span class="date">{{ item.date | date: '%m %B %Y' }}</span>{{item.body | markdownify}}</li>
+            {% endfor %}
+          </ul>
+        </div>
+      </div>
     </div>
     {% include "shared/footer" %}    
   </body>

execute.c

@@ -19,85 +19,56 @@
 #include "parser.h"
 #include "builtin.h"
 
-typedef struct {
-  jv value;
-  int pathidx;
-} stackval;
-
 struct jq_state {
   struct forkable_stack data_stk;
   struct forkable_stack frame_stk;
   struct forkable_stack fork_stk;
-  jv* pathbuf;
-  int pathsize; // number of allocated elements
+  jv path;
+  int subexp_nest;
   int debug_trace_enabled;
+  int initial_execution;
 };
 
-int path_push(jq_state *jq, stackval sv, jv val) {
-  int pos = sv.pathidx;
-  assert(pos <= jq->pathsize);
-  assert(pos >= 0);
-  if (pos == jq->pathsize) {
-    int oldpathsize = jq->pathsize;
-    jq->pathsize = oldpathsize ? oldpathsize * 2 : 100;
-    jq->pathbuf = jv_mem_realloc(jq->pathbuf, sizeof(jq->pathbuf[0]) * jq->pathsize);
-    for (int i=oldpathsize; i<jq->pathsize; i++) {
-      jq->pathbuf[i] = jv_invalid();
-    }
-  }
-  jv_free(jq->pathbuf[pos]);
-  jq->pathbuf[pos] = val;
-  return pos + 1;
-}
-
-stackval stackval_replace(stackval value, jv newjs) {
-  jv_free(value.value);
-  stackval s = {newjs, value.pathidx};
-  return s;
-}
-
-
-// Probably all uses of this function are bugs
-stackval stackval_root(jv v) {
-  stackval s = {v, 0};
-  return s;
-}
-
 typedef struct {
   FORKABLE_STACK_HEADER;
-  stackval sv;
+  jv val;
 } data_stk_elem;
 
-void stack_push(jq_state *jq, stackval val) {
-  assert(jv_is_valid(val.value));
+void stack_push(jq_state *jq, jv val) {
+  assert(jv_is_valid(val));
   data_stk_elem* s = forkable_stack_push(&jq->data_stk, sizeof(data_stk_elem));
-  s->sv = val;
+  s->val = val;
 }
 
-stackval stack_pop(jq_state *jq) {
+jv stack_pop(jq_state *jq) {
   data_stk_elem* s = forkable_stack_peek(&jq->data_stk);
-  stackval sv = s->sv;
+  jv val = s->val;
   if (!forkable_stack_pop_will_free(&jq->data_stk)) {
-    sv.value = jv_copy(sv.value);
+    val = jv_copy(val);
   }
   forkable_stack_pop(&jq->data_stk);
-  assert(jv_is_valid(sv.value));
-  return sv;
+  assert(jv_is_valid(val));
+  return val;
 }
 
 
-
 struct forkpoint {
   FORKABLE_STACK_HEADER;
   struct forkable_stack_state saved_data_stack;
   struct forkable_stack_state saved_call_stack;
+  int path_len, subexp_nest;
+  uint16_t* return_address;
 };
 
 
-void stack_save(jq_state *jq){
+void stack_save(jq_state *jq, uint16_t* retaddr){
   struct forkpoint* fork = forkable_stack_push(&jq->fork_stk, sizeof(struct forkpoint));
   forkable_stack_save(&jq->data_stk, &fork->saved_data_stack);
   forkable_stack_save(&jq->frame_stk, &fork->saved_call_stack);
+  fork->path_len = 
+    jv_get_kind(jq->path) == JV_KIND_ARRAY ? jv_array_length(jv_copy(jq->path)) : 0;
+  fork->subexp_nest = jq->subexp_nest;
+  fork->return_address = retaddr;
 }
 
 void stack_switch(jq_state *jq) {
@@ -106,26 +77,48 @@ void stack_switch(jq_state *jq) {
   forkable_stack_switch(&jq->frame_stk, &fork->saved_call_stack);
 }
 
-int stack_restore(jq_state *jq){
+void path_append(jq_state* jq, jv component) {
+  if (jq->subexp_nest == 0 && jv_get_kind(jq->path) == JV_KIND_ARRAY) {
+    int n1 = jv_array_length(jv_copy(jq->path));
+    jq->path = jv_array_append(jq->path, component);
+    int n2 = jv_array_length(jv_copy(jq->path));
+    assert(n2 == n1 + 1);
+  } else {
+    jv_free(component);
+  }
+}
+
+uint16_t* stack_restore(jq_state *jq){
   while (!forkable_stack_empty(&jq->data_stk) && 
          forkable_stack_pop_will_free(&jq->data_stk)) {
-    jv_free(stack_pop(jq).value);
+    jv_free(stack_pop(jq));
   }
   while (!forkable_stack_empty(&jq->frame_stk) && 
          forkable_stack_pop_will_free(&jq->frame_stk)) {
     frame_pop(&jq->frame_stk);
   }
+
   if (forkable_stack_empty(&jq->fork_stk)) {
     return 0;
-  } else {
-    struct forkpoint* fork = forkable_stack_peek(&jq->fork_stk);
-    forkable_stack_restore(&jq->data_stk, &fork->saved_data_stack);
-    forkable_stack_restore(&jq->frame_stk, &fork->saved_call_stack);
-    forkable_stack_pop(&jq->fork_stk);
-    return 1;
   }
+
+  struct forkpoint* fork = forkable_stack_peek(&jq->fork_stk);
+  uint16_t* retaddr = fork->return_address;
+  forkable_stack_restore(&jq->data_stk, &fork->saved_data_stack);
+  forkable_stack_restore(&jq->frame_stk, &fork->saved_call_stack);
+  int path_len = fork->path_len;
+  if (jv_get_kind(jq->path) == JV_KIND_ARRAY) {
+    assert(path_len >= 0);
+    jq->path = jv_array_slice(jq->path, 0, path_len);
+  } else {
+    assert(path_len == 0);
+  }
+  jq->subexp_nest = fork->subexp_nest;
+  forkable_stack_pop(&jq->fork_stk);
+  return retaddr;
 }
 
+
 static struct closure make_closure(struct forkable_stack* stk, frame_ptr fr, uint16_t* pc) {
   uint16_t level = *pc++;
   uint16_t idx = *pc++;
@@ -154,13 +147,11 @@ void print_error(jv value) {
 jv jq_next(jq_state *jq) {
   jv cfunc_input[MAX_CFUNCTION_ARGS];
 
-  assert(!forkable_stack_empty(&jq->frame_stk));
-  uint16_t* pc = *frame_current_retaddr(&jq->frame_stk);
-  frame_pop(&jq->frame_stk);
-  
-  assert(!forkable_stack_empty(&jq->frame_stk));
+  uint16_t* pc = stack_restore(jq);
+  assert(pc);
 
-  int backtracking = 0;
+  int backtracking = !jq->initial_execution;
+  jq->initial_execution = 0;
   while (1) {
     uint16_t opcode = *pc;
 
@@ -179,8 +170,10 @@ jv jq_next(jq_state *jq) {
           param = forkable_stack_peek_next(&jq->data_stk, param);
         }
         if (!param) break;
-        jv_dump(jv_copy(param->sv.value), 0);
-        printf("<%d>", jv_get_refcnt(param->sv.value));
+        jv_dump(jv_copy(param->val), 0);
+        //printf("<%d>", jv_get_refcnt(param->val));
+        //printf(" -- ");
+        //jv_dump(jv_copy(jq->path), 0);
       }
 
       if (backtracking) printf("\t<backtracking>");
@@ -199,46 +192,52 @@ jv jq_next(jq_state *jq) {
     case LOADK: {
       jv v = jv_array_get(jv_copy(frame_current_bytecode(&jq->frame_stk)->constants), *pc++);
       assert(jv_is_valid(v));
-      stack_push(jq, stackval_replace(stack_pop(jq), v));
+      jv_free(stack_pop(jq));
+      stack_push(jq, v);
       break;
     }
 
     case DUP: {
-      stackval v = stack_pop(jq);
-      stackval v2 = v;
-      v2.value = jv_copy(v.value);
+      jv v = stack_pop(jq);
+      stack_push(jq, jv_copy(v));
       stack_push(jq, v);
-      stack_push(jq, v2);
       break;
     }
 
     case DUP2: {
-      stackval keep = stack_pop(jq);
-      stackval v = stack_pop(jq);
-      stackval v2 = v;
-      v2.value = jv_copy(v.value);
-      stack_push(jq, v);
+      jv keep = stack_pop(jq);
+      jv v = stack_pop(jq);
+      stack_push(jq, jv_copy(v));
       stack_push(jq, keep);
-      stack_push(jq, v2);
+      stack_push(jq, v);
       break;
     }
 
-    case SWAP: {
-      stackval a = stack_pop(jq);
-      stackval b = stack_pop(jq);
+    case SUBEXP_BEGIN: {
+      jv v = stack_pop(jq);
+      stack_push(jq, jv_copy(v));
+      stack_push(jq, v);
+      jq->subexp_nest++;
+      break;
+    }
+
+    case SUBEXP_END: {
+      assert(jq->subexp_nest > 0);
+      jq->subexp_nest--;
+      jv a = stack_pop(jq);
+      jv b = stack_pop(jq);
       stack_push(jq, a);
       stack_push(jq, b);
       break;
     }
       
     case POP: {
-      jv_free(stack_pop(jq).value);
+      jv_free(stack_pop(jq));
       break;
     }
 
     case APPEND: {
-      // FIXME paths
-      jv v = stack_pop(jq).value;
+      jv v = stack_pop(jq);
       uint16_t level = *pc++;
       uint16_t vidx = *pc++;
       frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
@@ -249,15 +248,50 @@ jv jq_next(jq_state *jq) {
     }
 
     case INSERT: {
-      stackval stktop = stack_pop(jq);
-      jv v = stack_pop(jq).value;
-      jv k = stack_pop(jq).value;
-      stackval objv = stack_pop(jq);
-      assert(jv_get_kind(k) == JV_KIND_STRING);
-      assert(jv_get_kind(objv.value) == JV_KIND_OBJECT);
-      objv.value = jv_object_set(objv.value, k, v);
-      stack_push(jq, objv);
-      stack_push(jq, stktop);
+      jv stktop = stack_pop(jq);
+      jv v = stack_pop(jq);
+      jv k = stack_pop(jq);
+      jv objv = stack_pop(jq);
+      assert(jv_get_kind(objv) == JV_KIND_OBJECT);
+      if (jv_get_kind(k) == JV_KIND_STRING) {
+        stack_push(jq, jv_object_set(objv, k, v));
+        stack_push(jq, stktop);
+      } else {
+        print_error(jv_invalid_with_msg(jv_string_fmt("Cannot use %s as object key",
+                                                      jv_kind_name(jv_get_kind(k)))));
+        jv_free(stktop);
+        jv_free(v);
+        jv_free(k);
+        jv_free(objv);
+        goto do_backtrack;
+      }
+      break;
+    }
+
+    case ON_BACKTRACK(RANGE):
+    case RANGE: {
+      uint16_t level = *pc++;
+      uint16_t v = *pc++;
+      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
+      jv* var = frame_local_var(fp, v);
+      jv max = stack_pop(jq);
+      if (jv_get_kind(*var) != JV_KIND_NUMBER ||
+          jv_get_kind(max) != JV_KIND_NUMBER) {
+        print_error(jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric")));
+        jv_free(max);
+        goto do_backtrack;
+      } else if (jv_number_value(jv_copy(*var)) >= jv_number_value(jv_copy(max))) {
+        /* finished iterating */
+        goto do_backtrack;
+      } else {
+        jv curr = jv_copy(*var);
+        *var = jv_number(jv_number_value(*var) + 1);
+
+        stack_save(jq, pc - 3);
+        stack_push(jq, jv_copy(max));
+        stack_switch(jq);
+        stack_push(jq, curr);
+      }
       break;
     }
 
@@ -272,7 +306,25 @@ jv jq_next(jq_state *jq) {
         jv_dump(jv_copy(*var), 0);
         printf("\n");
       }
-      stack_push(jq, stackval_replace(stack_pop(jq), jv_copy(*var)));
+      jv_free(stack_pop(jq));
+      stack_push(jq, jv_copy(*var));
+      break;
+    }
+
+      // Does a load but replaces the variable with null
+    case LOADVN: {
+      uint16_t level = *pc++;
+      uint16_t v = *pc++;
+      frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
+      jv* var = frame_local_var(fp, v);
+      if (jq->debug_trace_enabled) {
+        printf("V%d = ", v);
+        jv_dump(jv_copy(*var), 0);
+        printf("\n");
+      }
+      jv_free(stack_pop(jq));
+      stack_push(jq, *var);
+      *var = jv_null();
       break;
     }
 
@@ -281,39 +333,64 @@ jv jq_next(jq_state *jq) {
       uint16_t v = *pc++;
       frame_ptr fp = frame_get_level(&jq->frame_stk, frame_current(&jq->frame_stk), level);
       jv* var = frame_local_var(fp, v);
-      stackval val = stack_pop(jq);
+      jv val = stack_pop(jq);
       if (jq->debug_trace_enabled) {
         printf("V%d = ", v);
-        jv_dump(jv_copy(val.value), 0);
+        jv_dump(jv_copy(val), 0);
         printf("\n");
       }
       jv_free(*var);
-      *var = val.value;
+      *var = val;
       break;
     }
 
-    case GETPATH: {
-      stackval path_end = stack_pop(jq);
-      stackval path_start = stack_pop(jq);
-      jv_free(path_end.value);
-      jv path = jv_array();
-      for (int i=path_start.pathidx; i<path_end.pathidx; i++) {
-        path = jv_array_set(path, i, jv_copy(jq->pathbuf[i]));
-      }
-      stack_push(jq, stackval_replace(path_start, path));
+    case PATH_BEGIN: {
+      jv v = stack_pop(jq);
+      stack_push(jq, jq->path);
+
+      stack_save(jq, pc - 1);
+      stack_switch(jq);
+
+      stack_push(jq, jv_number(jq->subexp_nest));
+      stack_push(jq, v);
+
+      jq->path = jv_array();
+      jq->subexp_nest = 0;
       break;
     }
 
+    case PATH_END: {
+      jv v = stack_pop(jq);
+      jv_free(v); // discard value, only keep path
+
+      int old_subexp_nest = (int)jv_number_value(stack_pop(jq));
+
+      jv path = jq->path;
+      jq->path = stack_pop(jq);
+
+      stack_save(jq, pc - 1);
+      stack_push(jq, jv_copy(path));
+      stack_switch(jq);
+
+      stack_push(jq, path);
+      jq->subexp_nest = old_subexp_nest;
+      break;
+    }
+
+    case ON_BACKTRACK(PATH_BEGIN):
+    case ON_BACKTRACK(PATH_END): {
+      jv_free(jq->path);
+      jq->path = stack_pop(jq);
+      goto do_backtrack;
+    }
+
     case INDEX: {
-      stackval t = stack_pop(jq);
-      jv k = stack_pop(jq).value;
-      int pathidx = path_push(jq, t, jv_copy(k));
-      jv v = jv_get(t.value, k);
+      jv t = stack_pop(jq);
+      jv k = stack_pop(jq);
+      path_append(jq, jv_copy(k));
+      jv v = jv_get(t, k);
       if (jv_is_valid(v)) {
-        stackval sv;
-        sv.value = v;
-        sv.pathidx = pathidx;
-        stack_push(jq, sv);
+        stack_push(jq, v);
       } else {
         print_error(v);
         goto do_backtrack;
@@ -330,8 +407,8 @@ jv jq_next(jq_state *jq) {
 
     case JUMP_F: {
       uint16_t offset = *pc++;
-      stackval t = stack_pop(jq);
-      jv_kind kind = jv_get_kind(t.value);
+      jv t = stack_pop(jq);
+      jv_kind kind = jv_get_kind(t);
       if (kind == JV_KIND_FALSE || kind == JV_KIND_NULL) {
         pc += offset;
       }
@@ -340,68 +417,70 @@ jv jq_next(jq_state *jq) {
     }
 
     case EACH: 
-      stack_push(jq, stackval_root(jv_number(-1)));
+      stack_push(jq, jv_number(-1));
       // fallthrough
     case ON_BACKTRACK(EACH): {
-      int idx = jv_number_value(stack_pop(jq).value);
-      stackval container = stack_pop(jq);
+      int idx = jv_number_value(stack_pop(jq));
+      jv container = stack_pop(jq);
 
-      int keep_going;
+      int keep_going, is_last = 0;
       jv key, value;
-      if (jv_get_kind(container.value) == JV_KIND_ARRAY) {
+      if (jv_get_kind(container) == JV_KIND_ARRAY) {
         if (opcode == EACH) idx = 0;
         else idx = idx + 1;
-        keep_going = idx < jv_array_length(jv_copy(container.value));
+        int len = jv_array_length(jv_copy(container));
+        keep_going = idx < len;
+        is_last = idx == len - 1;
         if (keep_going) {
           key = jv_number(idx);
-          value = jv_array_get(jv_copy(container.value), idx);
+          value = jv_array_get(jv_copy(container), idx);
         }
-      } else if (jv_get_kind(container.value) == JV_KIND_OBJECT) {
-        if (opcode == EACH) idx = jv_object_iter(container.value);
-        else idx = jv_object_iter_next(container.value, idx);
-        keep_going = jv_object_iter_valid(container.value, idx);
+      } else if (jv_get_kind(container) == JV_KIND_OBJECT) {
+        if (opcode == EACH) idx = jv_object_iter(container);
+        else idx = jv_object_iter_next(container, idx);
+        keep_going = jv_object_iter_valid(container, idx);
         if (keep_going) {
-          key = jv_object_iter_key(container.value, idx);
-          value = jv_object_iter_value(container.value, idx);
+          key = jv_object_iter_key(container, idx);
+          value = jv_object_iter_value(container, idx);
         }
       } else {
         assert(opcode == EACH);
         print_error(jv_invalid_with_msg(jv_string_fmt("Cannot iterate over %s",
-                                                      jv_kind_name(jv_get_kind(container.value)))));
+                                                      jv_kind_name(jv_get_kind(container)))));
         keep_going = 0;
       }
 
       if (!keep_going) {
-        jv_free(container.value);
+        jv_free(container);
         goto do_backtrack;
+      } else if (is_last) {
+        // we don't need to make a backtrack point
+        jv_free(container);
+        path_append(jq, key);
+        stack_push(jq, value);
       } else {
-        stack_save(jq);
+        stack_save(jq, pc - 1);
         stack_push(jq, container);
-        stack_push(jq, stackval_root(jv_number(idx)));
-        frame_push_backtrack(&jq->frame_stk, pc - 1);
+        stack_push(jq, jv_number(idx));
         stack_switch(jq);
-        
-        stackval sv = {value,
-                       path_push(jq, container, key)};
-        stack_push(jq, sv);
+        path_append(jq, key);
+        stack_push(jq, value);
       }
       break;
     }
 
     do_backtrack:
     case BACKTRACK: {
-      if (!stack_restore(jq)) {
+      pc = stack_restore(jq);
+      if (!pc) {
         return jv_invalid();
       }
-      pc = *frame_current_retaddr(&jq->frame_stk);
-      frame_pop(&jq->frame_stk);
       backtracking = 1;
       break;
     }
 
     case FORK: {
-      stack_save(jq);
-      frame_push_backtrack(&jq->frame_stk, pc - 1);
+      stack_save(jq, pc - 1);
       stack_switch(jq);
       pc++; // skip offset this time
       break;
@@ -412,26 +491,20 @@ jv jq_next(jq_state *jq) {
       pc += offset;
       break;
     }
-
-    case YIELD: {
-      jv value = stack_pop(jq).value;
-      frame_push_backtrack(&jq->frame_stk, pc);
-      return value;
-    }
       
     case CALL_BUILTIN: {
       int nargs = *pc++;
-      stackval top = stack_pop(jq);
-      cfunc_input[0] = top.value;
+      jv top = stack_pop(jq);
+      cfunc_input[0] = top;
       for (int i = 1; i < nargs; i++) {
-        cfunc_input[i] = stack_pop(jq).value;
+        cfunc_input[i] = stack_pop(jq);
       }
       struct cfunction* func = &frame_current_bytecode(&jq->frame_stk)->globals->cfunctions[*pc++];
-      top.value = cfunction_invoke(func, cfunc_input);
-      if (jv_is_valid(top.value)) {
+      top = cfunction_invoke(func, cfunc_input);
+      if (jv_is_valid(top)) {
         stack_push(jq, top);
       } else {
-        print_error(top.value);
+        print_error(top);
         goto do_backtrack;
       }
       break;
@@ -456,10 +529,25 @@ jv jq_next(jq_state *jq) {
     }
 
     case RET: {
-      pc = *frame_current_retaddr(&jq->frame_stk);
-      frame_pop(&jq->frame_stk);
+      uint16_t* retaddr = *frame_current_retaddr(&jq->frame_stk);
+      if (retaddr) {
+        // function return
+        pc = retaddr;
+        frame_pop(&jq->frame_stk);
+      } else {
+        // top-level return, yielding value
+        jv value = stack_pop(jq);
+        stack_save(jq, pc - 1);
+        stack_push(jq, jv_null());
+        stack_switch(jq);
+        return value;
+      }
       break;
     }
+    case ON_BACKTRACK(RET): {
+      // resumed after top-level return
+      goto do_backtrack;
+    }
     }
   }
 }
@@ -469,19 +557,22 @@ void jq_init(struct bytecode* bc, jv input, jq_state **jq, int flags) {
   jq_state *new_jq;
   new_jq = jv_mem_alloc(sizeof(*new_jq));
   memset(new_jq, 0, sizeof(*new_jq));
-  forkable_stack_init(&new_jq->data_stk, sizeof(stackval) * 100);
+  new_jq->path = jv_null();
+  forkable_stack_init(&new_jq->data_stk, sizeof(data_stk_elem) * 100);
   forkable_stack_init(&new_jq->frame_stk, 1024);
   forkable_stack_init(&new_jq->fork_stk, 1024);
   
-  stack_push(new_jq, stackval_root(input));
+  stack_push(new_jq, input);
   struct closure top = {bc, -1};
   frame_push(&new_jq->frame_stk, top, 0);
-  frame_push_backtrack(&new_jq->frame_stk, bc->code);
+  stack_save(new_jq, bc->code);
+  stack_switch(new_jq);
   if (flags & JQ_DEBUG_TRACE) {
     new_jq->debug_trace_enabled = 1;
   } else {
     new_jq->debug_trace_enabled = 0;
   }
+  new_jq->initial_execution = 1;
   *jq = new_jq;
 }
 
@@ -500,10 +591,7 @@ void jq_teardown(jq_state **jq) {
   forkable_stack_free(&old_jq->data_stk);
   forkable_stack_free(&old_jq->frame_stk);
 
-  for (int i=0; i<old_jq->pathsize; i++) {
-    jv_free(old_jq->pathbuf[i]);
-  }
-  jv_mem_free(old_jq->pathbuf);
+  jv_free(old_jq->path);
   jv_mem_free(old_jq);
 }
 

frame_layout.h

@@ -11,8 +11,6 @@ struct continuation {
   struct bytecode* bc;
   stack_idx env;
   uint16_t* retaddr;
-  // FIXME: probably not necessary as a separate field
-  int is_backtrack_frame;
 };
 
 typedef union frame_elem {
@@ -86,27 +84,15 @@ static frame_ptr frame_push(struct forkable_stack* stk, struct closure cl, uint1
   cc->bc = cl.bc;
   cc->env = cl.env;
   cc->retaddr = retaddr;
-  cc->is_backtrack_frame = 0;
   for (int i=0; i<cl.bc->nlocals; i++) {
     *frame_local_var(fp, i) = jv_invalid();
   }
   return fp;
 }
 
-static frame_ptr frame_push_backtrack(struct forkable_stack* stk, uint16_t* retaddr) {
-  struct continuation cc = *frame_self(frame_current(stk));
-  frame_ptr fp = forkable_stack_push(stk, sizeof(union frame_elem) * 2);
-  assert(!cc.is_backtrack_frame);
-  cc.is_backtrack_frame = 1;
-  cc.retaddr = retaddr;
-  *frame_self(fp) = cc;
-  return fp;
-}
-
 static void frame_pop(struct forkable_stack* stk) {
   frame_ptr fp = frame_current(stk);
-  if (forkable_stack_pop_will_free(stk) && 
-      !frame_self(fp)->is_backtrack_frame) {
+  if (forkable_stack_pop_will_free(stk)) {
     int nlocals = frame_self(fp)->bc->nlocals;
     for (int i=0; i<nlocals; i++) {
       jv_free(*frame_local_var(fp, i));

jq_test.c

@@ -63,7 +63,7 @@ static void run_jq_tests(FILE *testdata) {
         pass = 0;
       }
       jv as_string = jv_dump_string(jv_copy(expected), rand() & ~JV_PRINT_COLOUR);
-      jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length(jv_copy(as_string)));
+      jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length_bytes(jv_copy(as_string)));
       assert(jv_equal(jv_copy(expected), jv_copy(reparsed)));
       jv_free(as_string);
       jv_free(reparsed);
@@ -115,9 +115,9 @@ static void jv_test() {
     jv_free(a2);
 
 
-    assert(a.val.complex.ptr->count == 1);
+    assert(a.val.nontrivial.ptr->count == 1);
     a = jv_array_append(a, jv_copy(a));
-    assert(a.val.complex.ptr->count == 1);
+    assert(a.val.nontrivial.ptr->count == 1);
 
     assert(jv_array_length(jv_copy(a)) == 2);
     assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
@@ -150,9 +150,9 @@ static void jv_test() {
 
     jv_free(subarray);
 
-    void* before = sub2.val.complex.ptr;
+    void* before = sub2.val.nontrivial.ptr;
     sub2 = jv_array_append(sub2, jv_number(200));
-    void* after = sub2.val.complex.ptr;
+    void* after = sub2.val.nontrivial.ptr;
     assert(before == after);
     jv_free(sub2);
 
@@ -164,6 +164,18 @@ static void jv_test() {
     jv_free(a3);
 
 
+    jv a4 = jv_array();
+    a4 = jv_array_append(a4, jv_number(1));
+    a4 = jv_array_append(a4, jv_number(2));
+    jv a5 = jv_copy(a4);
+    a4 = jv_array_append(a4, jv_number(3));
+    a4 = jv_array_slice(a4, 0, 1);
+    assert(jv_array_length(jv_copy(a4)) == 1);
+    a4 = jv_array_append(a4, jv_number(4));
+    assert(jv_array_length(a4) == 2);
+    assert(jv_array_length(a5) == 2);
+
+
     assert(jv_array_length(jv_copy(a)) == 2);
     assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
     assert(jv_array_length(jv_array_get(jv_copy(a), 1)) == 1);
@@ -179,8 +191,8 @@ static void jv_test() {
     assert(jv_equal(jv_string("foo"), jv_string_sized("foo", 3)));
     char nasty[] = "foo\0";
     jv shortstr = jv_string(nasty), longstr = jv_string_sized(nasty, sizeof(nasty));
-    assert(jv_string_length(shortstr) == (int)strlen(nasty));
-    assert(jv_string_length(longstr) == (int)sizeof(nasty));
+    assert(jv_string_length_bytes(shortstr) == (int)strlen(nasty));
+    assert(jv_string_length_bytes(longstr) == (int)sizeof(nasty));
 
   
     char a1s[] = "hello", a2s[] = "hello", bs[] = "goodbye";
@@ -201,7 +213,7 @@ static void jv_test() {
     for (int i=0; i<(int)sizeof(big); i++) big[i] = 'a';
     big[sizeof(big)-1] = 0;
     jv str = jv_string_fmt("%s", big);
-    assert(jv_string_length(jv_copy(str)) == sizeof(big) - 1);
+    assert(jv_string_length_bytes(jv_copy(str)) == sizeof(big) - 1);
     assert(!strcmp(big, jv_string_value(str)));
     jv_free(str);
   }

jv.c

@@ -8,25 +8,26 @@
 
 #include "jv_alloc.h"
 #include "jv.h"
+#include "jv_unicode.h"
 
 /*
  * Internal refcounting helpers
  */
 
-static void jvp_refcnt_init(jv_complex* c) {
+static void jvp_refcnt_init(jv_nontrivial* c) {
   c->ptr->count = 1;
 }
 
-static void jvp_refcnt_inc(jv_complex* c) {
+static void jvp_refcnt_inc(jv_nontrivial* c) {
   c->ptr->count++;
 }
 
-static int jvp_refcnt_dec(jv_complex* c) {
+static int jvp_refcnt_dec(jv_nontrivial* c) {
   c->ptr->count--;
   return c->ptr->count == 0;
 }
 
-static int jvp_refcnt_unshared(jv_complex* c) {
+static int jvp_refcnt_unshared(jv_nontrivial* c) {
   assert(c->ptr->count > 0);
   return c->ptr->count == 1;
 }
@@ -86,9 +87,9 @@ typedef struct {
 jv jv_invalid_with_msg(jv err) {
   jv x;
   x.kind = JV_KIND_INVALID;
-  x.val.complex.i[0] = x.val.complex.i[1] = 0;
+  x.val.nontrivial.i[0] = x.val.nontrivial.i[1] = 0;
   jvp_invalid* i = jv_mem_alloc(sizeof(jvp_invalid));
-  x.val.complex.ptr = &i->refcnt;
+  x.val.nontrivial.ptr = &i->refcnt;
   i->refcnt.count = 1;
   i->errmsg = err;
   return x;
@@ -99,7 +100,7 @@ jv jv_invalid() {
 }
 
 jv jv_invalid_get_msg(jv inv) {
-  jv x = jv_copy(((jvp_invalid*)inv.val.complex.ptr)->errmsg);
+  jv x = jv_copy(((jvp_invalid*)inv.val.nontrivial.ptr)->errmsg);
   jv_free(inv);
   return x;
 }
@@ -111,7 +112,7 @@ int jv_invalid_has_msg(jv inv) {
   return r;
 }
 
-static void jvp_invalid_free(jv_complex* x) {
+static void jvp_invalid_free(jv_nontrivial* x) {
   if (jvp_refcnt_dec(x)) {
     jv_free(((jvp_invalid*)x->ptr)->errmsg);
     jv_mem_free(x->ptr);
@@ -153,7 +154,7 @@ typedef struct {
   jv elements[];
 } jvp_array;
 
-static jvp_array* jvp_array_ptr(jv_complex* a) {
+static jvp_array* jvp_array_ptr(jv_nontrivial* a) {
   return (jvp_array*)a->ptr;
 }
 
@@ -165,12 +166,12 @@ static jvp_array* jvp_array_alloc(unsigned size) {
   return a;
 }
 
-static jv_complex jvp_array_new(unsigned size) {
-  jv_complex r = {&jvp_array_alloc(size)->refcnt, {0, 0}};
+static jv_nontrivial jvp_array_new(unsigned size) {
+  jv_nontrivial r = {&jvp_array_alloc(size)->refcnt, {0, 0}};
   return r;
 }
 
-static void jvp_array_free(jv_complex* a) {
+static void jvp_array_free(jv_nontrivial* a) {
   if (jvp_refcnt_dec(a)) {
     jvp_array* array = jvp_array_ptr(a);
     for (int i=0; i<array->length; i++) {
@@ -180,11 +181,11 @@ static void jvp_array_free(jv_complex* a) {
   }
 }
 
-static int jvp_array_length(jv_complex* a) {
+static int jvp_array_length(jv_nontrivial* a) {
   return a->i[1] - a->i[0];
 }
 
-static jv* jvp_array_read(jv_complex* a, int i) {
+static jv* jvp_array_read(jv_nontrivial* a, int i) {
   if (i >= 0 && i < jvp_array_length(a)) {
     jvp_array* array = jvp_array_ptr(a);
     assert(i + a->i[0] < array->length);
@@ -194,7 +195,7 @@ static jv* jvp_array_read(jv_complex* a, int i) {
   }
 }
 
-static jv* jvp_array_write(jv_complex* a, int i) {
+static jv* jvp_array_write(jv_nontrivial* a, int i) {
   assert(i >= 0);
   jvp_array* array = jvp_array_ptr(a);
 
@@ -214,7 +215,7 @@ static jv* jvp_array_write(jv_complex* a, int i) {
         array->elements[j] = JV_NULL;
       }
       array->length = imax(pos + 1, array->length);
-      a->i[1] = imax(pos + 1, array->length);
+      a->i[1] = imax(pos + 1, a->i[1]);
       return &array->elements[pos];
     }
   }
@@ -237,7 +238,7 @@ static jv* jvp_array_write(jv_complex* a, int i) {
   return &new_array->elements[i];
 }
 
-static int jvp_array_equal(jv_complex* a, jv_complex* b) {
+static int jvp_array_equal(jv_nontrivial* a, jv_nontrivial* b) {
   if (jvp_array_length(a) != jvp_array_length(b)) 
     return 0;
   if (jvp_array_ptr(a) == jvp_array_ptr(b) &&
@@ -251,11 +252,11 @@ static int jvp_array_equal(jv_complex* a, jv_complex* b) {
   return 1;
 }
 
-static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
+static jv_nontrivial jvp_array_slice(jv_nontrivial* a, int start, int end) {
   // FIXME: maybe slice should reallocate if the slice is small enough
   assert(start <= end);
   assert(a->i[0] + end <= a->i[1]);
-  jv_complex slice = *a;
+  jv_nontrivial slice = *a;
   slice.i[0] += start;
   slice.i[1] = slice.i[0] + (end - start);
   return slice;
@@ -268,7 +269,7 @@ static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
 jv jv_array_sized(int n) {
   jv j;
   j.kind = JV_KIND_ARRAY;
-  j.val.complex = jvp_array_new(n);
+  j.val.nontrivial = jvp_array_new(n);
   return j;
 }
 
@@ -278,14 +279,14 @@ jv jv_array() {
 
 int jv_array_length(jv j) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
-  int len = jvp_array_length(&j.val.complex);
+  int len = jvp_array_length(&j.val.nontrivial);
   jv_free(j);
   return len;
 }
 
 jv jv_array_get(jv j, int idx) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
-  jv* slot = jvp_array_read(&j.val.complex, idx);
+  jv* slot = jvp_array_read(&j.val.nontrivial, idx);
   jv val;
   if (slot) {
     val = jv_copy(*slot);
@@ -299,7 +300,7 @@ jv jv_array_get(jv j, int idx) {
 jv jv_array_set(jv j, int idx, jv val) {
   assert(jv_get_kind(j) == JV_KIND_ARRAY);
   // copy/free of val,j coalesced
-  jv* slot = jvp_array_write(&j.val.complex, idx);
+  jv* slot = jvp_array_write(&j.val.nontrivial, idx);
   jv_free(*slot);
   *slot = val;
   return j;
@@ -325,7 +326,7 @@ jv jv_array_concat(jv a, jv b) {
 jv jv_array_slice(jv a, int start, int end) {
   assert(jv_get_kind(a) == JV_KIND_ARRAY);
   // copy/free of a coalesced
-  a.val.complex = jvp_array_slice(&a.val.complex, start, end);
+  a.val.nontrivial = jvp_array_slice(&a.val.nontrivial, start, end);
   return a;
 }
 
@@ -365,7 +366,7 @@ typedef struct {
   char data[];
 } jvp_string;
 
-static jvp_string* jvp_string_ptr(jv_complex* a) {
+static jvp_string* jvp_string_ptr(jv_nontrivial* a) {
   return (jvp_string*)a->ptr;
 }
 
@@ -376,16 +377,16 @@ static jvp_string* jvp_string_alloc(uint32_t size) {
   return s;
 }
 
-static jv_complex jvp_string_new(const char* data, uint32_t length) {
+static jv_nontrivial jvp_string_new(const char* data, uint32_t length) {
   jvp_string* s = jvp_string_alloc(length);
   s->length_hashed = length << 1;
   memcpy(s->data, data, length);
   s->data[length] = 0;
-  jv_complex r = {&s->refcnt, {0,0}};
+  jv_nontrivial r = {&s->refcnt, {0,0}};
   return r;
 }
 
-static void jvp_string_free(jv_complex* s) {
+static void jvp_string_free(jv_nontrivial* s) {
   if (jvp_refcnt_dec(s)) {
     jvp_string* str = jvp_string_ptr(s);
     jv_mem_free(str);
@@ -393,12 +394,12 @@ static void jvp_string_free(jv_complex* s) {
 }
 
 static void jvp_string_free_p(jvp_string* s) {
-  jv_complex p = {&s->refcnt,{0,0}};
+  jv_nontrivial p = {&s->refcnt,{0,0}};
   jvp_string_free(&p);
 }
 
 static jvp_string* jvp_string_copy_p(jvp_string* s) {
-  jv_complex p = {&s->refcnt,{0,0}};
+  jv_nontrivial p = {&s->refcnt,{0,0}};
   jvp_refcnt_inc(&p);
   return s;
 }
@@ -413,7 +414,7 @@ static uint32_t jvp_string_remaining_space(jvp_string* s) {
   return r;
 }
 
-static void jvp_string_append(jv_complex* string, const char* data, uint32_t len) {
+static void jvp_string_append(jv_nontrivial* string, const char* data, uint32_t len) {
   jvp_string* s = jvp_string_ptr(string);
   uint32_t currlen = jvp_string_length(s);
     
@@ -433,7 +434,7 @@ static void jvp_string_append(jv_complex* string, const char* data, uint32_t len
     memcpy(news->data + currlen, data, len);
     news->data[currlen + len] = 0;
     jvp_string_free(string);
-    jv_complex r = {&news->refcnt, {0,0}};
+    jv_nontrivial r = {&news->refcnt, {0,0}};
     *string = r;
   }
 }
@@ -508,7 +509,7 @@ static int jvp_string_equal_hashed(jvp_string* a, jvp_string* b) {
   return memcmp(a->data, b->data, jvp_string_length(a)) == 0;
 }
 
-static int jvp_string_equal(jv_complex* a, jv_complex* b) {
+static int jvp_string_equal(jv_nontrivial* a, jv_nontrivial* b) {
   jvp_string* stra = jvp_string_ptr(a);
   jvp_string* strb = jvp_string_ptr(b);
   if (jvp_string_length(stra) != jvp_string_length(strb)) return 0;
@@ -522,7 +523,7 @@ static int jvp_string_equal(jv_complex* a, jv_complex* b) {
 jv jv_string_sized(const char* str, int len) {
   jv j;
   j.kind = JV_KIND_STRING;
-  j.val.complex = jvp_string_new(str, len);
+  j.val.nontrivial = jvp_string_new(str, len);
   return j;
 }
 
@@ -530,34 +531,44 @@ jv jv_string(const char* str) {
   return jv_string_sized(str, strlen(str));
 }
 
-int jv_string_length(jv j) {
+int jv_string_length_bytes(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  int r = jvp_string_length(jvp_string_ptr(&j.val.complex));
+  int r = jvp_string_length(jvp_string_ptr(&j.val.nontrivial));
   jv_free(j);
   return r;
 }
 
+int jv_string_length_codepoints(jv j) {
+  assert(jv_get_kind(j) == JV_KIND_STRING);
+  const char* i = jv_string_value(j);
+  const char* end = i + jv_string_length_bytes(jv_copy(j));
+  int c = 0, len = 0;
+  while ((i = jvp_utf8_next(i, end, &c))) len++;
+  jv_free(j);
+  return len;
+}
+
 uint32_t jv_string_hash(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.complex));
+  uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.nontrivial));
   jv_free(j);
   return hash;
 }
 
 const char* jv_string_value(jv j) {
   assert(jv_get_kind(j) == JV_KIND_STRING);
-  return jvp_string_ptr(&j.val.complex)->data;
+  return jvp_string_ptr(&j.val.nontrivial)->data;
 }
 
 jv jv_string_concat(jv a, jv b) {
-  jvp_string* sb = jvp_string_ptr(&b.val.complex);
-  jvp_string_append(&a.val.complex, sb->data, jvp_string_length(sb));
+  jvp_string* sb = jvp_string_ptr(&b.val.nontrivial);
+  jvp_string_append(&a.val.nontrivial, sb->data, jvp_string_length(sb));
   jv_free(b);
   return a;
 }
 
 jv jv_string_append_buf(jv a, const char* buf, int len) {
-  jvp_string_append(&a.val.complex, buf, len);
+  jvp_string_append(&a.val.nontrivial, buf, len);
   return a;
 }
 
@@ -602,8 +613,8 @@ typedef struct {
 } jvp_object;
 
 
-/* warning: complex justification of alignment */
-static jv_complex jvp_object_new(int size) {
+/* warning: nontrivial justification of alignment */
+static jv_nontrivial jvp_object_new(int size) {
   // Allocates an object of (size) slots and (size*2) hash buckets.
 
   // size must be a power of two
@@ -620,7 +631,7 @@ static jv_complex jvp_object_new(int size) {
   }
   obj->first_free = size - 1;
   int* hashbuckets = (int*)(&obj->elements[size]);
-  jv_complex r = {&obj->refcnt, 
+  jv_nontrivial r = {&obj->refcnt, 
                   {size*2 - 1, (char*)hashbuckets - (char*)obj}};
   for (int i=0; i<size*2; i++) {
     hashbuckets[i] = -1;
@@ -628,39 +639,39 @@ static jv_complex jvp_object_new(int size) {
   return r;
 }
 
-static jvp_object* jvp_object_ptr(jv_complex* o) {
+static jvp_object* jvp_object_ptr(jv_nontrivial* o) {
   return (jvp_object*)o->ptr;
 }
 
-static uint32_t jvp_object_mask(jv_complex* o) {
+static uint32_t jvp_object_mask(jv_nontrivial* o) {
   return o->i[0];
 }
 
-static int jvp_object_size(jv_complex* o) {
+static int jvp_object_size(jv_nontrivial* o) {
   return (o->i[0] + 1) >> 1;
 }
 
-static int* jvp_object_buckets(jv_complex* o) {
+static int* jvp_object_buckets(jv_nontrivial* o) {
   int* buckets = (int*)((char*)o->ptr + o->i[1]);
   assert(buckets == (int*)&jvp_object_ptr(o)->elements[jvp_object_size(o)]);
   return buckets;
 }
 
-static int* jvp_object_find_bucket(jv_complex* object, jvp_string* key) {
+static int* jvp_object_find_bucket(jv_nontrivial* object, jvp_string* key) {
   return jvp_object_buckets(object) + (jvp_object_mask(object) & jvp_string_hash(key));
 }
 
-static struct object_slot* jvp_object_get_slot(jv_complex* object, int slot) {
+static struct object_slot* jvp_object_get_slot(jv_nontrivial* object, int slot) {
   assert(slot == -1 || (slot >= 0 && slot < jvp_object_size(object)));
   if (slot == -1) return 0;
   else return &jvp_object_ptr(object)->elements[slot];
 }
 
-static struct object_slot* jvp_object_next_slot(jv_complex* object, struct object_slot* slot) {
+static struct object_slot* jvp_object_next_slot(jv_nontrivial* object, struct object_slot* slot) {
   return jvp_object_get_slot(object, slot->next);
 }
 
-static struct object_slot* jvp_object_find_slot(jv_complex* object, jvp_string* keystr, int* bucket) {
+static struct object_slot* jvp_object_find_slot(jv_nontrivial* object, jvp_string* keystr, int* bucket) {
   for (struct object_slot* curr = jvp_object_get_slot(object, *bucket); 
        curr; 
        curr = jvp_object_next_slot(object, curr)) {
@@ -671,7 +682,7 @@ static struct object_slot* jvp_object_find_slot(jv_complex* object, jvp_string*
   return 0;
 }
 
-static struct object_slot* jvp_object_add_slot(jv_complex* object, jvp_string* key, int* bucket) {
+static struct object_slot* jvp_object_add_slot(jv_nontrivial* object, jvp_string* key, int* bucket) {
   jvp_object* o = jvp_object_ptr(object);
   int newslot_idx = o->first_free;
   struct object_slot* newslot = jvp_object_get_slot(object, newslot_idx);
@@ -684,7 +695,7 @@ static struct object_slot* jvp_object_add_slot(jv_complex* object, jvp_string* k
   return newslot;
 }
 
-static void jvp_object_free_slot(jv_complex* object, struct object_slot* slot) {
+static void jvp_object_free_slot(jv_nontrivial* object, struct object_slot* slot) {
   jvp_object* o = jvp_object_ptr(object);
   slot->next = o->first_free;
   assert(slot->string);
@@ -694,14 +705,14 @@ static void jvp_object_free_slot(jv_complex* object, struct object_slot* slot) {
   o->first_free = slot - jvp_object_get_slot(object, 0);
 }
 
-static jv* jvp_object_read(jv_complex* object, jvp_string* key) {
+static jv* jvp_object_read(jv_nontrivial* object, jvp_string* key) {
   int* bucket = jvp_object_find_bucket(object, key);
   struct object_slot* slot = jvp_object_find_slot(object, key, bucket);
   if (slot == 0) return 0;
   else return &slot->value;
 }
 
-static void jvp_object_free(jv_complex* o) {
+static void jvp_object_free(jv_nontrivial* o) {
   if (jvp_refcnt_dec(o)) {
     for (int i=0; i<jvp_object_size(o); i++) {
       struct object_slot* slot = jvp_object_get_slot(o, i);
@@ -714,10 +725,10 @@ static void jvp_object_free(jv_complex* o) {
   }
 }
 
-static void jvp_object_rehash(jv_complex* object) {
+static void jvp_object_rehash(jv_nontrivial* object) {
   assert(jvp_refcnt_unshared(object));
   int size = jvp_object_size(object);
-  jv_complex new_object = jvp_object_new(size * 2);
+  jv_nontrivial new_object = jvp_object_new(size * 2);
   for (int i=0; i<size; i++) {
     struct object_slot* slot = jvp_object_get_slot(object, i);
     if (!slot->string) continue;
@@ -733,11 +744,11 @@ static void jvp_object_rehash(jv_complex* object) {
   *object = new_object;
 }
 
-static void jvp_object_unshare(jv_complex* object) {
+static void jvp_object_unshare(jv_nontrivial* object) {
   if (jvp_refcnt_unshared(object))
     return;
 
-  jv_complex new_object = jvp_object_new(jvp_object_size(object));
+  jv_nontrivial new_object = jvp_object_new(jvp_object_size(object));
   jvp_object_ptr(&new_object)->first_free = jvp_object_ptr(object)->first_free;
   for (int i=0; i<jvp_object_size(&new_object); i++) {
     struct object_slot* old_slot = jvp_object_get_slot(object, i);
@@ -758,7 +769,7 @@ static void jvp_object_unshare(jv_complex* object) {
   assert(jvp_refcnt_unshared(object));
 }
 
-static jv* jvp_object_write(jv_complex* object, jvp_string* key) {
+static jv* jvp_object_write(jv_nontrivial* object, jvp_string* key) {
   jvp_object_unshare(object);
   int* bucket = jvp_object_find_bucket(object, key);
   struct object_slot* slot = jvp_object_find_slot(object, key, bucket);
@@ -781,7 +792,7 @@ static jv* jvp_object_write(jv_complex* object, jvp_string* key) {
   return &slot->value;
 }
 
-static int jvp_object_delete(jv_complex* object, jvp_string* key) {
+static int jvp_object_delete(jv_nontrivial* object, jvp_string* key) {
   jvp_object_unshare(object);
   int* bucket = jvp_object_find_bucket(object, key);
   int* prev_ptr = bucket;
@@ -798,7 +809,7 @@ static int jvp_object_delete(jv_complex* object, jvp_string* key) {
   return 0;
 }
 
-static int jvp_object_length(jv_complex* object) {
+static int jvp_object_length(jv_nontrivial* object) {
   int n = 0;
   for (int i=0; i<jvp_object_size(object); i++) {
     struct object_slot* slot = jvp_object_get_slot(object, i);
@@ -807,7 +818,7 @@ static int jvp_object_length(jv_complex* object) {
   return n;
 }
 
-static int jvp_object_equal(jv_complex* o1, jv_complex* o2) {
+static int jvp_object_equal(jv_nontrivial* o1, jv_nontrivial* o2) {
   int len2 = jvp_object_length(o2);
   int len1 = 0;
   for (int i=0; i<jvp_object_size(o1); i++) {
@@ -829,14 +840,14 @@ static int jvp_object_equal(jv_complex* o1, jv_complex* o2) {
 jv jv_object() {
   jv j;
   j.kind = JV_KIND_OBJECT;
-  j.val.complex = jvp_object_new(8);
+  j.val.nontrivial = jvp_object_new(8);
   return j;
 }
 
 jv jv_object_get(jv object, jv key) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
-  jv* slot = jvp_object_read(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jv* slot = jvp_object_read(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv val;
   if (slot) {
     val = jv_copy(*slot);
@@ -852,7 +863,7 @@ jv jv_object_set(jv object, jv key, jv value) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
   // copy/free of object, key, value coalesced
-  jv* slot = jvp_object_write(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jv* slot = jvp_object_write(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv_free(*slot);
   *slot = value;
   return object;
@@ -861,14 +872,14 @@ jv jv_object_set(jv object, jv key, jv value) {
 jv jv_object_delete(jv object, jv key) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(jv_get_kind(key) == JV_KIND_STRING);
-  jvp_object_delete(&object.val.complex, jvp_string_ptr(&key.val.complex));
+  jvp_object_delete(&object.val.nontrivial, jvp_string_ptr(&key.val.nontrivial));
   jv_free(key);
   return object;
 }
 
 int jv_object_length(jv object) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
-  int n = jvp_object_length(&object.val.complex);
+  int n = jvp_object_length(&object.val.nontrivial);
   jv_free(object);
   return n;
 }
@@ -919,7 +930,7 @@ int jv_object_iter(jv object) {
 int jv_object_iter_next(jv object, int iter) {
   assert(jv_get_kind(object) == JV_KIND_OBJECT);
   assert(iter != ITER_FINISHED);
-  jv_complex* o = &object.val.complex;
+  jv_nontrivial* o = &object.val.nontrivial;
   struct object_slot* slot;
   do {
     iter++;
@@ -931,18 +942,18 @@ int jv_object_iter_next(jv object, int iter) {
 }
 
 jv jv_object_iter_key(jv object, int iter) {
-  jvp_string* s = jvp_object_get_slot(&object.val.complex, iter)->string;
+  jvp_string* s = jvp_object_get_slot(&object.val.nontrivial, iter)->string;
   assert(s);
   jv j;
   j.kind = JV_KIND_STRING;
-  j.val.complex.ptr = &s->refcnt;
-  j.val.complex.i[0] = 0;
-  j.val.complex.i[1] = 0;
+  j.val.nontrivial.ptr = &s->refcnt;
+  j.val.nontrivial.i[0] = 0;
+  j.val.nontrivial.i[1] = 0;
   return jv_copy(j);
 }
 
 jv jv_object_iter_value(jv object, int iter) {
-  return jv_copy(jvp_object_get_slot(&object.val.complex, iter)->value);
+  return jv_copy(jvp_object_get_slot(&object.val.nontrivial, iter)->value);
 }
 
 /*
@@ -953,20 +964,20 @@ jv jv_copy(jv j) {
       jv_get_kind(j) == JV_KIND_STRING || 
       jv_get_kind(j) == JV_KIND_OBJECT ||
       jv_get_kind(j) == JV_KIND_INVALID) {
-    jvp_refcnt_inc(&j.val.complex);
+    jvp_refcnt_inc(&j.val.nontrivial);
   }
   return j;
 }
 
 void jv_free(jv j) {
   if (jv_get_kind(j) == JV_KIND_ARRAY) {
-    jvp_array_free(&j.val.complex);
+    jvp_array_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_STRING) {
-    jvp_string_free(&j.val.complex);
+    jvp_string_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_OBJECT) {
-    jvp_object_free(&j.val.complex);
+    jvp_object_free(&j.val.nontrivial);
   } else if (jv_get_kind(j) == JV_KIND_INVALID) {
-    jvp_invalid_free(&j.val.complex);
+    jvp_invalid_free(&j.val.nontrivial);
   }
 }
 
@@ -975,7 +986,7 @@ int jv_get_refcnt(jv j) {
   case JV_KIND_ARRAY:
   case JV_KIND_STRING:
   case JV_KIND_OBJECT:
-    return j.val.complex.ptr->count;
+    return j.val.nontrivial.ptr->count;
   default:
     return 1;
   }
@@ -991,20 +1002,20 @@ int jv_equal(jv a, jv b) {
     r = 0;
   } else if (jv_get_kind(a) == JV_KIND_NUMBER) {
     r = jv_number_value(a) == jv_number_value(b);
-  } else if (a.val.complex.ptr == b.val.complex.ptr &&
-             a.val.complex.i[0] == b.val.complex.i[0] &&
-             a.val.complex.i[1] == b.val.complex.i[1]) {
+  } else if (a.val.nontrivial.ptr == b.val.nontrivial.ptr &&
+             a.val.nontrivial.i[0] == b.val.nontrivial.i[0] &&
+             a.val.nontrivial.i[1] == b.val.nontrivial.i[1]) {
     r = 1;
   } else {
     switch (jv_get_kind(a)) {
     case JV_KIND_ARRAY:
-      r = jvp_array_equal(&a.val.complex, &b.val.complex);
+      r = jvp_array_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     case JV_KIND_STRING:
-      r = jvp_string_equal(&a.val.complex, &b.val.complex);
+      r = jvp_string_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     case JV_KIND_OBJECT:
-      r = jvp_object_equal(&a.val.complex, &b.val.complex);
+      r = jvp_object_equal(&a.val.nontrivial, &b.val.nontrivial);
       break;
     default:
       r = 1;

jv.h

@@ -25,13 +25,13 @@ typedef struct {
 typedef struct{
   jv_refcnt* ptr;
   int i[2];
-} jv_complex;
+} jv_nontrivial;
 
 typedef struct {
   jv_kind kind;
   union {
     double number;
-    jv_complex complex;
+    jv_nontrivial nontrivial;
   } val;
 } jv;
 
@@ -82,7 +82,8 @@ jv jv_array_slice(jv, int, int);
 
 jv jv_string(const char*);
 jv jv_string_sized(const char*, int);
-int jv_string_length(jv);
+int jv_string_length_bytes(jv);
+int jv_string_length_codepoints(jv);
 uint32_t jv_string_hash(jv);
 const char* jv_string_value(jv);
 jv jv_string_concat(jv, jv);

jv_aux.c

@@ -3,6 +3,41 @@
 #include <stdlib.h>
 #include "jv_alloc.h"
 
+static int parse_slice(jv array, jv slice, int* pstart, int* pend) {
+  // Array slices
+  int len = jv_array_length(jv_copy(array));
+  jv start_jv = jv_object_get(jv_copy(slice), jv_string("start"));
+  jv end_jv = jv_object_get(slice, jv_string("end"));
+  if (jv_get_kind(start_jv) == JV_KIND_NULL) {
+    jv_free(start_jv);
+    start_jv = jv_number(0);
+  }
+  if (jv_get_kind(end_jv) == JV_KIND_NULL) {
+    jv_free(end_jv);
+    end_jv = jv_number(len);
+  }
+  if (jv_get_kind(start_jv) != JV_KIND_NUMBER ||
+      jv_get_kind(end_jv) != JV_KIND_NUMBER) {
+    jv_free(start_jv);
+    jv_free(end_jv);
+    return 0;
+  } else {
+    int start = (int)jv_number_value(start_jv);
+    int end = (int)jv_number_value(end_jv);
+    if (start < 0) start = len + start;
+    if (end < 0) end = len + end;
+
+    if (start < 0) start = 0;
+    if (start > len) start = len;
+    if (end > len) end = len;
+    if (end < start) end = start;
+    assert(0 <= start && start <= end && end <= len);
+    *pstart = start;
+    *pend = end;
+    return 1;
+  }
+}
+
 jv jv_get(jv t, jv k) {
   jv v;
   if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) {
@@ -18,8 +53,18 @@ jv jv_get(jv t, jv k) {
       jv_free(v);
       v = jv_null();
     }
+  } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_OBJECT) {
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      v = jv_array_slice(t, start, end);
+    } else {
+      v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+      jv_free(t);
+    }
   } else if (jv_get_kind(t) == JV_KIND_NULL && 
-             (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER)) {
+             (jv_get_kind(k) == JV_KIND_STRING || 
+              jv_get_kind(k) == JV_KIND_NUMBER || 
+              jv_get_kind(k) == JV_KIND_OBJECT)) {
     jv_free(t);
     jv_free(k);
     v = jv_null();
@@ -48,10 +93,49 @@ jv jv_set(jv t, jv k, jv v) {
              (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
     if (isnull) t = jv_array();
     t = jv_array_set(t, (int)jv_number_value(k), v);
+  } else if (jv_get_kind(k) == JV_KIND_OBJECT &&
+             (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
+    if (isnull) t = jv_array();
+    int start, end;
+    if (parse_slice(t, k, &start, &end)) {
+      if (jv_get_kind(v) == JV_KIND_ARRAY) {
+        int array_len = jv_array_length(jv_copy(t));
+        assert(0 <= start && start <= end && end <= array_len);
+        int slice_len = end - start;
+        int insert_len = jv_array_length(jv_copy(v));
+        if (slice_len < insert_len) {
+          // array is growing
+          int shift = insert_len - slice_len;
+          for (int i = array_len - 1; i >= end; i--) {
+            t = jv_array_set(t, i + shift, jv_array_get(jv_copy(t), i));
+          }
+        } else if (slice_len > insert_len) {
+          // array is shrinking
+          int shift = slice_len - insert_len;
+          for (int i = end; i < array_len; i++) {
+            t = jv_array_set(t, i - shift, jv_array_get(jv_copy(t), i));
+          }
+          t = jv_array_slice(t, 0, array_len - shift);
+        }
+        for (int i=0; i < insert_len; i++) {
+          t = jv_array_set(t, start + i, jv_array_get(jv_copy(v), i));
+        }
+        jv_free(v);
+      } else {
+        jv_free(t);
+        jv_free(v);
+        t = jv_invalid_with_msg(jv_string_fmt("A slice of an array can only be assigned another array"));
+      }
+    } else {
+      jv_free(t);
+      jv_free(k);
+      jv_free(v);
+      t = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+    }
   } else {
     jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s",
-                                               jv_kind_name(jv_get_kind(t)),
-                                               jv_kind_name(jv_get_kind(v))));
+                                               jv_kind_name(jv_get_kind(k)),
+                                               jv_kind_name(jv_get_kind(t))));
     jv_free(t);
     jv_free(k);
     jv_free(v);
@@ -96,23 +180,39 @@ jv jv_dels(jv t, jv keys) {
   if (jv_get_kind(t) == JV_KIND_NULL || jv_array_length(jv_copy(keys)) == 0) {
     // no change
   } else if (jv_get_kind(t) == JV_KIND_ARRAY) {
+    // extract slices, they must be handled differently
+    jv orig_keys = keys;
+    keys = jv_array();
     jv new_array = jv_array();
+    jv starts = jv_array(), ends = jv_array();
+    jv_array_foreach(orig_keys, i, key) {
+      if (jv_get_kind(key) == JV_KIND_NUMBER) {
+        keys = jv_array_append(keys, key);
+      } else if (jv_get_kind(key) == JV_KIND_OBJECT) {
+        int start, end;
+        if (parse_slice(t, key, &start, &end)) {
+          starts = jv_array_append(starts, jv_number(start));
+          ends = jv_array_append(ends, jv_number(end));
+        } else {
+          jv_free(new_array);
+          jv_free(key);
+          new_array = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
+          goto arr_out;
+        }
+      } else {
+        jv_free(new_array);
+        new_array = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array",
+                                                      jv_kind_name(jv_get_kind(key))));
+        jv_free(key);
+        goto arr_out;
+      }
+    }
+
     int kidx = 0;
     jv_array_foreach(t, i, elem) {
       int del = 0;
       while (kidx < jv_array_length(jv_copy(keys))) {
-        jv nextdel = jv_array_get(jv_copy(keys), kidx);
-        if (jv_get_kind(nextdel) != JV_KIND_NUMBER) {
-          jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array",
-                                                     jv_kind_name(jv_get_kind(nextdel))));
-          jv_free(nextdel);
-          jv_free(new_array);
-          jv_free(elem);
-          new_array = err;
-          goto arr_out; // break twice
-        }
-        int delidx = (int)jv_number_value(nextdel);
-        jv_free(nextdel);
+        int delidx = (int)jv_number_value(jv_array_get(jv_copy(keys), kidx));
         if (i == delidx) {
           del = 1;
         }
@@ -121,12 +221,21 @@ jv jv_dels(jv t, jv keys) {
         }
         kidx++;
       }
+      for (int sidx=0; !del && sidx<jv_array_length(jv_copy(starts)); sidx++) {
+        if ((int)jv_number_value(jv_array_get(jv_copy(starts), sidx)) <= i &&
+            i < (int)jv_number_value(jv_array_get(jv_copy(ends), sidx))) {
+          del = 1;
+        }
+      }
       if (!del)
         new_array = jv_array_append(new_array, elem);
       else
         jv_free(elem);
     }
   arr_out:
+    jv_free(starts);
+    jv_free(ends);
+    jv_free(orig_keys);
     jv_free(t);
     t = new_array;
   } else if (jv_get_kind(t) == JV_KIND_OBJECT) {
@@ -271,8 +380,8 @@ jv jv_delpaths(jv object, jv paths) {
 static int string_cmp(const void* pa, const void* pb){
   const jv* a = pa;
   const jv* b = pb;
-  int lena = jv_string_length(jv_copy(*a));
-  int lenb = jv_string_length(jv_copy(*b));
+  int lena = jv_string_length_bytes(jv_copy(*a));
+  int lenb = jv_string_length_bytes(jv_copy(*b));
   int minlen = lena < lenb ? lena : lenb;
   int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen);
   if (r == 0) r = lena - lenb;

jv_print.c

@@ -45,7 +45,7 @@ static void put_space(int n, FILE* fout, jv* strout) {
 static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) {
   assert(jv_get_kind(str) == JV_KIND_STRING);
   const char* i = jv_string_value(str);
-  const char* end = i + jv_string_length(jv_copy(str));
+  const char* end = i + jv_string_length_bytes(jv_copy(str));
   const char* cstart;
   int c = 0;
   char buf[32];
@@ -124,6 +124,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
     }
   }
   switch (jv_get_kind(x)) {
+  default:
   case JV_KIND_INVALID:
     assert(0 && "Invalid value");
     break;

lexer.l

@@ -49,7 +49,7 @@ struct lexer_param;
 "and" { return AND; }
 "or" { return OR; }
 "end" { return END; }
-"fold" { return FOLD; }
+"reduce" { return REDUCE; }
 "//" { return DEFINEDOR; }
 "|=" { return SETPIPE; }
 "+=" { return SETPLUS; }
@@ -93,7 +93,7 @@ struct lexer_param;
   (\\[^u(]|\\u[a-zA-Z0-9]{0,4})+ {
     /* pass escapes to the json parser */
     jv escapes = jv_string_fmt("\"%.*s\"", yyleng, yytext);
-    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length(jv_copy(escapes)));
+    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length_bytes(jv_copy(escapes)));
     jv_free(escapes);
     return QQSTRING_TEXT;
   }

main.c

@@ -1,4 +1,3 @@
-#define _POSIX_SOURCE
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
@@ -21,13 +20,13 @@ static void usage() {
   fprintf(stderr, "For a description of the command line options and\n");
   fprintf(stderr, "how to write jq filters (and why you might want to)\n");
   fprintf(stderr, "see the jq manpage, or the online documentation at\n");
-  fprintf(stderr, "http://stedolan.github.com/jq\n\n");
+  fprintf(stderr, "http://stedolan.github.io/jq\n\n");
   exit(1);
 }
 
 static void die() {
   fprintf(stderr, "Use %s --help for help with command-line options,\n", progname);
-  fprintf(stderr, "or see the jq documentation at http://stedolan.github.com/jq\n");
+  fprintf(stderr, "or see the jq documentation at http://stedolan.github.io/jq\n");
   exit(1);
 }
 
@@ -70,7 +69,7 @@ static void process(jv value, int flags) {
   jv result;
   while (jv_is_valid(result = jq_next(jq))) {
     if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
-      fwrite(jv_string_value(result), 1, jv_string_length(jv_copy(result)), stdout);
+      fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
       jv_free(result);
     } else {
       int dumpopts;

opcode_list.h

@@ -1,12 +1,11 @@
 OP(LOADK, CONSTANT, 1, 1)
 OP(DUP,   NONE,     1, 2)
 OP(DUP2,  NONE,     2, 3)
-OP(SWAP,  NONE,     2, 2)
 OP(POP,   NONE,     1, 0)
 OP(LOADV, VARIABLE, 1, 1)
+OP(LOADVN, VARIABLE, 1, 1)
 OP(STOREV, VARIABLE, 1, 0)
 OP(INDEX, NONE,     2, 1)
-OP(YIELD, NONE, 1, 0)
 OP(EACH,  NONE,     1, 1)
 OP(FORK,  BRANCH,   0, 0)
 OP(JUMP,  BRANCH,   0, 0)
@@ -14,8 +13,13 @@ OP(JUMP_F,BRANCH,   1, 0)
 OP(BACKTRACK, NONE, 0, 0)
 OP(APPEND, VARIABLE,1, 0)
 OP(INSERT, NONE,    4, 2)
+OP(RANGE, VARIABLE, 1, 1)
 
-OP(GETPATH, NONE,   2, 1)
+OP(SUBEXP_BEGIN,  NONE,     1, 2)
+OP(SUBEXP_END,    NONE,     2, 2)
+
+OP(PATH_BEGIN, NONE, 1, 2)
+OP(PATH_END,   NONE, 2, 1)
 
 OP(CALL_BUILTIN, CFUNC, -1, 1)
 

parser.y

@@ -58,7 +58,7 @@ struct lexer_param;
 %token THEN "then"
 %token ELSE "else"
 %token ELSE_IF "elif"
-%token FOLD "fold"
+%token REDUCE "reduce"
 %token END "end"
 %token AND "and"
 %token OR "or"
@@ -141,6 +141,17 @@ static block gen_index(block obj, block key) {
   return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX));
 }
 
+static block gen_slice_index(block obj, block start, block end) {
+  block key = BLOCK(gen_subexp(gen_const(jv_object())),
+                    gen_subexp(gen_const(jv_string("start"))),
+                    gen_subexp(start),
+                    gen_op_simple(INSERT),
+                    gen_subexp(gen_const(jv_string("end"))),
+                    gen_subexp(end),
+                    gen_op_simple(INSERT));
+  return BLOCK(key, obj, gen_op_simple(INDEX));
+}
+
 static block gen_binop(block a, block b, int op) {
   const char* funcname = 0;
   switch (op) {
@@ -163,6 +174,16 @@ static block gen_binop(block a, block b, int op) {
 static block gen_format(block a, jv fmt) {
   return BLOCK(a, gen_call("format", BLOCK(gen_lambda(gen_const(fmt)))));
 }
+
+static block gen_definedor_assign(block object, block val) {
+  block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
+                         gen_noop(), OP_HAS_VARIABLE);
+  return BLOCK(gen_op_simple(DUP),
+               val, tmp,
+               gen_call("_modify", BLOCK(gen_lambda(object),
+                                         gen_lambda(gen_definedor(gen_noop(), 
+                                                                  gen_op_var_bound(LOADV, tmp))))));
+}
  
 static block gen_update(block object, block val, int optype) {
   block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
@@ -205,8 +226,8 @@ Term "as" '$' IDENT '|' Exp {
   jv_free($4);
 } |
 
-"fold" Term "as" '$' IDENT '(' Exp ')' {
-  $$ = gen_fold(jv_string_value($5), $2, $7);
+"reduce" Term "as" '$' IDENT '(' Exp ';' Exp ')' {
+  $$ = gen_reduce(jv_string_value($5), $2, $7, $9);
   jv_free($5);
 } |
 
@@ -235,7 +256,7 @@ Exp "//" Exp {
 } |
 
 Exp "//=" Exp {
-  $$ = gen_update($1, gen_definedor(gen_noop(), $3), 0);
+  $$ = gen_definedor_assign($1, $3);
 } |
 
 Exp "|=" Exp {
@@ -398,6 +419,15 @@ Term '[' Exp ']' {
 Term '[' ']' {
   $$ = block_join($1, gen_op_simple(EACH)); 
 } |
+Term '[' Exp ':' Exp ']' {
+  $$ = gen_slice_index($1, $3, $5);
+} |
+Term '[' Exp ':' ']' {
+  $$ = gen_slice_index($1, $3, gen_const(jv_null()));
+} |
+Term '[' ':' Exp ']' {
+  $$ = gen_slice_index($1, gen_const(jv_null()), $4);
+} |
 LITERAL {
   $$ = gen_const($1); 
 } |

tests/all.test

@@ -166,6 +166,25 @@ null
 2
 3
 
+#
+# Slices
+#
+
+[.[3:2], .[-5:4], .[:-2], .[-2:], .[3:3][1:], .[10:]]
+[0,1,2,3,4,5,6]
+[[], [2,3], [0,1,2,3,4], [5,6], [], []]
+
+del(.[2:4],.[0],.[-2:])
+[0,1,2,3,4,5,6,7]
+[1,4,5]
+
+.[2:4] = ([], ["a","b"], ["a","b","c"])
+[0,1,2,3,4,5,6,7]
+[0,1,4,5,6,7]
+[0,1,"a","b",4,5,6,7]
+[0,1,"a","b","c",4,5,6,7]
+
+
 #
 # Variables
 #
@@ -283,8 +302,8 @@ null
 [false, false, false, false, false, false, false, false, true ]
 
 [.[] | length]
-[[], {}, [1,2], {"a":42}, "asdf"]
-[0, 0, 2, 1, 4]
+[[], {}, [1,2], {"a":42}, "asdf", "\u03bc"]
+[0, 0, 2, 1, 4, 1]
 
 map(keys)
 [{}, {"abcd":1,"abc":2,"abcde":3}, {"x":1, "z": 3, "y":2}]
@@ -360,7 +379,7 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
 # []
 # 1001
 
-fold 0 as $s (.[] | $s + .)
+reduce .[] as $x (0; . + $x)
 [1,2,4]
 7
 
@@ -475,10 +494,9 @@ def inc(x): x |= .+1; inc(.[].a)
 [{"foo":[1,2], "bar": 42}, {"foo":[1], "bar": null}, {"foo":[null,false,3], "bar": 18}, {"foo":[], "bar":42}, {"foo": [null,false,null], "bar": 41}]
 [[1,2], [1], [3], [42], [41]]
 
-# FIXME: behaviour of update operators
-# .[] //= .[0]
-# ["hello",true,false,[false],null]
-# ["hello",true,"hello",[false],"hello"]
+.[] //= .[0]
+["hello",true,false,[false],null]
+["hello",true,"hello",[false],"hello"]
 
 .[] | [.[0] and .[1], .[0] or .[1]]
 [[true,[]], [false,1], [42,null], [null,false]]
@@ -574,3 +592,11 @@ unique
 .foo[.baz]
 {"foo":{"bar":4},"baz":"bar"}
 4
+
+.[] | .error = "no, it's OK"
+[{"error":true}]
+{"error": "no, it's OK"}
+
+[{a:1}] | .[] | .a=999
+null
+{"a": 999}