array and object destructuring (fix #533)

`. as [$i, $j, $k] | ...`
`. as {a: $i, b: $j} | ...`
`. as [[[$i]], {a: $j}] | ...`
`foreach . as [$i, $j, $k] (...)`
`reduce . as {a: $i, b: $j} (...)`

compile.c

@@ -675,15 +675,25 @@ block gen_collect(block expr) {
                gen_op_bound(LOADVN, array_var));
 }
 
-block gen_reduce(const char* varname, block source, block init, block body) {
+static block bind_matcher(block matcher, block body) {
+  // cannot call block_bind(matcher, body) because that requires
+  // block_has_only_binders(matcher), which is not true here as matchers
+  // may also contain code to extract the correct elements
+  for (inst* i = matcher.first; i; i = i->next) {
+    if (i->op == STOREV && !i->bound_by)
+      block_bind_subblock(inst_block(i), body, OP_HAS_VARIABLE, 0);
+  }
+  return BLOCK(matcher, body);
+}
+
+block gen_reduce(block source, block matcher, block init, block body) {
   block res_var = gen_op_var_fresh(STOREV, "reduce");
   block loop = BLOCK(gen_op_simple(DUPN),
                      source,
-                     block_bind(gen_op_unbound(STOREV, varname),
+                     bind_matcher(matcher,
-                                BLOCK(gen_op_bound(LOADVN, res_var),
+                                  BLOCK(gen_op_bound(LOADVN, res_var),
-                                      body,
+                                        body,
-                                      gen_op_bound(STOREV, res_var)),
+                                        gen_op_bound(STOREV, res_var))),
-                                OP_HAS_VARIABLE),
                      gen_op_simple(BACKTRACK));
   return BLOCK(gen_op_simple(DUP),
                init,
@@ -693,37 +703,36 @@ block gen_reduce(const char* varname, block source, block init, block body) {
                gen_op_bound(LOADVN, res_var));
 }
 
-block gen_foreach(const char* varname, block source, block init, block update, block extract) {
+block gen_foreach(block source, block matcher, block init, block update, block extract) {
   block output = gen_op_targetlater(JUMP);
   block state_var = gen_op_var_fresh(STOREV, "foreach");
   block loop = BLOCK(gen_op_simple(DUPN),
                      // get a value from the source expression:
                      source,
-                     // bind the $varname to that value for all the code in
+                     // destructure the value into variable(s) for all the code
-                     // this block_bind() to see:
+                     // in the body to see
-                     block_bind(gen_op_unbound(STOREV, varname),
+                     bind_matcher(matcher,
-                                // load the loop state variable
+                                  // load the loop state variable
-                                BLOCK(gen_op_bound(LOADVN, state_var),
+                                  BLOCK(gen_op_bound(LOADVN, state_var),
-                                      // generate updated state
+                                        // generate updated state
-                                      update,
+                                        update,
-                                      // save the updated state for value extraction
+                                        // save the updated state for value extraction
-                                      gen_op_simple(DUP),
+                                        gen_op_simple(DUP),
-                                      // save new state
+                                        // save new state
-                                      gen_op_bound(STOREV, state_var),
+                                        gen_op_bound(STOREV, state_var),
-                                      // extract an output...
+                                        // extract an output...
-                                      extract,
+                                        extract,
-                                      // ...and output it by jumping
+                                        // ...and output it by jumping
-                                      // past the BACKTRACK that comes
+                                        // past the BACKTRACK that comes
-                                      // right after the loop body,
+                                        // right after the loop body,
-                                      // which in turn is there
+                                        // which in turn is there
-                                      // because...
+                                        // because...
-                                      //
+                                        //
-                                      // (Incidentally, extract can also
+                                        // (Incidentally, extract can also
-                                      // backtrack, e.g., if it calls
+                                        // backtrack, e.g., if it calls
-                                      // empty, in which case we don't
+                                        // empty, in which case we don't
-                                      // get here.)
+                                        // get here.)
-                                      output),
+                                        output)));
-                                OP_HAS_VARIABLE));
   block foreach = BLOCK(gen_op_simple(DUP),
                         init,
                         state_var,
@@ -809,13 +818,39 @@ block gen_or(block a, block b) {
 }
 
 block gen_var_binding(block var, const char* name, block body) {
+  return gen_destructure(var, gen_op_unbound(STOREV, name), body);
+}
+
+block gen_array_matcher(block left, block curr) {
+  int index;
+  if (block_is_noop(left))
+    index = 0;
+  else {
+    // `left` was returned by this function, so the third inst is the
+    // constant containing the previously used index
+    assert(left.first->op == DUP);
+    assert(left.first->next->op == SUBEXP_BEGIN);
+    assert(left.first->next->next->op == LOADK);
+    index = 1 + (int) jv_number_value(left.first->next->next->imm.constant);
+  }
+
+  // `left` goes at the end so that the const index is in a predictable place
+  return BLOCK(gen_op_simple(DUP), gen_subexp(gen_const(jv_number(index))),
+               gen_op_simple(INDEX), curr, left);
+}
+
+block gen_object_matcher(block name, block curr) {
+  return BLOCK(gen_op_simple(DUP), gen_subexp(name), gen_op_simple(INDEX),
+               curr);
+}
+
+block gen_destructure(block var, block matcher, block body) {
   // var bindings can be added after coding the program; leave the TOP first.
   block top = gen_noop();
   if (body.first && body.first->op == TOP)
     top = inst_block(block_take(&body));
-  return BLOCK(top, gen_op_simple(DUP), var,
+
-               block_bind(gen_op_unbound(STOREV, name),
+  return BLOCK(top, gen_op_simple(DUP), var, bind_matcher(matcher, body));
-                          body, OP_HAS_VARIABLE));
 }
 
 // Like gen_var_binding(), but bind `break`'s wildcard unbound variable

compile.h

@@ -41,14 +41,17 @@ block gen_subexp(block a);
 block gen_both(block a, block b);
 block gen_const_object(block expr);
 block gen_collect(block expr);
-block gen_reduce(const char* varname, block source, block init, block body);
+block gen_reduce(block source, block matcher, block init, block body);
-block gen_foreach(const char* varname, block source, block init, block update, block extract);
+block gen_foreach(block source, block matcher, block init, block update, block extract);
 block gen_definedor(block a, block b);
 block gen_condbranch(block iftrue, block iffalse);
 block gen_and(block a, block b);
 block gen_or(block a, block b);
 
 block gen_var_binding(block var, const char* name, block body);
+block gen_array_matcher(block left, block curr);
+block gen_object_matcher(block name, block curr);
+block gen_destructure(block var, block matcher, block body);
 
 block gen_cond(block cond, block iftrue, block iffalse);
 block gen_try_handler(block handler);

docs/content/3.manual/manual.yml

@@ -2147,6 +2147,11 @@ sections:
           with `$x` set to that value.  Thus `as` functions as something of a
           foreach loop.
 
+          Multiple variables may be declared using a single `as` expression by
+          providing a pattern that matches the structure of the input:
+
+              . as {realnames: $names, posts: [$first, $second]} | ...
+
           Variables are scoped over the rest of the expression that defines
           them, so
 
@@ -2171,6 +2176,9 @@ sections:
           - program: '. as $i|[(.*2|. as $i| $i), $i]'
             input: '5'
             output: ['[10,5]']
+          - program: '. as [$a, $b, {c: $c}] | $a + $b + $c'
+            input: '[2, 3, {c: 4, d: 5}]'
+            output: ['9']
 
       - title: 'Defining Functions'
         body: |

parser.y

@@ -107,7 +107,7 @@ struct lexer_param;
 
 %type <blk> Exp Term MkDict MkDictPair ExpD ElseBody QQString
 %type <blk> FuncDef FuncDefs String Import Imports Param Params
-%type <blk> Arg Args Module
+%type <blk> Arg Args Module Pattern ArrayPats ObjPats ObjPat
 %type <literal> Keyword
 %{
 #include "lexer.h"
@@ -308,24 +308,20 @@ FuncDef Exp %prec ';' {
   $$ = block_bind_referenced($1, $2, OP_IS_CALL_PSEUDO);
 } |
 
-Term "as" '$' IDENT '|' Exp {
+Term "as" Pattern '|' Exp {
-  $$ = gen_var_binding($1, jv_string_value($4), $6);
+  $$ = gen_destructure($1, $3, $5);
-  jv_free($4);
 } |
 
-"reduce" Term "as" '$' IDENT '(' Exp ';' Exp ')' {
+"reduce" Term "as" Pattern '(' Exp ';' Exp ')' {
-  $$ = gen_reduce(jv_string_value($5), $2, $7, $9);
+  $$ = gen_reduce($2, $4, $6, $8);
-  jv_free($5);
 } |
 
-"foreach" Term "as" '$' IDENT '(' Exp ';' Exp ';' Exp ')' {
+"foreach" Term "as" Pattern '(' Exp ';' Exp ';' Exp ')' {
-  $$ = gen_foreach(jv_string_value($5), $2, $7, $9, $11);
+  $$ = gen_foreach($2, $4, $6, $8, $10);
-  jv_free($5);
 } |
 
-"foreach" Term "as" '$' IDENT '(' Exp ';' Exp ')' {
+"foreach" Term "as" Pattern '(' Exp ';' Exp ')' {
-  $$ = gen_foreach(jv_string_value($5), $2, $7, $9, gen_noop());
+  $$ = gen_foreach($2, $4, $6, $8, gen_noop());
-  jv_free($5);
 } |
 
 "if" Exp "then" Exp ElseBody {
@@ -736,6 +732,48 @@ Exp {
   $$ = gen_lambda($1);
 }
 
+Pattern:
+'$' IDENT {
+  $$ = gen_op_unbound(STOREV, jv_string_value($2));
+  jv_free($2);
+} |
+'[' ArrayPats ']' {
+  $$ = BLOCK($2, gen_op_simple(POP));
+} |
+'{' ObjPats '}' {
+  $$ = BLOCK($2, gen_op_simple(POP));
+}
+
+ArrayPats:
+Pattern {
+  $$ = gen_array_matcher(gen_noop(), $1);
+} |
+ArrayPats ',' Pattern {
+  $$ = gen_array_matcher($1, $3);
+}
+
+ObjPats:
+ObjPat {
+  $$ = $1;
+} |
+ObjPats ',' ObjPat {
+  $$ = BLOCK($1, $3);
+}
+
+ObjPat:
+IDENT ':' Pattern {
+  $$ = gen_object_matcher(gen_const($1), $3);
+} |
+Keyword ':' Pattern {
+  $$ = gen_object_matcher(gen_const($1), $3);
+} |
+String ':' Pattern {
+  $$ = gen_object_matcher($1, $3);
+} |
+'(' Exp ')' ':' Pattern {
+  $$ = gen_object_matcher($2, $5);
+}
+
 Keyword:
 "as" {
   $$ = jv_string("as");

tests/all.test

@@ -271,6 +271,14 @@ jq: error: *label-foo/0 is not defined at <top-level>, line 1:
 null
 [0,1,2,3,4]
 
+[foreach .[] as [$i, $j] (0; . + $i - $j)]
+[[2,1], [5,3], [6,4]]
+[1,3,5]
+
+[foreach .[] as {a:$a} (0; . + $a; -.)]
+[{"a":1}, {"b":2}, {"a":3, "b":4}]
+[-1, -1, -4]
+
 [limit(3; .[])]
 [11,22,33,44,55,66,77,88,99]
 [11,22,33]
@@ -373,6 +381,37 @@ null
 null
 [1,1,1]
 
+[1, {c:3, d:4}] as [$a, {c:$b, b:$c}] | $a, $b, $c
+null
+1
+3
+null
+
+. as {as: $kw, "str": $str, ("e"+"x"+"p"): $exp} | [$kw, $str, $exp]
+{"as": 1, "str": 2, "exp": 3}
+[1, 2, 3]
+
+.[] as [$a, $b] | [$b, $a]
+[[1], [1, 2, 3]]
+[null, 1]
+[2, 1]
+
+. as $i | . as [$i] | $i
+[0]
+0
+
+. as [$i] | . as $i | $i
+[0]
+[0]
+
+%%FAIL
+. as [] | null
+jq: error: syntax error, unexpected ']', expecting '$' or '[' or '{' (Unix shell quoting issues?) at <top-level>, line 1:
+
+%%FAIL
+. as {} | null
+jq: error: syntax error, unexpected '}' (Unix shell quoting issues?) at <top-level>, line 1:
+
 # [.,(.[] | {x:.},.),.,.[]]
 
 #
@@ -610,6 +649,14 @@ reduce .[] as $x (0; . + $x)
 [1,2,4]
 7
 
+reduce .[] as [$i, {j:$j}] (0; . + $i - $j)
+[[2,{"j":1}], [5,{"j":3}], [6,{"j":4}]]
+5
+
+reduce [[1,2,10], [3,4,10]][] as [$i,$j] (0; . + $i * $j)
+null
+14
+
 . as $dot|any($dot[];not)
 [1,2,3,4,true,false,1,2,3,4,5]
 true