Streaming parser torture tests

tests/modules/streaming.jq

@@ -0,0 +1,41 @@
+
+# Filter and adjust streamed values so that only values from the .th
+# level are output.
+def trunc(inputs):
+  . as $n | inputs | . as $input | if (.[0]|length) > 1 then setpath([0];$input[0][1:]) else empty end;
+
+# Reduce streamed values back to normal
+def tovalues(i):
+  foreach i as $item (
+    # [<current value being built>,
+    #  <is current value valid?>,
+    #  <previous, complete value>,
+    #  <is previous value valid?>]
+    [null,false,null,false];
+
+    # Updator
+    # 
+    # If the new $item is a top-level value,
+    # then clear out the current value
+    if ($item[0]|length) == 0 then [null,false,.[2],.[3]]
+    # else if the new $item terminates the current value,
+    # then rotate the current value into the previous value slot.
+    elif ($item|length) == 1 and ($item[0]|length) < 2 then [null,false,.[0],.[1]]
+    else . end |
+    . as $state |
+    # If the new $item has a leaf, upate the current value
+    if ($item|length) > 1 and ($item[0]|length) > 0 then
+      [.[0]|setpath(($item|.[0]); ($item|.[1])),    # update current value
+      true,                                         # current value is now valid (if, perhaps, incomplete)
+      $state[2],                                    # previous value is unchanged
+      $state[3]]                                    # previous value is unchanged
+    else .
+    end;
+
+    # Extractor
+    #
+    # If previous value is valid, output it
+    if ($item[0]|length) == 1 and ($item|length == 1) and .[3] then .[2] else empty end,
+    # and/or if the new $item is a top-level scalar, output it
+    if ($item[0]|length) == 0 then $item[1] else empty end
+    );

tests/run

@@ -12,8 +12,20 @@ fi
 
 cat $@ | $VALGRIND $Q ./jq --run-tests
 
+set -x
+
+mods=$PWD/tests/modules
+
+clean=true
 d=
-trap '[ -n "$d" ] && rm -rf "$d"' EXIT
+clean () {
+    if ! $clean; then
+        echo "See temp files in $d!"
+    elif [ -n "$d" ]; then
+        rm -rf "$d"
+    fi
+}
+trap clean EXIT
 d=`mktemp -d -t || true`
 if [ -z "$d" ]; then
     echo "Your OS does not support mktemp(1) -d" 1>&2
@@ -123,11 +135,48 @@ $VALGRIND $Q ./jq -c '. as $d|path(..) as $p|$d|getpath($p)|scalars_or_empty|[$p
 $VALGRIND $Q ./jq --stream -c '.|select(length==2)' < "$PWD/tests/torture/input0.json" > $d/out1
 diff $d/out0 $d/out1
 
+clean=false
+if which seq > /dev/null 2>&1; then
+    # XXX We should try every prefix of input0.json, but that makes this
+    # test very, very slow when run with valgrind, and the whole point
+    # is to run it with valgrind.
+    #
+    #len=`wc -c < "$PWD/tests/torture/input0.json"`
+    if [ -z "$VALGRIND" ]; then
+        start=1
+        end=`wc -c < "$PWD/tests/torture/input0.json"`
+    else
+        start=120
+        end=151
+    fi
+    for i in `seq $start $end`; do
+        dd "if=tests/torture/input0.json" bs=$i count=1 2>/dev/null |
+            $VALGRIND ./jq -c . > $d/out0 2>$d/err || true
+        if [ -n "$VALGRIND" ]; then
+            grep '^==[0-9][0-9]*== ERROR SUMMARY: 0 errors' $d/err > /dev/null
+        else
+            tail -1 $d/err | egrep -i 'assert|abort|core' && false
+        fi
+
+        dd "if=tests/torture/input0.json" bs=$i count=1 2>/dev/null |
+            $VALGRIND ./jq -cn --stream -L "$mods" 'import streaming; tovalues(inputs)' > $d/out1 2>$d/err || true
+        if [ -n "$VALGRIND" ]; then
+            grep '^==[0-9][0-9]*== ERROR SUMMARY: 0 errors' $d/err > /dev/null
+        else
+            tail -1 $d/err | egrep -i 'assert|abort|core' && false
+        fi
+
+        diff $d/out0 $d/out1
+    done
+else
+    echo "Not doing torture tests"
+fi
+
 ## Fuzz parser
 
+clean=false
 if dd if=/dev/urandom bs=16 count=1024 > $d/rand 2>/dev/null; then
     # Have a /dev/urandom, good
-    set -x
     $VALGRIND $Q ./jq --seq . $d/rand >/dev/null 2>&1
     $VALGRIND $Q ./jq --seq --stream . $d/rand >/dev/null 2>&1
     dd if=/dev/urandom bs=16 count=1024 > $d/rand 2>/dev/null
@@ -137,11 +186,10 @@ if dd if=/dev/urandom bs=16 count=1024 > $d/rand 2>/dev/null; then
     $VALGRIND $Q ./jq --seq . $d/rand >/dev/null 2>&1
     $VALGRIND $Q ./jq --seq --stream . $d/rand >/dev/null 2>&1
 fi
+clean=true
 
 ## Test library/module system
 
-mods=$PWD/tests/modules
-
 if [ "`HOME="$mods" $VALGRIND $Q ./jq -nr fg`" != foobar ]; then
     echo "Bug #479 appears to be back" 1>&2
     exit 1
@@ -202,3 +250,5 @@ if [ -n "$VALGRIND" ] && ! grep 'ERROR SUMMARY: 0 errors from 0 contexts' $d/out
     cat $d/out
     exit 1
 fi
+
+exit 0