From 53993a994287c86989ab81b02c78e253da9e58d3 Mon Sep 17 00:00:00 2001 From: pkoppstein Date: Sat, 19 Jul 2014 01:08:38 -0400 Subject: [PATCH] Add capture; document regular expression filters Fix #493. --- builtin.c | 18 ++- docs/content/3.manual/manual.yml | 188 +++++++++++++++++++------------ docs/content/index/index.yml | 5 +- tests/all.test | 4 + 4 files changed, 138 insertions(+), 77 deletions(-) diff --git a/builtin.c b/builtin.c index dea5fbbf..238a1fa9 100644 --- a/builtin.c +++ b/builtin.c @@ -970,11 +970,23 @@ static const char* const jq_builtins[] = { "def flatten: reduce .[] as $i ([]; if $i | type == \"array\" then . + ($i | flatten) else . + [$i] end);", "def flatten(x): reduce .[] as $i ([]; if $i | type == \"array\" and x > 0 then . + ($i | flatten(x-1)) else . + [$i] end);", "def range(x): range(0;x);", + // regular expressions: "def match(re; mode): _match_impl(re; mode; false)|.[];", - "def match(val): if val | type == \"string\" then match(val; null) elif val | type == \"array\" and (val | length) > 1 then match(val[0]; val[1]) elif val | type == \"array\" and (val | length > 0) then match(val[0]; null) else error((val | type) + \" not a string or array\") end;", + "def match(val): (val|type) as $vt | if $vt == \"string\" then match(val; null)" + " elif $vt == \"array\" and (val | length) > 1 then match(val[0]; val[1])" + " elif $vt == \"array\" and (val | length) > 0 then match(val[0]; null)" + " else error( $vt + \" not a string or array\") end;", "def test(re; mode): _match_impl(re; mode; true);", - "def test(val): if val |type == \"string\" then test(val; null) elif val | type == \"array\" and (val | length) > 1 then test(val[0]; val[1]) elif val | type == \"array\" and (val | length > 0) then test(val[0]; null) else error((val | type) + \" not a string or array\") end;", -// "def test(re): _match(re; null; 1);", + "def test(val): (val|type) as $vt | if $vt == \"string\" then test(val; null)" + " elif $vt == \"array\" and (val | length) > 1 then test(val[0]; val[1])" + " elif $vt == \"array\" and (val | length) > 0 then test(val[0]; null)" + " else error( $vt + \" not a string or array\") end;", + // Ex.: "a1" | capture( "(?[a-z*])" ).x => "a" + "def capture(re; mods): match(re; mods) | reduce ( .captures | .[] | select(.name != null) | { (.name) : .string } ) as $pair ({}; . + $pair);", + "def capture(val): (val|type) as $vt | if $vt == \"string\" then capture(val; null)" + " elif $vt == \"array\" and (val | length) > 1 then capture(val[0]; val[1])" + " elif $vt == \"array\" and (val | length) > 0 then capture(val[0]; null)" + " else error( $vt + \" not a string or array\") end;", // range/3, with a `by` expression argument "def range(init; upto; by): " " def _range: " diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index bd343856..6522a6a0 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -559,6 +559,7 @@ sections: input: 'null' output: ['{"k": {"a": 0, "b": 2, "c": 3}}'] + - title: "`length`" body: | @@ -1167,78 +1168,6 @@ sections: input: '["foobar", "barfoo"]' output: ['[false, true]'] - - title: "`match(val)`, `match(regex; modifiers)`" - body: | - - The filter `match(val)` performs PCRE regex matching on its input. - `val` can be either a string or an array. If it is an array, - the first element is the regex specifier and the optional - second element is the modifier flags. - The accepted modifier flags are: - - * `g` - Global search (find all matches, not just the first) - * `i` - Case insensitive search - * `x` - Extended regex format (ignore whitespaces) - * `m` - Multi line mode ('.' will match newlines) - * `s` - Single line mode ('^' -> '\A', '$' -> '\Z') - * `p` - Both s and m modes are enabled - * `l` - Find longest possible matches - * `n` - Ignore empty matches - - The filter outputs an object for each match it finds. Matches have - the following fields: - - * `offset` - offset in UTF-8 codepoints from the beginning of the input - * `length` - length in UTF-8 codepoints of the match - * `string` - the string that it matched - * `captures` - an array of objects representing capturing groups. - - Capturing group objects have the following fields: - - * `offset` - offset in UTF-8 codepoints from the beginning of the input - * `length` - length in UTF-8 codepoints of this capturing group - * `string` - the string that was captured - * `name` - the name of the capturing group (or `null` if it was unnamed) - - Capturing groups that did not match anything return an offset of -1 - - examples: - - program: 'match("(abc)+"; "g")' - input: '"abc abc"' - output: - - '{"offset": 0, "length": 3, "string": "abc", "captures": [{"offset": 0, "length": 3, "string": "abc", "name": null}]}' - - '{"offset": 4, "length": 3, "string": "abc", "captures": [{"offset": 4, "length": 3, "string": "abc", "name": null}]}' - - program: 'match("foo")' - input: '"foo bar foo"' - output: ['{"offset": 0, "length": 3, "string": "foo", "captures": []}'] - - program: 'match(["foo", "ig"])' - input: '"foo bar FOO"' - output: - - '{"offset": 0, "length": 3, "string": "foo", "captures": []}' - - '{"offset": 8, "length": 3, "string": "FOO", "captures": []}' - - program: 'match("foo (?bar)? foo"; "ig")' - input: '"foo bar foo foo foo"' - output: - - '{"offset": 0, "length": 11, "string": "foo bar foo", "captures": [{"offset": 4, "length": 3, "string": "bar", "name": "bar123"}]}' - - '{"offset": 12, "length": 8, "string": "foo foo", "captures": [{"offset": -1, "length": 0, "string": null, "name": "bar123"}]}' - - - - title: "`test(val)`, `test(regex)`, `test(regex; modifiers)`" - body: | - - Like `match`, but does not return match objects, only `true` or `false` - for whether or not the regex matches the input. - - examples: - - program: 'test("foo")' - input: '"foo"' - output: ['true'] - - program: 'test("foo"; "i")' - input: '"Foo"' - output: ['true'] - - program: 'test("foo")' - input: '"bar"' - output: ['false'] - title: "`ltrimstr(str)`" body: | @@ -1651,6 +1580,121 @@ sections: input: '[{}, true, {"a":1}' output: ['[null, 1]'] + + - title: Regular expressions (PCRE) + body: | + + jq uses the Oniguruma regular expression library, as do php, + ruby, TextMate, Sublime Text, etc, so the description here + will focus on jq specifics. + + The jq regex filters are defined so that they can be used using + one of these patterns: + + STRING | FILTER( REGEX ) + STRING | FILTER( REGEX; FLAGS ) + STRING | FILTER( [REGEX] ) + STRING | FILTER( [REGEX, FLAGS] ) + + where: + * STRING, REGEX and FLAGS are jq strings and subject to jq string interpolation; + * REGEX, after string interpolation, should be a valid PCRE regex; + * FILTER is one of `test`, `match`, or `capture`, as described below. + + FLAGS is a string consisting of one of more of the supported flags: + + * `g` - Global search (find all matches, not just the first) + * `i` - Case insensitive search + * `m` - Multi line mode ('.' will match newlines) + * `n` - Ignore empty matches + * `p` - Both s and m modes are enabled + * `s` - Single line mode ('^' -> '\A', '$' -> '\Z') + * `l` - Find longest possible matches + * `x` - Extended regex format (ignore whitespace and comments) + + To match whitespace in an x pattern use an escape such as \s, e.g. + + * test( "a\\sb", "x" ). + + Note that certain flags may also be specified within REGEX, e.g. + + * jq -n '("test", "TEst", "teST", "TEST") | test( "(?i)te(?-i)st" )' + + evaluates to: true, true, false, false. + + entries: + - title: "`test(val)`, `test(regex; flags)`" + body: | + + Like `match`, but does not return match objects, only `true` or `false` + for whether or not the regex matches the input. + + examples: + - program: 'test("foo")' + input: '"foo"' + output: 'true' + - program: 'test("a b c # spaces are ignored"; "ix")' + input: '"xabcd" "ABC"' + output: true + true + + - title: "`match(val)`, `match(regex; flags)`" + body: | + + **match** outputs an object for each match it finds. Matches have + the following fields: + + * `offset` - offset in UTF-8 codepoints from the beginning of the input + * `length` - length in UTF-8 codepoints of the match + * `string` - the string that it matched + * `captures` - an array of objects representing capturing groups. + + Capturing group objects have the following fields: + + * `offset` - offset in UTF-8 codepoints from the beginning of the input + * `length` - length in UTF-8 codepoints of this capturing group + * `string` - the string that was captured + * `name` - the name of the capturing group (or `null` if it was unnamed) + + Capturing groups that did not match anything return an offset of -1 + + examples: + - program: 'match("(abc)+"; "g")' + input: '"abc abc"' + output: + - '{"offset": 0, "length": 3, "string": "abc", "captures": [{"offset": 0, "length": 3, "string": "abc", "name": null}]}' + - '{"offset": 4, "length": 3, "string": "abc", "captures": [{"offset": 4, "length": 3, "string": "abc", "name": null}]}' + - program: 'match("foo")' + input: '"foo bar foo"' + output: ['{"offset": 0, "length": 3, "string": "foo", "captures": []}'] + - program: 'match(["foo", "ig"])' + input: '"foo bar FOO"' + output: + - '{"offset": 0, "length": 3, "string": "foo", "captures": []}' + - '{"offset": 8, "length": 3, "string": "FOO", "captures": []}' + - program: 'match("foo (?bar)? foo"; "ig")' + input: '"foo bar foo foo foo"' + output: + - '{"offset": 0, "length": 11, "string": "foo bar foo", "captures": [{"offset": 4, "length": 3, "string": "bar", "name": "bar123"}]}' + - '{"offset": 12, "length": 8, "string": "foo foo", "captures": [{"offset": -1, "length": 0, "string": null, "name": "bar123"}]}' + + - program: '[ match("."; "g")] | length' + input: '"abc"' + output: 3 + + + - title: "`capture(val)`, `capture(regex; flags)`" + body: | + + Collects the named captures in a JSON object, with the name + of each capture as the key, and the matched string as the + corresponding value. + + example: + - program: 'capture("(?[a-z]+)-(?[0-9]+)")' + input: '"xyzzy-14"' + output: '{ "a": "xyzzy", "n": "14" }'' + - title: Advanced features body: | Variables are an absolute necessity in most programming languages, but diff --git a/docs/content/index/index.yml b/docs/content/index/index.yml index 6daf9bb0..40471927 100644 --- a/docs/content/index/index.yml +++ b/docs/content/index/index.yml @@ -2,7 +2,8 @@ headline: jq blurb: | - jq is a lightweight and flexible command-line JSON processor. + jq is a lightweight and flexible command-line JSON processor with + support for regular expressions. body1: | @@ -14,7 +15,7 @@ body2: | jq is written in portable C, and it has zero runtime dependencies. You can download a single binary, `scp` it to a far away - machine, and expect it to work. + machine of the same type, and expect it to work. body3: | diff --git a/tests/all.test b/tests/all.test index 778606a8..f375d562 100644 --- a/tests/all.test +++ b/tests/all.test @@ -771,6 +771,10 @@ def inc(x): x |= .+1; inc(.[].a) "ā" [true] +capture("(?[a-z]+)-(?[0-9]+)") +"xyzzy-14" +{"a":"xyzzy","n":"14"} + [.[]|ltrimstr("foo")] ["fo", "foo", "barfoo", "foobar", "afoo"] ["fo","","barfoo","bar","afoo"]