diff --git a/AUTHORS b/AUTHORS index 98fdf672..2ef6ae09 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,3 +4,4 @@ Stephen Dolan Contributions by Anthony Shortland - rpmbuild target Lee Thompson - autoconf stuff +Nicolas Williams - library-fication, autoconf stuff, misc diff --git a/builtin.c b/builtin.c index 721cd3fc..087741fa 100644 --- a/builtin.c +++ b/builtin.c @@ -77,6 +77,64 @@ static jv f_negate(jv input) { return ret; } +static jv f_startswith(jv a, jv b) { + int alen = jv_string_length_bytes(jv_copy(a)); + int blen = jv_string_length_bytes(jv_copy(b)); + jv ret; + + if (blen <= alen && memcmp(jv_string_value(a), jv_string_value(b), blen) == 0) + ret = jv_true(); + else + ret = jv_false(); + jv_free(a); + jv_free(b); + return ret; +} + +static jv f_endswith(jv a, jv b) { + const char *astr = jv_string_value(a); + const char *bstr = jv_string_value(b); + size_t alen = jv_string_length_bytes(jv_copy(a)); + size_t blen = jv_string_length_bytes(jv_copy(b)); + jv ret;; + + if (alen < blen || + memcmp(astr + (alen - blen), bstr, blen) != 0) + ret = jv_false(); + else + ret = jv_true(); + jv_free(a); + jv_free(b); + return ret; +} + +static jv f_ltrimstr(jv input, jv left) { + if (jv_get_kind(f_startswith(jv_copy(input), jv_copy(left))) != JV_KIND_TRUE) { + jv_free(left); + return input; + } + /* + * FIXME It'd be better to share the suffix with the original input -- + * that we could do, we just can't share prefixes. + */ + int prefixlen = jv_string_length_bytes(left); + jv res = jv_string_sized(jv_string_value(input) + prefixlen, + jv_string_length_bytes(jv_copy(input)) - prefixlen); + jv_free(input); + return res; +} + +static jv f_rtrimstr(jv input, jv right) { + if (jv_get_kind(f_endswith(jv_copy(input), jv_copy(right))) == JV_KIND_TRUE) { + jv res = jv_string_sized(jv_string_value(input), + jv_string_length_bytes(jv_copy(input)) - jv_string_length_bytes(right)); + jv_free(input); + return res; + } + jv_free(right); + return input; +} + static jv f_minus(jv input, jv a, jv b) { jv_free(input); if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { @@ -107,6 +165,19 @@ static jv f_multiply(jv input, jv a, jv b) { jv_free(input); if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { return jv_number(jv_number_value(a) * jv_number_value(b)); + } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_NUMBER) { + int n; + size_t alen = jv_string_length_bytes(jv_copy(a)); + jv res = a; + + for (n = jv_number_value(b) - 1; n > 0; n--) + res = jv_string_append_buf(res, jv_string_value(a), alen); + + if (n < 0) { + jv_free(a); + return jv_null(); + } + return res; } else { return type_error2(a, b, "cannot be multiplied"); } @@ -116,6 +187,8 @@ static jv f_divide(jv input, jv a, jv b) { jv_free(input); if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { return jv_number(jv_number_value(a) / jv_number_value(b)); + } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) { + return jv_string_split(a, b); } else { return type_error2(a, b, "cannot be divided"); } @@ -180,6 +253,19 @@ static jv f_contains(jv a, jv b) { } } +static jv f_dump(jv input) { + return jv_dump_string(input, 0); +} + +static jv f_json_parse(jv input) { + if (jv_get_kind(input) != JV_KIND_STRING) + return type_error(input, "only strings can be parsed"); + jv res = jv_parse_sized(jv_string_value(input), + jv_string_length_bytes(jv_copy(input))); + jv_free(input); + return res; +} + static jv f_tonumber(jv input) { if (jv_get_kind(input) == JV_KIND_NUMBER) { return input; @@ -201,6 +287,8 @@ static jv f_length(jv input) { return jv_number(jv_object_length(input)); } else if (jv_get_kind(input) == JV_KIND_STRING) { return jv_number(jv_string_length_codepoints(input)); + } else if (jv_get_kind(input) == JV_KIND_NUMBER) { + return jv_number(fabs(jv_number_value(input))); } else if (jv_get_kind(input) == JV_KIND_NULL) { jv_free(input); return jv_number(0); @@ -490,9 +578,18 @@ static const struct cfunction function_list[] = { {(cfunction_ptr)f_multiply, "_multiply", 3}, {(cfunction_ptr)f_divide, "_divide", 3}, {(cfunction_ptr)f_mod, "_mod", 3}, + {(cfunction_ptr)f_dump, "tojson", 1}, + {(cfunction_ptr)f_json_parse, "fromjson", 1}, {(cfunction_ptr)f_tonumber, "tonumber", 1}, {(cfunction_ptr)f_tostring, "tostring", 1}, {(cfunction_ptr)f_keys, "keys", 1}, + {(cfunction_ptr)f_startswith, "startswith", 2}, + {(cfunction_ptr)f_endswith, "endswith", 2}, + {(cfunction_ptr)f_ltrimstr, "ltrimstr", 2}, + {(cfunction_ptr)f_rtrimstr, "rtrimstr", 2}, + {(cfunction_ptr)jv_string_split, "split", 2}, + {(cfunction_ptr)jv_string_explode, "explode", 1}, + {(cfunction_ptr)jv_string_implode, "implode", 1}, {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking {(cfunction_ptr)jv_getpath, "getpath", 2}, {(cfunction_ptr)jv_delpaths, "delpaths", 2}, @@ -579,12 +676,14 @@ static const char* const jq_builtins[] = { "def from_entries: map({(.key): .value}) | add;", "def with_entries(f): to_entries | map(f) | from_entries;", "def reverse: [.[length - 1 - range(0;length)]];", + "def index(i): .[i][0];", + "def rindex(i): .[i][-1:][0];", }; -int builtins_bind_one(block* bb, const char* code) { +static int builtins_bind_one(jq_state *jq, block* bb, const char* code) { struct locfile src; - locfile_init(&src, code, strlen(code)); + locfile_init(&src, jq, code, strlen(code)); block funcs; int nerrors = jq_parse_library(&src, &funcs); if (nerrors == 0) { @@ -594,14 +693,14 @@ int builtins_bind_one(block* bb, const char* code) { return nerrors; } -int slurp_lib(block* bb) { +static int slurp_lib(jq_state *jq, block* bb) { int nerrors = 0; char* home = getenv("HOME"); if (home) { // silently ignore no $HOME jv filename = jv_string_append_str(jv_string(home), "/.jq"); jv data = jv_load_file(jv_string_value(filename), 1); if (jv_is_valid(data)) { - nerrors = builtins_bind_one(bb, jv_string_value(data) ); + nerrors = builtins_bind_one(jq, bb, jv_string_value(data) ); } jv_free(filename); jv_free(data); @@ -609,14 +708,14 @@ int slurp_lib(block* bb) { return nerrors; } -int builtins_bind(block* bb) { - int nerrors = slurp_lib(bb); +int builtins_bind(jq_state *jq, block* bb) { + int nerrors = slurp_lib(jq, bb); if (nerrors) { block_free(*bb); return nerrors; } for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) { - nerrors = builtins_bind_one(bb, jq_builtins[i]); + nerrors = builtins_bind_one(jq, bb, jq_builtins[i]); assert(!nerrors); } *bb = bind_bytecoded_builtins(*bb); diff --git a/builtin.h b/builtin.h index 1aab133f..bca9bae1 100644 --- a/builtin.h +++ b/builtin.h @@ -1,9 +1,10 @@ #ifndef BUILTIN_H #define BUILTIN_H +#include "jq.h" #include "bytecode.h" #include "compile.h" -int builtins_bind(block*); +int builtins_bind(jq_state *, block*); #endif diff --git a/compile.c b/compile.c index 248d363e..f634284f 100644 --- a/compile.c +++ b/compile.c @@ -1,4 +1,6 @@ +#ifndef _GNU_SOURCE #define _GNU_SOURCE // for strdup +#endif #include #include #include diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 467617ac..d114d31e 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -80,6 +80,12 @@ sections: Instead of running the filter for each JSON object in the input, read the entire input stream into a large array and run the filter just once. + + * `--online-input`/`-I`: + + When the top-level input value is an array produce its elements + instead of the array. This allows on-line processing of + potentially very large top-level arrays' elements. * `--raw-input`/`-R`: @@ -136,6 +142,13 @@ sections: Read filter from the file rather than from a command line, like awk's -f option. You can also use '#' to make comments. + * `-e` / `--exit-status`: + + Sets the exit status of jq to 10 if the last output values was + `false`, 11 if the last output value was `null`, or 0 if the + last output value was valid. Normally jq exits with 0, or 1 if + there was any usage problem or other error. + * `--arg name value`: This option passes a value to the jq program as a predefined @@ -199,11 +212,12 @@ sections: returns the third element of the array. The `.[10:15]` syntax can be used to return a subarray of an - array. The array returned by `.[10:15]` will be of length 5, - containing the elements from index 10 (inclusive) to index - 15 (exclusive). Either index may be negative (in which case - it counts backwards from the end of the array), or omitted - (in which case it refers to the start or end of the array). + array or substring of a string. The array returned by + `.[10:15]` will be of length 5, containing the elements from + index 10 (inclusive) to index 15 (exclusive). Either index may + be negative (in which case it counts backwards from the end of + the array), or omitted (in which case it refers to the start + or end of the array). examples: - program: '.[0]' @@ -217,6 +231,10 @@ sections: - program: '.[2:4]' input: '["a","b","c","d","e"]' output: ['["c", "d"]'] + + - program: '.[2:4]' + input: '"abcdefghi"' + output: ['"cd"'] - program: '.[:3]' input: '["a","b","c","d","e"]' @@ -458,10 +476,19 @@ sections: These operators only work on numbers, and do the expected. + Multiplying a string by a number produces the concatenation of + that string that many times. + + Dividing a string by another splits the first using the second + as separators. + examples: - program: '10 / . * 3' input: 5 output: [6] + - program: '. / ", "' + input: '"a, b,c,d, e"' + output: ['["a","b,c,d","e"]'] - title: `length` body: | @@ -806,6 +833,79 @@ sections: input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}' output: ['false'] + - title: `startswith` + body: | + + Outputs true if . starts with the given string argument. + + examples: + - program: '[.[]|startswith("foo")]' + input: '["fo", "foo", "barfoo", "foobar", "barfoob"]' + output: ['[false, true, false, true, false]'] + + - title: `endswith` + body: | + + Outputs true if . ends with the given string argument. + + examples: + - program: '[.[]|endswith("foo")]' + input: '["foobar", "barfoo"]' + output: ['[false, true, true, false, false]'] + + - title: `ltrimstr` + body: | + + Outputs its input with the given prefix string removed, if it + starts with it. + + examples: + - program: '[.[]|ltrimstr("foo")]' + input: ["fo", "foo", "barfoo", "foobar", "afoo"] + output: ['["fo","","barfoo","bar","afoo"]'] + + - title: `rtrimstr` + body: | + + Outputs its input with the given suffix string removed, if it + starts with it. + + examples: + - program: '[.[]|rtrimstr("foo")]' + input: ["fo", "foo", "barfoo", "foobar", "foob"] + output: ['["fo","","bar","foobar","foob"]'] + + - title: `explode` + body: | + + Converts an input string into an array of the string's + codepoint numbers. + + examples: + - program: 'explode' + input: '"foobar"' + output: ['[102,111,111,98,97,114]'] + + - title: `implode` + body: | + + The inverse of explode. + + examples: + - program: 'implode' + input: '[65, 66, 67]' + output: ['"ABC"'] + + - title: `split` + body: | + + Splits an input string on the separator argument. + + examples: + - program: 'split(", ")' + input: '"a, b,c,d, e"' + output: ['["a","b,c,d","e"]'] + - title: `recurse` body: | @@ -850,6 +950,24 @@ sections: input: '42' output: ['"The input was 42, which is one less than 43"'] + - title: "Convert to/from JSON" + + The `tojson` and `fromjson` builtins dump values as JSON texts + or parse JSON texts into values, respectively. The tojson + builtin differs from tostring in that tostring returns strings + unmodified, while tojson encodes strings as JSON strings. + + examples: + - program: '[.[]|tostring]' + input: '[1, "foo", ["foo"]]' + output: '["1","foo","[\"foo\"]"]' + - program: '[.[]|tojson]' + input: '[1, "foo", ["foo"]]' + output: '["1","\"foo\"","[\"foo\"]"]' + - program: '[.[]|tojson|fromjson]' + input: '[1, "foo", ["foo"]]' + output: '[1,"foo",["foo"]]' + - title: "Format strings and escaping" body: | diff --git a/execute.c b/execute.c index 4b05d793..4ed20d8d 100644 --- a/execute.c +++ b/execute.c @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -19,6 +21,9 @@ struct jq_state { void *nomem_handler_data; struct bytecode* bc; + jq_err_cb err_cb; + void *err_cb_data; + struct stack stk; stack_ptr curr_frame; stack_ptr stk_top; @@ -238,15 +243,22 @@ static void jq_reset(jq_state *jq) { jq->subexp_nest = 0; } - - - -void print_error(jv value) { +static void print_error(jq_state *jq, jv value) { assert(!jv_is_valid(value)); jv msg = jv_invalid_get_msg(value); - if (jv_get_kind(msg) == JV_KIND_STRING) { - fprintf(stderr, "jq: error: %s\n", jv_string_value(msg)); - } + jv msg2; + if (jv_get_kind(msg) == JV_KIND_STRING) + msg2 = jv_string_fmt("jq: error: %s", jv_string_value(msg)); + else + msg2 = jv_string_fmt("jq: error: "); + jv_free(msg); + + if (jq->err_cb) + jq->err_cb(jq->err_cb_data, msg2); + else if (jv_get_kind(msg2) == JV_KIND_STRING) + fprintf(stderr, "%s\n", jv_string_value(msg2)); + else + fprintf(stderr, "jq: error: %s\n", strerror(ENOMEM)); jv_free(msg); } #define ON_BACKTRACK(op) ((op)+NUM_OPCODES) @@ -368,8 +380,8 @@ jv jq_next(jq_state *jq) { stack_push(jq, jv_object_set(objv, k, v)); stack_push(jq, stktop); } else { - print_error(jv_invalid_with_msg(jv_string_fmt("Cannot use %s as object key", - jv_kind_name(jv_get_kind(k))))); + print_error(jq, jv_invalid_with_msg(jv_string_fmt("Cannot use %s as object key", + jv_kind_name(jv_get_kind(k))))); jv_free(stktop); jv_free(v); jv_free(k); @@ -387,7 +399,7 @@ jv jq_next(jq_state *jq) { jv max = stack_pop(jq); if (jv_get_kind(*var) != JV_KIND_NUMBER || jv_get_kind(max) != JV_KIND_NUMBER) { - print_error(jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric"))); + print_error(jq, jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric"))); jv_free(max); goto do_backtrack; } else if (jv_number_value(jv_copy(*var)) >= jv_number_value(jv_copy(max))) { @@ -499,7 +511,7 @@ jv jq_next(jq_state *jq) { if (jv_is_valid(v)) { stack_push(jq, v); } else { - print_error(v); + print_error(jq, v); goto do_backtrack; } break; @@ -552,8 +564,8 @@ jv jq_next(jq_state *jq) { } } else { assert(opcode == EACH); - print_error(jv_invalid_with_msg(jv_string_fmt("Cannot iterate over %s", - jv_kind_name(jv_get_kind(container))))); + print_error(jq, jv_invalid_with_msg(jv_string_fmt("Cannot iterate over %s", + jv_kind_name(jv_get_kind(container))))); keep_going = 0; } @@ -624,7 +636,7 @@ jv jq_next(jq_state *jq) { if (jv_is_valid(top)) { stack_push(jq, top); } else { - print_error(top); + print_error(jq, top); goto do_backtrack; } break; @@ -682,10 +694,23 @@ jq_state *jq_init(void) { jq->fork_top = 0; jq->curr_frame = 0; + jq->err_cb = NULL; + jq->err_cb_data = NULL; + jq->path = jv_null(); return jq; } +void jq_set_error_cb(jq_state *jq, jq_err_cb cb, void *data) { + jq->err_cb = cb; + jq->err_cb_data = data; +} + +void jq_get_error_cb(jq_state *jq, jq_err_cb *cb, void **data) { + *cb = jq->err_cb; + *data = jq->err_cb_data; +} + void jq_set_nomem_handler(jq_state *jq, void (*nomem_handler)(void *), void *data) { jv_nomem_handler(nomem_handler, data); jq->nomem_handler = nomem_handler; @@ -729,7 +754,7 @@ int jq_compile_args(jq_state *jq, const char* str, jv args) { jv_nomem_handler(jq->nomem_handler, jq->nomem_handler_data); assert(jv_get_kind(args) == JV_KIND_ARRAY); struct locfile locations; - locfile_init(&locations, str, strlen(str)); + locfile_init(&locations, jq, str, strlen(str)); block program; jq_reset(jq); if (jq->bc) { @@ -746,13 +771,21 @@ int jq_compile_args(jq_state *jq, const char* str, jv args) { jv_free(name); } jv_free(args); - nerrors = builtins_bind(&program); + nerrors = builtins_bind(jq, &program); if (nerrors == 0) { nerrors = block_compile(program, &locations, &jq->bc); } } if (nerrors) { - fprintf(stderr, "%d compile %s\n", nerrors, nerrors > 1 ? "errors" : "error"); + jv s = jv_string_fmt("%d compile %s", nerrors, + nerrors > 1 ? "errors" : "error"); + if (jq->err_cb != NULL) + jq->err_cb(jq->err_cb_data, s); + else if (!jv_is_valid(s)) + fprintf(stderr, "Error formatting jq compilation errors: %s\n", strerror(errno)); + else + fprintf(stderr, "%s\n", jv_string_value(s)); + jv_free(s); } locfile_free(&locations); return jq->bc != NULL; diff --git a/jq.h b/jq.h index e35845c6..66b79387 100644 --- a/jq.h +++ b/jq.h @@ -1,12 +1,17 @@ #ifndef _JQ_H_ #define _JQ_H_ +#include #include enum {JQ_DEBUG_TRACE = 1}; typedef struct jq_state jq_state; +typedef void (*jq_err_cb)(void *, jv); + jq_state *jq_init(void); +void jq_set_error_cb(jq_state *, jq_err_cb, void *); +void jq_get_error_cb(jq_state *, jq_err_cb *, void **); void jq_set_nomem_handler(jq_state *, void (*)(void *), void *); int jq_compile(jq_state *, const char* str); int jq_compile_args(jq_state *, const char* str, jv args); diff --git a/jv.c b/jv.c index 71824456..0c0bbb76 100644 --- a/jv.c +++ b/jv.c @@ -417,6 +417,14 @@ static jv_nontrivial jvp_string_new(const char* data, uint32_t length) { return r; } +static jv_nontrivial jvp_string_empty_new(uint32_t length) { + jvp_string* s = jvp_string_alloc(length); + s->length_hashed = 0; + memset(s->data, 0, length); + jv_nontrivial r = {&s->refcnt, {0,0}}; + return r; +} + static void jvp_string_free(jv_nontrivial* s) { if (jvp_refcnt_dec(s)) { jvp_string* str = jvp_string_ptr(s); @@ -560,6 +568,13 @@ jv jv_string_sized(const char* str, int len) { return j; } +jv jv_string_empty(int len) { + jv j; + j.kind = JV_KIND_STRING; + j.val.nontrivial = jvp_string_empty_new(len); + return j; +} + jv jv_string(const char* str) { return jv_string_sized(str, strlen(str)); } @@ -581,6 +596,105 @@ int jv_string_length_codepoints(jv j) { return len; } +#ifndef HAVE_MEMMEM +static const void *memmem(const void *haystack, size_t haystacklen, + const void *needle, size_t needlelen) +{ + const char *h = haystack; + const char *n = needle; + size_t hi, hi2, ni; + + if (haystacklen < needlelen || haystacklen == 0) + return NULL; + for (hi = 0; hi < (haystacklen - needlelen + 1); hi++) { + for (ni = 0, hi2 = hi; ni < needlelen; ni++, hi2++) { + if (h[hi2] != n[ni]) + goto not_this; + } + + return &h[hi]; + +not_this: + continue; + } + return NULL; +} +#endif /* HAVE_MEMMEM */ + +jv jv_string_indexes(jv j, jv k) { + assert(jv_get_kind(j) == JV_KIND_STRING); + assert(jv_get_kind(k) == JV_KIND_STRING); + const char *jstr = jv_string_value(j); + const char *idxstr = jv_string_value(k); + const char *p; + int jlen = jv_string_length_bytes(jv_copy(j)); + int idxlen = jv_string_length_bytes(jv_copy(k)); + jv a = jv_array(); + + p = jstr; + while ((p = memmem(p, (jstr + jlen) - p, idxstr, idxlen)) != NULL) { + a = jv_array_append(a, jv_number(p - jstr)); + p += idxlen; + } + jv_free(j); + jv_free(k); + return a; +} + +jv jv_string_split(jv j, jv sep) { + assert(jv_get_kind(j) == JV_KIND_STRING); + assert(jv_get_kind(sep) == JV_KIND_STRING); + const char *jstr = jv_string_value(j); + const char *sepstr = jv_string_value(sep); + const char *p, *s; + int jlen = jv_string_length_bytes(jv_copy(j)); + int seplen = jv_string_length_bytes(jv_copy(sep)); + jv a = jv_array(); + + assert(jv_get_refcnt(a) == 1); + + for (p = jstr; p < jstr + jlen; p = s + seplen) { + s = memmem(p, (jstr + jlen) - p, sepstr, seplen); + if (s == NULL) + s = jstr + jlen; + a = jv_array_append(a, jv_string_sized(p, s - p)); + } + jv_free(j); + jv_free(sep); + return a; +} + +jv jv_string_explode(jv j) { + assert(jv_get_kind(j) == JV_KIND_STRING); + const char* i = jv_string_value(j); + int len = jv_string_length_bytes(jv_copy(j)); + const char* end = i + len; + jv a = jv_array_sized(len); + int c; + while ((i = jvp_utf8_next(i, end, &c))) + a = jv_array_append(a, jv_number(c)); + jv_free(j); + return a; +} + +jv jv_string_implode(jv j) { + assert(jv_get_kind(j) == JV_KIND_ARRAY); + int len = jv_array_length(jv_copy(j)); + jv s = jv_string_empty(len); + int i; + + assert(len >= 0); + + for (i = 0; i < len; i++) { + jv n = jv_array_get(jv_copy(j), i); + assert(jv_get_kind(n) == JV_KIND_NUMBER); + s = jv_string_append_codepoint(s, jv_number_value(n)); + } + + jv_free(j); + return s; +} + unsigned long jv_string_hash(jv j) { assert(jv_get_kind(j) == JV_KIND_STRING); uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.nontrivial)); @@ -593,6 +707,63 @@ const char* jv_string_value(jv j) { return jvp_string_ptr(&j.val.nontrivial)->data; } +jv jv_string_slice(jv j, int start, int end) { + assert(jv_get_kind(j) == JV_KIND_STRING); + const char *s = jv_string_value(j); + int len = jv_string_length_bytes(jv_copy(j)); + int i; + const char *p, *e; + int c; + jv res; + + if (start < 0) start = len + start; + if (end < 0) end = len + end; + + if (start < 0) start = 0; + if (start > len) start = len; + if (end > len) end = len; + if (end < start) end = start; + if (start < 0 || start > end || end > len) + return jv_invalid_with_msg(jv_string("Invalid string slice indices")); + assert(0 <= start && start <= end && end <= len); + + /* Look for byte offset corresponding to start codepoints */ + for (p = s, i = 0; i < start; i++) { + p = jvp_utf8_next(p, s + len, &c); + if (p == NULL) { + jv_free(j); + return jv_string_empty(16); + } + if (c == -1) { + jv_free(j); + return jv_invalid_with_msg(jv_string("Invalid UTF-8 string")); + } + } + /* Look for byte offset corresponding to end codepoints */ + for (e = p; e != NULL && i < end; i++) { + e = jvp_utf8_next(e, s + len, &c); + if (e == NULL) { + e = s + len; + break; + } + if (c == -1) { + jv_free(j); + return jv_invalid_with_msg(jv_string("Invalid UTF-8 string")); + } + } + + /* + * NOTE: Ideally we should do here what jvp_array_slice() does instead + * of allocating a new string as we do! However, we assume NUL- + * terminated strings all over, and in the jv API, so for now we waste + * memory like a drunken navy programmer. There's probably nothing we + * can do about it. + */ + res = jv_string_sized(p, e - p); + jv_free(j); + return res; +} + jv jv_string_concat(jv a, jv b) { jvp_string* sb = jvp_string_ptr(&b.val.nontrivial); jvp_string_append(&a.val.nontrivial, sb->data, jvp_string_length(sb)); @@ -612,18 +783,25 @@ jv jv_string_append_buf(jv a, const char* buf, int len) { return a; } +jv jv_string_append_codepoint(jv a, uint32_t c) { + char buf[5]; + int len = jvp_utf8_encode(c, buf); + jvp_string_append(&a.val.nontrivial, buf, len); + return a; +} + jv jv_string_append_str(jv a, const char* str) { return jv_string_append_buf(a, str, strlen(str)); } -jv jv_string_fmt(const char* fmt, ...) { +jv jv_string_vfmt(const char* fmt, va_list ap) { int size = 1024; while (1) { char* buf = jv_mem_alloc(size); - va_list args; - va_start(args, fmt); - int n = vsnprintf(buf, size, fmt, args); - va_end(args); + va_list ap2; + va_copy(ap2, ap); + int n = vsnprintf(buf, size, fmt, ap2); + va_end(ap2); if (n < size) { jv ret = jv_string_sized(buf, n); jv_mem_free(buf); @@ -635,6 +813,14 @@ jv jv_string_fmt(const char* fmt, ...) { } } +jv jv_string_fmt(const char* fmt, ...) { + va_list args; + va_start(args, fmt); + jv res = jv_string_vfmt(fmt, args); + va_end(args); + return res; +} + /* * Objects (internal helpers) */ diff --git a/jv.h b/jv.h index 0f50359a..efc6fecb 100644 --- a/jv.h +++ b/jv.h @@ -1,6 +1,9 @@ #ifndef JV_H #define JV_H +#include +#include + typedef enum { JV_KIND_INVALID, JV_KIND_NULL, @@ -75,14 +78,22 @@ jv jv_array_slice(jv, int, int); jv jv_string(const char*); jv jv_string_sized(const char*, int); +jv jv_string_empty(int len); int jv_string_length_bytes(jv); int jv_string_length_codepoints(jv); unsigned long jv_string_hash(jv); const char* jv_string_value(jv); +jv jv_string_indexes(jv j, jv k); +jv jv_string_slice(jv j, int start, int end); jv jv_string_concat(jv, jv); +jv jv_string_vfmt(const char*, va_list); jv jv_string_fmt(const char*, ...); +jv jv_string_append_codepoint(jv a, uint32_t c); jv jv_string_append_buf(jv a, const char* buf, int len); jv jv_string_append_str(jv a, const char* str); +jv jv_string_split(jv j, jv sep); +jv jv_string_explode(jv j); +jv jv_string_implode(jv j); jv jv_object(); jv jv_object_get(jv object, jv key); @@ -122,6 +133,9 @@ void jv_nomem_handler(jv_nomem_handler_f, void *); jv jv_load_file(const char *, int); +typedef enum { + JV_PARSE_EXPLODE_TOPLEVEL_ARRAY = 1 +} jv_parser_flags; struct jv_parser; struct jv_parser* jv_parser_new(); void jv_parser_set_buf(struct jv_parser*, const char*, int, int); diff --git a/jv_aux.c b/jv_aux.c index 89f36e69..76712aee 100644 --- a/jv_aux.c +++ b/jv_aux.c @@ -3,15 +3,19 @@ #include #include "jv_alloc.h" -static int parse_slice(jv array, jv slice, int* pstart, int* pend) { +static int parse_slice(jv j, jv slice, int* pstart, int* pend) { // Array slices - int len = jv_array_length(jv_copy(array)); jv start_jv = jv_object_get(jv_copy(slice), jv_string("start")); jv end_jv = jv_object_get(slice, jv_string("end")); if (jv_get_kind(start_jv) == JV_KIND_NULL) { jv_free(start_jv); start_jv = jv_number(0); } + int len; + if (jv_get_kind(j) == JV_KIND_ARRAY) + len = jv_array_length(jv_copy(j)); + else + len = jv_string_length_codepoints(jv_copy(j)); if (jv_get_kind(end_jv) == JV_KIND_NULL) { jv_free(end_jv); end_jv = jv_number(len); @@ -61,6 +65,16 @@ jv jv_get(jv t, jv k) { v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers")); jv_free(t); } + } else if (jv_get_kind(t) == JV_KIND_STRING && jv_get_kind(k) == JV_KIND_OBJECT) { + int start, end; + if (parse_slice(t, k, &start, &end)) { + v = jv_string_slice(t, start, end); + } else { + v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an string slice must be numbers")); + jv_free(t); + } + } else if (jv_get_kind(t) == JV_KIND_STRING && jv_get_kind(k) == JV_KIND_STRING) { + v = jv_string_indexes(t, k); } else if (jv_get_kind(t) == JV_KIND_NULL && (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER || diff --git a/jv_file.c b/jv_file.c index 54ed36be..a6331607 100644 --- a/jv_file.c +++ b/jv_file.c @@ -18,7 +18,7 @@ jv jv_load_file(const char* filename, int raw) { data = jv_string(""); } else { data = jv_array(); - parser = jv_parser_new(); + parser = jv_parser_new(0); } while (!feof(file) && !ferror(file)) { char buf[4096]; diff --git a/jv_parse.c b/jv_parse.c index cc1e7b96..4667f1f9 100644 --- a/jv_parse.c +++ b/jv_parse.c @@ -28,6 +28,8 @@ struct jv_parser { int stackpos; int stacklen; jv next; + + jv_parser_flags flags; char* tokenbuf; int tokenpos; @@ -45,7 +47,8 @@ struct jv_parser { }; -static void parser_init(struct jv_parser* p) { +static void parser_init(struct jv_parser* p, jv_parser_flags flags) { + p->flags = flags; p->stack = 0; p->stacklen = p->stackpos = 0; p->next = jv_invalid(); @@ -110,6 +113,9 @@ static pfunc token(struct jv_parser* p, char ch) { break; case ',': + if (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY) && + jv_get_kind(p->stack[0]) == JV_KIND_ARRAY) + return 0; if (!jv_is_valid(p->next)) return "Expected value before ','"; if (p->stackpos == 0) @@ -133,16 +139,22 @@ static pfunc token(struct jv_parser* p, char ch) { if (p->stackpos == 0 || jv_get_kind(p->stack[p->stackpos-1]) != JV_KIND_ARRAY) return "Unmatched ']'"; if (jv_is_valid(p->next)) { - p->stack[p->stackpos-1] = jv_array_append(p->stack[p->stackpos-1], p->next); - p->next = jv_invalid(); + if (p->stackpos != 1 || !(p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY)) { + p->stack[p->stackpos-1] = jv_array_append(p->stack[p->stackpos-1], p->next); + p->next = jv_invalid(); + } } else { if (jv_array_length(jv_copy(p->stack[p->stackpos-1])) != 0) { // this case hits on input like [1,2,3,] return "Expected another array element"; } } - jv_free(p->next); - p->next = p->stack[--p->stackpos]; + if (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY)) { + jv_free(p->stack[--p->stackpos]); + } else { + jv_free(p->next); + p->next = p->stack[--p->stackpos]; + } break; case '}': @@ -315,7 +327,9 @@ static chclass classify(char c) { static const presult OK = "output produced"; static int check_done(struct jv_parser* p, jv* out) { - if (p->stackpos == 0 && jv_is_valid(p->next)) { + if ((p->stackpos == 0 && jv_is_valid(p->next)) || + (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY) && + jv_get_kind(p->stack[0]) == JV_KIND_ARRAY && jv_is_valid(p->next))) { *out = p->next; p->next = jv_invalid(); return 1; @@ -370,9 +384,9 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) { return answer; } -struct jv_parser* jv_parser_new() { +struct jv_parser* jv_parser_new(jv_parser_flags flags) { struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser)); - parser_init(p); + parser_init(p, flags); return p; } @@ -444,7 +458,7 @@ jv jv_parser_next(struct jv_parser* p) { jv jv_parse_sized(const char* string, int length) { struct jv_parser parser; - parser_init(&parser); + parser_init(&parser, 0); jv_parser_set_buf(&parser, string, length, 0); jv value = jv_parser_next(&parser); if (jv_is_valid(value)) { diff --git a/locfile.c b/locfile.c index f1f17d0f..48cacd6f 100644 --- a/locfile.c +++ b/locfile.c @@ -1,12 +1,16 @@ +#include +#include +#include #include #include -#include -#include +#include +#include "jq.h" #include "jv_alloc.h" #include "locfile.h" -void locfile_init(struct locfile* l, const char* data, int length) { +void locfile_init(struct locfile* l, jq_state *jq, const char* data, int length) { + l->jq = jq; l->data = data; l->length = length; l->nlines = 1; @@ -43,23 +47,57 @@ static int locfile_line_length(struct locfile* l, int line) { } void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) { + jq_err_cb cb; + void *cb_data; va_list fmtargs; va_start(fmtargs, fmt); - vfprintf(stderr, fmt, fmtargs); - va_end(fmtargs); - fprintf(stderr, "\n"); + int startline; + int offset; + + if (loc.start != -1) { + startline = locfile_get_line(l, loc.start); + offset = l->linemap[startline]; + } + + jq_get_error_cb(l->jq, &cb, &cb_data); + + jv m1 = jv_string_vfmt(fmt, fmtargs); + if (!jv_is_valid(m1)) { + jv_free(m1); + goto enomem; + } + jv m2; if (loc.start == -1) { - fprintf(stderr, "\n"); + m2 = jv_string_fmt("%s\n", jv_string_value(m1)); + if (cb) + cb(cb_data, m2); + else + fprintf(stderr, "%s", jv_string_value(m2)); + jv_free(m1); + jv_free(m2); return; } - int startline = locfile_get_line(l, loc.start); - int offset = l->linemap[startline]; - fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset); - fprintf(stderr, "%*s", loc.start - offset, ""); - for (int i = loc.start; - i < loc.end && i < offset + locfile_line_length(l, startline); - i++){ - fprintf(stderr, "^"); + m2 = jv_string_fmt("%s\n%.*s%*s", jv_string_value(m1), + locfile_line_length(l, startline), l->data + offset, + loc.start - offset, ""); + jv_free(m1); + if (!jv_is_valid(m2)) { + jv_free(m2); + goto enomem; } - fprintf(stderr, "\n"); + if (cb) + cb(cb_data, m2); + else + fprintf(stderr, "%s", jv_string_value(m2)); + jv_free(m2); + return; + +enomem: + if (cb != NULL) + cb(cb_data, jv_invalid()); + else if (errno == ENOMEM || errno == 0) + vfprintf(stderr, "Error formatting jq compilation error: %s", strerror(errno ? errno : ENOMEM)); + else + vfprintf(stderr, "Error formatting jq compilation error: %s", strerror(errno)); + return; } diff --git a/locfile.h b/locfile.h index 35907974..4e20df27 100644 --- a/locfile.h +++ b/locfile.h @@ -1,6 +1,8 @@ #ifndef _LOCFILE_H #define _LOCFILE_H +#include "jq.h" + typedef struct { int start, end; } location; @@ -12,9 +14,11 @@ struct locfile { int length; int* linemap; int nlines; + char *error; + jq_state *jq; }; -void locfile_init(struct locfile* l, const char* data, int length); +void locfile_init(struct locfile* l, jq_state *jq, const char* data, int length); void locfile_free(struct locfile* l); diff --git a/main.c b/main.c index c9e09ce8..c8da96a2 100644 --- a/main.c +++ b/main.c @@ -59,17 +59,26 @@ enum { FROM_FILE = 512, + EXIT_STATUS = 8192, + /* debugging only */ DUMP_DISASM = 2048, }; static int options = 0; -static void process(jq_state *jq, jv value, int flags) { +static int process(jq_state *jq, jv value, int flags) { + int ret = 0; jq_start(jq, value, flags); jv result; while (jv_is_valid(result = jq_next(jq))) { if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) { fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout); + if (jv_get_kind(result) == JV_KIND_FALSE) + ret = 10; + else if (jv_get_kind(result) == JV_KIND_NULL) + ret = 11; + else + ret = 0; jv_free(result); } else { int dumpopts; @@ -86,11 +95,18 @@ static void process(jq_state *jq, jv value, int flags) { if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR; if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR; if (options & UNBUFFERED_OUTPUT) dumpopts |= JV_PRINT_UNBUFFERED; + if (jv_get_kind(result) == JV_KIND_FALSE) + ret = 10; + else if (jv_get_kind(result) == JV_KIND_NULL) + ret = 11; + else + ret = 0; jv_dump(result, dumpopts); } printf("\n"); } jv_free(result); + return ret; } FILE* current_input; @@ -143,6 +159,7 @@ int main(int argc, char* argv[]) { ninput_files = 0; int further_args_are_files = 0; int jq_flags = 0; + jv_parser_flags parser_flags = 0; jv program_arguments = jv_array(); for (int i=1; i= argc - 2) { fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname); @@ -251,7 +272,7 @@ int main(int argc, char* argv[]) { } if (options & PROVIDE_NULL) { - process(jq, jv_null(), jq_flags); + ret = process(jq, jv_null(), jq_flags); } else { jv slurped; if (options & SLURP) { @@ -261,7 +282,7 @@ int main(int argc, char* argv[]) { slurped = jv_array(); } } - struct jv_parser* parser = jv_parser_new(); + struct jv_parser* parser = jv_parser_new(parser_flags); char buf[4096]; while (read_more(buf, sizeof(buf))) { if (options & RAW_INPUT) { @@ -271,7 +292,7 @@ int main(int argc, char* argv[]) { slurped = jv_string_concat(slurped, jv_string(buf)); } else { if (buf[len-1] == '\n') buf[len-1] = 0; - process(jq, jv_string(buf), jq_flags); + ret = process(jq, jv_string(buf), jq_flags); } } } else { @@ -281,7 +302,7 @@ int main(int argc, char* argv[]) { if (options & SLURP) { slurped = jv_array_append(slurped, value); } else { - process(jq, value, jq_flags); + ret = process(jq, value, jq_flags); } } if (jv_invalid_has_msg(jv_copy(value))) { @@ -299,11 +320,13 @@ int main(int argc, char* argv[]) { if (ret != 0) goto out; if (options & SLURP) { - process(jq, slurped, jq_flags); + ret = process(jq, slurped, jq_flags); } } out: jv_mem_free(input_filenames); jq_teardown(&jq); + if (ret >= 10 && ret <= 11 && !(options & EXIT_STATUS)) + return 0; return ret; } diff --git a/parser.y b/parser.y index b068b430..6794116b 100644 --- a/parser.y +++ b/parser.y @@ -485,6 +485,26 @@ IDENT '(' Exp ';' Exp ')' { $$ = gen_location(@1, $$); jv_free($1); } | +IDENT '(' Exp ';' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7))); + $$ = gen_location(@1, $$); + jv_free($1); +} | +IDENT '(' Exp ';' Exp ';' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9))); + $$ = gen_location(@1, $$); + jv_free($1); +} | +IDENT '(' Exp ';' Exp ';' Exp ';' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9), gen_lambda($11))); + $$ = gen_location(@1, $$); + jv_free($1); +} | +IDENT '(' Exp ';' Exp ';' Exp ';' Exp ';' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9), gen_lambda($11), gen_lambda($13))); + $$ = gen_location(@1, $$); + jv_free($1); +} | '(' error ')' { $$ = gen_noop(); } | '[' error ']' { $$ = gen_noop(); } | Term '[' error ']' { $$ = $1; } | diff --git a/tests/all.test b/tests/all.test index f8019972..71918a6f 100644 --- a/tests/all.test +++ b/tests/all.test @@ -75,6 +75,10 @@ null "" "<script>hax</script>" +[.[]|tojson|fromjson] +["foo", 1, ["a", 1, "b", 2, {"foo":"bar"}]] +["foo",1,["a",1,"b",2,{"foo":"bar"}]] + # # Dictionary construction syntax # @@ -178,6 +182,10 @@ null [0,1,2,3,4,5,6] [[], [2,3], [0,1,2,3,4], [5,6], [], []] +[.[3:2], .[-5:4], .[:-2], .[-2:], .[3:3][1:], .[10:]] +"abcdefghi" +["","","abcdefg","hi","",""] + del(.[2:4],.[0],.[-2:]) [0,1,2,3,4,5,6,7] [1,4,5] @@ -574,6 +582,59 @@ def inc(x): x |= .+1; inc(.[].a) {} [true, true, false] +# string operations +[.[]|startswith("foo")] +["fo", "foo", "barfoo", "foobar", "barfoob"] +[false, true, false, true, false] + +[.[]|endswith("foo")] +["fo", "foo", "barfoo", "foobar", "barfoob"] +[false, true, true, false, false] + +[.[]|ltrimstr("foo")] +["fo", "foo", "barfoo", "foobar", "afoo"] +["fo","","barfoo","bar","afoo"] + +[.[]|rtrimstr("foo")] +["fo", "foo", "barfoo", "foobar", "foob"] +["fo","","bar","foobar","foob"] + +.[","] +"a,bc,def,ghij,klmno" +[1,4,8,13] + +.[", "] +"a,bc,def,ghij,klmno" +[] + +.[", "] +"a,b,, c, d,ef, , ghi, jklmn, o" +[4,7,13,15,20,27] + +[(index(","), rindex(","))] +"a,bc,def,ghij,klmno" +[1,13] + +[.[]|split(",")] +["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] +[["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]] + +[.[]|split(", ")] +["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] +[["a","bc","def","ghij","jklmn","a,b","c,d","e,f"],["a,b,c,d","e,f,g,h"]] + +[.[] * 3] +["a", "ab", "abc"] +["aaa", "ababab", "abcabcabc"] + +[.[] / ","] +["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] +[["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]] + +[.[] / ", "] +["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] +[["a","bc","def","ghij","jklmn","a,b","c,d","e,f"],["a,b,c,d","e,f,g,h"]] + map(.[1] as $needle | .[0] | contains($needle)) [[[],[]], [[1,2,3], [1,2]], [[1,2,3], [3,1]], [[1,2,3], [4]], [[1,2,3], [1,4]]] [true, true, true, false, false]