navigine/jq
1
0 Fork 0
mirror of https://github.com/jqlang/jq.git synced 2026-11-03 14:05:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: navigine/jq:jq-1.5rc1
navigine/jq:master navigine/jq:nicowilliams/inst-timing navigine/jq:pull/2548/head navigine/jq:owenthereal/release navigine/jq:owenthereal/2561 navigine/jq:dec_literal_number navigine/jq:no-more-ruby navigine/jq:bugfix/aix-issues-jq1.6 navigine/jq:fix-destructuring-alternation navigine/jq:macos-strptime navigine/jq:bugfix/aix-issues navigine/jq:qsort-stability navigine/jq:jq-1.5-branch navigine/jq:tco-in-compiler navigine/jq:libjq navigine/jq:header-cleanup navigine/jq:docs navigine/jq:autotools navigine/jq:haskell-version
navigine/jq:jq-1.8.2 navigine/jq:jq-1.8.2rc1 navigine/jq:jq-1.8.1 navigine/jq:jq-1.8.0 navigine/jq:jq-1.7.1 navigine/jq:jq-1.7 navigine/jq:jq-1.7rc2 navigine/jq:jq-1.7rc1 navigine/jq:jq-1.6 navigine/jq:1.6rc2 navigine/jq:jq-1.6rc1 navigine/jq:jq-1.5 navigine/jq:jq-1.5rc2 navigine/jq:jq-1.5rc1 navigine/jq:jq-1.4 navigine/jq:jq-1.3 navigine/jq:jq-1.2 navigine/jq:jq-1.1 navigine/jq:jq-1.0
..
compare: navigine/jq:docs
navigine/jq:master navigine/jq:nicowilliams/inst-timing navigine/jq:pull/2548/head navigine/jq:owenthereal/release navigine/jq:owenthereal/2561 navigine/jq:dec_literal_number navigine/jq:no-more-ruby navigine/jq:bugfix/aix-issues-jq1.6 navigine/jq:fix-destructuring-alternation navigine/jq:macos-strptime navigine/jq:bugfix/aix-issues navigine/jq:qsort-stability navigine/jq:jq-1.5-branch navigine/jq:tco-in-compiler navigine/jq:libjq navigine/jq:header-cleanup navigine/jq:docs navigine/jq:autotools navigine/jq:haskell-version
navigine/jq:jq-1.8.2 navigine/jq:jq-1.8.2rc1 navigine/jq:jq-1.8.1 navigine/jq:jq-1.8.0 navigine/jq:jq-1.7.1 navigine/jq:jq-1.7 navigine/jq:jq-1.7rc2 navigine/jq:jq-1.7rc1 navigine/jq:jq-1.6 navigine/jq:1.6rc2 navigine/jq:jq-1.6rc1 navigine/jq:jq-1.5 navigine/jq:jq-1.5rc2 navigine/jq:jq-1.5rc1 navigine/jq:jq-1.4 navigine/jq:jq-1.3 navigine/jq:jq-1.2 navigine/jq:jq-1.1 navigine/jq:jq-1.0
3 Commits
jq-1.5rc1 .. docs
Author SHA1 Message Date
Stephen Dolan 6ed2da6d43 Merge pull request #132 from dolmen/fix/github-urls 
Fix Github URLs: stedolan.github.com -> stedolan.github.io
 2013-05-23 09:16:34 -07:00
Olivier Mengué 444f0aacca Fix Github URLs: stedolan.github.com -> stedolan.github.io 2013-05-23 09:36:54 +02:00
Stephen Dolan 22006172ba fix typo and clean up docs build 2013-05-20 13:24:48 +01:00
98 changed files with 4171 additions and 11403 deletions
Expand all files Collapse all files

No files matched your search

.gitignore
+1 -26
View File
@@ -1,16 +1,7 @@
*.o
*.a
*.lo
*.la
*.lai
*.so
*.so.*
*~
.*.sw[a-p]
tags
jq
!tests/modules/lib/jq/
jq.1
# Something delightfully recursive happens otherwise
@@ -23,31 +14,15 @@ parser.c
parser.h
# Autotools junk
.libs
.deps
libtool
*.log
stamp-h1
config.log
config.status
autom4te.cache
INSTALL
config.h
Makefile
jq-*.tar.gz
configure
aclocal.m4
Makefile.in
version.h
.remake-version-h
config.cache
*.rpm
m4/libtool.m4
m4/ltoptions.m4
m4/ltsugar.m4
m4/ltversion.m4
m4/lt~obsolete.m4
tests/all.trs
cscope.in.out
cscope.out
cscope.po.out
.travis.yml
-19
View File
@@ -1,19 +0,0 @@
sudo: false
language: c
before_script:
- sudo apt-get install -qq valgrind
- bundle install --gemfile=docs/Gemfile
- autoreconf -i
script:
- ./setup.sh
- ./configure && make
compiler:
- gcc
- clang
notifications:
email: false
AUTHORS
+3 -29
View File
@@ -1,32 +1,6 @@
Created By:
Stephen Dolan <mu@netsoc.tcd.ie>
Maintained by:
Nicolas Williams <nico@cryptonector.com>
Contributions by:
Anthony Shortland <anthony.shortland@me.com> - rpmbuild target
Lee Thompson <stagr.lee@gmail.com> - autoconf stuff
Santiago Lapresta <santiago.lapresta@gmail.com> - join, arrays, all, any, other filters
Nicolas Williams <nico@cryptonector.com> - library-fication, autoconf stuff, exception handling, various
Jingwen Owen Ou <jingweno@gmail.com> - jqplay.org and link to it
Andrew Rodland <andrew@cleverdomain.org> - bug fixes
Ankur <ankz.kothari@gmail.com> - bug fixes
Brendan Macmillan <melbourne.research@gmail.com> - bug fixes, load library from ~/.jq
William Langford <wlangfor@gmail.com> - module system, bug fixes
David R. MacIver <david@drmaciver.com> - bug fixes
Filippo Giunchedi <fgiunchedi@gmail.com> - bug fixes
Filippo Valsorda <filippo.valsorda@gmail.com> - recursive object merge (`*`)
Jack Pearkes <jackpearkes@gmail.com> - update tutorial
Joe Littlejohn <joe.littlejohn@nokia.com> - bug fixes
Juan Guerrero <juan.guerrero.lozano@gmail.com> - bug fixes
Kenny Shen <kenny.shen@zalora.com> - doc fixes
Markus Lanthaler <mark_lanthaler@gmx.net> - doc fixes
Michael Daines <michael@mdaines.com> - add sqrt; doc fixes
Mike McCabe <mccabe@archive.org> - math (libm) functions
Nicolas Pouillard <nicolas.pouillard@gmail.com> - add tests
Peter van Dijk <peter@7bits.nl> - doc fixes
Philipp Hagemeister <phihag@phihag.de> - doc fixes
polyester <paul@cleanclothes.org> - automake version update
Rémy Léone <remy.leone@gmail.com> - add .travis.yml
Ryoichi KATO <ryo1kato@gmail.com> - doc fixes
Contributions by
Anthony Shortland <anthony.shortland@me.com> - rpmbuild target
Lee Thompson <stagr.lee@gmail.com> - autoconf stuff
ChangeLog
-617
View File
@@ -1,617 +0,0 @@
2014-06-09 Nicolas Williams <nico@cryptonector.com>
Add libm.h to dist file list
Add note about cmd.exe quoting
Building docs fails on powerpc (#349)
2014-06-08 Nicolas Williams <nico@cryptonector.com>
Update site news
Also fix configure.ac to use git describe --tags
Fix scripts/version: use git describe --tags ...
After tagging as 1.4 scripts/version was still producing jq-1.3-....
Add `indices(s)`, improve `index(s)`, `rindex(s)`
Now these deal with arrays as input and `s` being an array or a scalar.
Improve `index` and `rindex` examples
Remove reference to `getpath` from docs
Document `index` and `rindex` (#389)
2014-06-07 Santiago Lapresta <santiago.lapresta@gmail.com>
Added `join` function
2014-06-07 Nicolas Williams <nico@cryptonector.com>
String * number should be commutative
2014-06-04 Nicolas Williams <nico@cryptonector.com>
Add cross-compilation notes to README
A detailed set of instruction as to how to setup a cross-compilation
environment for OS X and Win32/64 would be nice.
Add -j option to scripts/crosscompile
Add flags argument to jv_parser_new()
For extensibility. We might add streaming parser options, even binary
JSON encoding options.
2014-06-02 Nicolas Williams <nico@cryptonector.com>
Fix tests failures on Windows
And Solaris 8 and 9 too, no doubt. The problem was that non-standard
vsnprintf()s that return -1 when the buffer is too small were not
properly supported.
2014-05-20 Santiago Lapresta <santiago.lapresta@gmail.com>
Documented `del` command
2014-05-11 Santiago Lapresta <santiago.lapresta@gmail.com>
Added texts/examples to unique_by function
Added unique_by function
2014-04-17 Nicolas Williams <nico@cryptonector.com>
Make pthread tls configurable for Mingw build
For the Mingw build we don't want to pull in the pthread DLL just
because we can autodetect pthread support. That would make the jq.exe
binary not self-contained.
2014-04-16 Nicolas Williams <nico@cryptonector.com>
Add autoconf checks for pthreads; fix #340
2014-03-20 Jingwen Owen Ou <jingweno@gmail.com>
Add link to jqplay
2014-03-13 Nicolas Williams <nico@cryptonector.com>
Fix for #303 in the sources
2014-03-13 Santiago Lapresta <santiago.lapresta@gmail.com>
Added `arrays` and other filters
Arrays, objects, numbers, strings, booleans, nulls, values (non-nulls)
-- these builtins filter out those inputs that don't match the name of
the builtin.
This fixes #322 and #324.
2014-03-07 Filippo Valsorda <filippo.valsorda@gmail.com>
Add a recursive object merge strategy and bind it to *
This commit adds a jv_object_merge_recursive function, that performs
recursive object merging, and binds it to multiply when applied to
two objects.
Closes #320
2014-03-06 Nicolas Williams <nico@cryptonector.com>
Make libm tests more portable
2014-02-26 Andrew Rodland <andrew@cleverdomain.org>
Repair jv_show
2014-02-26 Andrew Rodland <andrew@cleverdomain.org>
Make jq --raw-output --unbuffered work
--unbuffered was only affecting the normal output case, not the --raw-output case. Make the two of them play together.
This also makes sure that the output is flushed *after* printing the newline, so a consumer doesn't lag a line behind.
2014-02-21 Nicolas Williams <nico@cryptonector.com>
Add cbrt (cube root)
Add missing trig functions and barebones test
Remove non-standard exp10()
2014-02-21 Mike McCabe <mccabe@archive.org>
Initial add of math functions.
2014-02-20 Nicolas Williams <nico@cryptonector.com>
Add `?`, `.[]?`, and `..` operators
Make XPath-like `//a/b` recursive structure traversal easier in jq,
which then becomes:
..|.a?.b?
The `?` operator suppresses errors about . not being an array or object.
The `..` operator is equivalent to calling the new `recurse_down`
built-in, which in turn is equivalent to
recurse(.[]?)
Note that `..a` is not supported; neither is `...a`. That could be add
added, but it doesn't seem worth the trouble of saving the need to type
a '|'.
2014-02-16 Santiago Lapresta <santiago.lapresta@gmail.com>
Added `all` and `any` builtins
2014-01-25 polyester <paul@cleanclothes.org>
work with newer versions of automake
when using a newer automake, the autoreconf step fails with warnings:
"linking libtool libraries using a non-POSIX archiver requires 'AM_PROG_AR' in 'configure.ac' "
This happens for instance on ubuntu 13.10.
Doing just that, adding 'AM_PROG_AR' to configure.ac fixes the problem.
2014-01-01 Nicolas Williams <nico@cryptonector.com>
Fix #201; check that bison accepts --warnings
2013-12-27 Joe Littlejohn <joe.littlejohn@nokia.com>
Fix rpm build (`make rpm`)
* Re-add VERSION as it's required for `./setup superclean`
and `make rpm`.
* Add *.rpm to git ignore, we never want them under version control.
2013-12-27 Filippo Giunchedi <fgiunchedi@gmail.com>
include additional files in jq.spec
this will probably need changing upon SONAME bump
fix rpm Makefile target and prerequisites
depend on dist and the specfile, plus use automake's variables
2013-12-26 Nicolas Williams <nico@cryptonector.com>
Document --version
2013-12-26 Nicolas Williams <nico@cryptonector.com>
Add jv_dumpf() and jv_show()
jv_dumpf() takes a FILE *.
jv_show() is intended for use in debuggers, so it dumps the jv to stderr
and it does not jv_free() the jv, so it's safe to
"call jv_show(some_jv, -1)" in a debugger. If flags == -1 then the jv
will be shown pretty-printed and in color.
2013-12-26 Nicolas Williams <nico@cryptonector.com>
Document .foo.bar in manual
Document exit numbers
Normalize errors for -e
2013-12-25 Nicolas Williams <nico@cryptonector.com>
Fix doc typos (.[foo] wanted to be .["foo"])
Add note to jq.1 about shell quoting
2013-12-20 Philipp Hagemeister <phihag@phihag.de>
Ignore the config/test-driver file
This file is automatically generated and does not need to be committed.
Fix @uri example
Previously, the @uri example didn't match the actual behavior of the current jq, as exclamation marks do not need to be encoded in URIs.
Replace the example with an input that needs encoding, and is encoded by jq.
2013-12-17 Stephen Dolan <mu@netsoc.tcd.ie>
Allow negated object values without parens. Fixes #247
2013-12-17 Nicolas Williams <nico@cryptonector.com>
Fix memmem() error
2013-12-13 Rémy Léone <remy.leone@gmail.com>
Adding a .travis.yml file to use the travis-ci.org
From wikipedia:
Travis CI is a hosted, distributed continuous integration service used
to build and test projects hosted at GitHub.
Travis CI is configured by adding a file named .travis.yml, which is a
YAML format text file, to the root directory of the GitHub repository.
Travis CI automatically detects when a commit has been made and pushed
to a GitHub repository that is using Travis CI, and each time this
happens, it will try to build the project and run tests. This includes
commits to all branches, not just to the master branch. When that
process has completed, it will notify a developer in the way it has been
configured to do so — for example, by sending an email containing the
test results (showing success or failure), or by posting a message on an
IRC channel. It can be configured to run the tests on a range of
different machines, with different software installed (such as older
versions of a programming language, to test for compatibility).
2013-12-13 Stephen Dolan <mu@netsoc.tcd.ie>
Make the testsuite run on machines without valgrind
Format more integers as integers, not scientific notation.
jq is now willing to put up to 15 zeros after an integer before
moving to scientific notation.
2013-12-11 Nicolas Williams <nico@cryptonector.com>
Complete more-arity feature not complete
And test
2013-12-10 David R. MacIver <david@drmaciver.com>
convert range bounds to integers in a way that avoids undefined behaviour
add checking of numeric indices to an array to see if they can reasonably be considered integers. Avoid undefined behaviour if out of bounds
2013-12-09 David R. MacIver <david@drmaciver.com>
some functions were missing prototypes. Add them
2013-12-08 David R. MacIver <david@drmaciver.com>
These vfprintfs are being used as if they were printfs. Fix that
consistent use of goto out in main
2013-12-08 Stephen Dolan <mu@netsoc.tcd.ie>
Refactor jv structure.
New structure layout is simpler and also faster. In particular, it's
now small enough to be passed in registers on amd64.
Make testsuite not leak when compiled with -DNDEBUG.
2013-12-08 David R. MacIver <david@drmaciver.com>
test for losing memory on compile errors
args to jq_compile_args were not getting freed when there were errors in the compile
2013-12-06 Nicolas Williams <nico@cryptonector.com>
Fix double-free typo in print_error()
Fix manual.yml
2013-12-04 Nicolas Williams <nico@cryptonector.com>
Conditionally #define _GNU_SOURCE in compile.c
Add tests for string index by string and builtins
Add index and rindex builtins
Add index strings by string; return string indexes
% jq '.[","]'
"a,bc,def,ghij,klmno"
[1,4,8,13]
%
Make length return abs value of numeric inputs
Add callback interface for errors
Printing to stderr is not the right answer for a library.
Add jv_string_vfmt()
Document ltrimstr and rtrimstr
Test ltrimstr and rtrimstr functions
Add ltrimstr and rtrimstr functions
Document -e / --exit-status argument
Add -e | --exit-status CLI option
Document tojson and fromjson builtins
Test tojson and fromjson
Add tojson and fromjson builtins
Document split function
Document string multiplication and division
Document string functions and slicing
Test string slicing
Add string slicing
Add tests for string division/splitting
Add string division by string (split on separator)
Test starts/endswith and string multiplication
Add string multiplication by number
Add startswith/endswith
Add explode/implode jq functions to match jv API
Use uint32_t for codepoint in jv_string_append_codepoint()
Add jv string utility functions
jv_string_empty()
-> return an empty string with given allocated length (for fast
appends)
jv_string_append_codepoint
-> append a single codepoint (int) to the given string
jv_string_explode
-> return an array of codepoints making up a string
jv_string_implode
-> return the UTF-8 encoding of an array of codepoint numbers
Support more arguments for defs
2013-12-04 Stephen Dolan <mu@netsoc.tcd.ie>
Preserve insertion order in objects. Closes #169.
2013-11-30 Nicolas Pouillard <nicolas.pouillard@gmail.com>
Add a few more test cases (from the man page)
2013-11-08 Stephen Dolan <mu@netsoc.tcd.ie>
Add a --unbuffered option. Closes #206
2013-11-07 Peter van Dijk <peter@7bits.nl>
count should be length
Example refers to a count function, which does not exist. Replacing it with length works.
2013-11-07 Stephen Dolan <mu@netsoc.tcd.ie>
Fix a crash on group_by of empty list. Fixes #208.
2013-10-16 Ryoichi KATO <ryo1kato@gmail.com>
Docs: add description of --from-file option
2013-10-06 Juan Guerrero <juan.guerrero.lozano@gmail.com>
Fix typo on error message
2013-09-19 Kenny Shen <kenny.shen@zalora.com>
Add missing -i flag in build instructions
2013-09-14 Michael Daines <michael@mdaines.com>
Add test showing calculation of standard deviation
2013-09-13 Mike Daines <michael@mdaines.com>
Fix typo
2013-09-11 Michael Daines <michael@mdaines.com>
Add sqrt operator
2013-09-04 Jack Pearkes <jackpearkes@gmail.com>
docs: update the tutorial to use GitHub's API
2013-09-01 Ankur <ankz.kothari@gmail.com>
Call AM_INIT_AUTOMAKE once only
Fixes build with automake-1.14
2013-08-19 Joe Littlejohn <joe.littlejohn@nokia.com>
Fix Makefile after refactoring of stacks in 05d90517b02
2013-06-23 Stephen Dolan <mu@netsoc.tcd.ie>
Remove #includes from jv.h
Fix the jv_parser interface.
Use libtool's built-in symbol exporting rather than a mapfile.
Move gen_utf8_tables to scripts
Move libtool m4 junk to config/ and delete some autogenerated files.
Remove Autoconf-generated config.h.
2013-06-22 Stephen Dolan <mu@netsoc.tcd.ie>
Build libjq only once, and link it statically to ./jq
This means ./jq is a real binary rather than a libtool turd.
Fix distcheck.
Update list of files to be distributed.
Utf8 fixes. Closes #161
Reject all overlong UTF8 sequences.
Fix various UTF8 parsing bugs.
In particular, parse bad UTF8 by replacing the broken bits with U+FFFD
and resychronise correctly after broken sequences.
Fix example in manual for `floor`. See #155.
2013-06-21 Nicolas Williams <nico@cryptonector.com>
Document floor
Add floor operator
Document mod
Add mod (and setmod) operators
Update .gitignore
Add libjq autoconf goo
Quiet setup.sh re: tmp dir
2013-06-21 Stephen Dolan <mu@netsoc.tcd.ie>
Move cfunction invocation code to the interpreter loop.
2013-06-18 Nicolas Williams <nico@cryptonector.com>
Fix serious bug in handling of --argfile
Fix leaks in jv_load_file()
2013-06-17 Stephen Dolan <mu@netsoc.tcd.ie>
Fold opcode.{c,h} into bytecode.{c,h}
Simplify block functions for variables
Saner build instructions in README.md
Closes #144
Remove some initialise-to-zero code.
This lets valgrind find more bugs - if a field isn't given a
well-defined value valgrind will now find it instead of seeing it
set to zero with memset.
2013-06-17 Nicolas Williams <nico@cryptonector.com>
Remove accidentally introduced use of fopen "e"
2013-06-16 Stephen Dolan <mu@netsoc.tcd.ie>
Merge pull request #114 from nicowilliams/nomem_handler
Add jv_nomem_handler()
2013-06-16 Nicolas Williams <nico@cryptonector.com>
Remove last remnant of main.h
2013-06-15 Nicolas Williams <nico@cryptonector.com>
Allow --run-tests to take a file argument
Fixup API to get closer to a libjq
2013-06-15 Nicolas Williams <nico@cryptonector.com>
Move slurp_file() into library as jv_load_file()
Needed as part of creating a libjq.
2013-06-14 Stephen Dolan <mu@netsoc.tcd.ie>
Clean up lots of stack and frame logic.
Move frame defs to execute.c
2013-06-13 Stephen Dolan <mu@netsoc.tcd.ie>
Simplify frame logic.
Unify all stacks. Passes tests, but needs cleanup.
2013-06-11 Stephen Dolan <mu@netsoc.tcd.ie>
Support ."foo" syntax for accessing fields. See #141.
2013-06-09 Stephen Dolan <mu@netsoc.tcd.ie>
Unify frame and data stacks
2013-06-05 Stephen Dolan <mu@netsoc.tcd.ie>
Speed up cached configure (./configure -C)
Clean up flex lines in build
Lex and parse .foo better.
'.as' is now valid, '. foo' is now invalid. See #141.
2013-06-04 Markus Lanthaler <mark_lanthaler@gmx.net>
Update README.md
Update the link to the documentation. All GitHub pages are now using the github.io domain.
2013-06-03 Stephen Dolan <mu@netsoc.tcd.ie>
Make jq --version print to stdout, not stderr
Better error handling for .foo case in parser. See #141.
Let the parser rather than the lexer handle invalid characters.
Add command-line option to sort object keys.
Closes #79.
Clean up Makefile.am (distcheck, rebuild version.h less often)
2013-05-31 Brendan Macmillan <melbourne.research@gmail.com>
Stop warning on fgets, simple version
Stop warning on fgets, complex version
2013-05-31 Stephen Dolan <mu@netsoc.tcd.ie>
Squash a warning on some GCC versions
2013-05-29 Stephen Dolan <mu@netsoc.tcd.ie>
Support for printing object keys in sorted order.
No command-line option to enable this yet. See #79.
2013-05-29 Brendan Macmillan <melbourne.research@gmail.com>
Bugfix multiline off-by-one (locfile.c)
locfile.h -> locfile.h + locfile.c
clean up includes of a few files
Hack bugfix for multiline off-by-one (locfile.c)
Load library from ~/.jq
2013-05-24 Stephen Dolan <mu@netsoc.tcd.ie>
Make jq --version report an actual git revision.
Closes #129.
2013-05-23 Nicolas Williams <nico@cryptonector.com>
Add --argfile variant of --arg (issue #117)
This is useful when one has a database (in JSON form) to query using jq
input data.
% echo '{"a":1, "c":5}' > db.json
% echo '"c"'|./jq --argfile f /tmp/a '$f[.]'
5
% echo '"a"'|./jq --argfile f /tmp/a '$f[.]'
1
% echo '"b"'|./jq --argfile f /tmp/a '$f[.]'
null
%
2013-05-23 Stephen Dolan <mu@netsoc.tcd.ie>
'make clean' won't delete jq.1 if it can't be rebuilt.
See #131
Makefile.am
+28 -62
View File
@@ -1,12 +1,13 @@
### C source files to be built and distributed.
LIBJQ_INCS = jq_parser.h builtin.h bytecode.h compile.h exec_stack.h \
libm.h jv_alloc.h jv_dtoa.h jv_unicode.h locfile.h \
opcode_list.h parser.y jv_utf8_tables.h lexer.l util.h linker.h
JQ_INCS = jq_parser.h builtin.h bytecode.h compile.h execute.h \
forkable_stack.h frame_layout.h jv.h jv_alloc.h jv_aux.h jv_dtoa.h \
jv_parse.h jv_unicode.h locfile.h opcode.h opcode_list.h parser.y \
jv_utf8_tables.h lexer.l
LIBJQ_SRC = locfile.c bytecode.c compile.c execute.c builtin.c jv.c \
jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_file.c \
jv_alloc.c jq_test.c util.c linker.c ${LIBJQ_INCS}
JQ_SRC = opcode.c bytecode.c compile.c execute.c builtin.c jv.c \
jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c jv_alloc.c \
jq_test.c ${JQ_INCS}
### C build options
@@ -14,15 +15,14 @@ LIBJQ_SRC = locfile.c bytecode.c compile.c execute.c builtin.c jv.c \
AM_CFLAGS = -Wextra -Wall -Wno-missing-field-initializers \
-Wno-unused-parameter -Wno-unused-function
ACLOCAL_AMFLAGS = -I config/m4
### Generating the lexer and parser
# While there is some autoconf macro support for lex/flex, it doesn't support
# header file creation so we'll use good old make
BUILT_SOURCES = lexer.h lexer.c parser.h parser.c version.h
BUILT_SOURCES = lexer.h lexer.c parser.h parser.c
lexer.c: lexer.l
$(AM_V_LEX) flex -o lexer.c --header-file=lexer.h $<
flex -o lexer.c --header-file=lexer.h $<
lexer.h: lexer.c
# Tell YACC (bison) autoconf macros that you want a header file created.
@@ -30,31 +30,14 @@ lexer.h: lexer.c
# OSX ships an old bison, so update with homebrew or macports
AM_YFLAGS = --warnings=all -d
### libjq
lib_LTLIBRARIES = libjq.la
libjq_la_SOURCES = ${LIBJQ_SRC}
libjq_la_LIBADD = -lm
libjq_la_LDFLAGS = -export-symbols-regex '^j[qv]_' -version-info 1:4:0
include_HEADERS = jv.h jq.h
### Building the jq binary
# Remake the version.h header file if, and only if, the git ID has changed
.PHONY: .FORCE
.FORCE:
generate_ver = ver="`{ $(srcdir)/scripts/version || echo '$(VERSION)' ; } | sed 's/.*/\#define JQ_VERSION \"&\"/'`"
.remake-version-h: .FORCE
@ $(generate_ver); test "x`cat version.h 2>/dev/null`" = "x$$ver" || touch .remake-version-h
version.h: .remake-version-h
$(AM_V_GEN) $(generate_ver); echo "$$ver" > $@
main.c: config.h
bin_PROGRAMS = jq
jq_SOURCES = main.c
jq_LDFLAGS = -static-libtool-libs
jq_LDADD = libjq.la -lm
jq_SOURCES = ${JQ_SRC} main.c
### Tests (make check)
@@ -73,23 +56,20 @@ man_MANS = jq.1
.PHONY: real_docs
if ENABLE_DOCS
jq.1: $(srcdir)/docs/content/3.manual/manual.yml
$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
jq.1.prebuilt: jq.1
$(AM_V_GEN) cp $^ $@ || { rm -f $@; false; }
( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage ) > $@ || { rm -f $@; false; }
jq.1.default: $(srcdir)/docs/default_manpage.md
$(AM_V_GEN) ( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
real_docs: jq.1 jq.1.prebuilt
@if cmp jq.1 $(srcdir)/jq.1.default > /dev/null; then\
( cd ${abs_srcdir}/docs; '$(BUNDLER)' exec rake manpage_default ) > $@ || { rm -f $@; false; }
real_docs: jq.1
if cmp jq.1 $(srcdir)/jq.1.default > /dev/null; then\
rm -f jq.1; $(MAKE) $(AM_MAKEFLAGS) jq.1;\
fi
else
jq.1: $(srcdir)/jq.1.prebuilt
$(AM_V_GEN) cp $^ $@
jq.1: $(srcdir)/jq.1.default
cp $^ $@
real_docs:
@echo "Ruby dependencies not found, cannot build manpage." > /dev/stderr
@echo "Follow the instructions in docs/README.md to install them" > /dev/stderr
@echo "and then rerun ./configure" > /dev/stderr
@echo "A pre-built, possibly out-of-date manpage used." > /dev/stderr
false
endif
@@ -113,31 +93,24 @@ DOC_FILES = docs/content docs/public docs/templates docs/site.yml \
# setup is only used by distribution developers, not package developers.
# Still, as a matter of allowing patching, its not a bad idea to distribute
# the developer setup script in the tarball.
EXTRA_DIST = $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER) jq.spec \
$(DOC_FILES) scripts/version parser.h parser.c lexer.h \
lexer.c tests/onig.supp tests/torture/input0.json \
tests/modules/.jq tests/modules/a.jq tests/modules/b/b.jq \
tests/modules/c/c.jq tests/modules/c/d.jq \
tests/modules/lib/jq/e/e.jq tests/modules/lib/jq/f.jq \
tests/modules/streaming.jq tests/modules/data.json \
tests/modules/syntaxerror/syntaxerror.jq \
tests/modules/test_bind_order.jq \
tests/modules/test_bind_order0.jq \
tests/modules/test_bind_order1.jq \
tests/modules/test_bind_order2.jq
EXTRA_DIST = config.h.in $(BUILT_SOURCES) $(man_MANS) $(TESTS) \
$(TEST_LOG_COMPILER) gen_utf8_tables.py jq.spec \
$(DOC_FILES)
# README.md is expected in Github projects, good stuff in it, so we'll
# distribute it and install it with the package in the doc directory.
docdir = ${datadir}/doc/${PACKAGE}
dist_doc_DATA = README.md COPYING AUTHORS README
releasetag:
git tag -s "jq-$$(cat VERSION)" -m "jq release $$(cat VERSION)"
RELEASE ?= 1
rpm: dist jq.spec
rpm: jq
@echo "Packaging jq as an RPM ..."
mkdir -p rpm/SOURCES rpm/BUILD rpm/BUILDROOT rpm/RPMS rpm/SPECS
cp jq-$(VERSION).tar.gz rpm/SOURCES/
rpmbuild -tb --define "_topdir ${PWD}/rpm" --define "_prefix /usr" --define "myver $(VERSION)" --define "myrel ${RELEASE}" rpm/SOURCES/jq-$(VERSION).tar.gz
cp jq-$$(cat VERSION).tar.gz rpm/SOURCES/
rpmbuild -tb --define "_topdir ${PWD}/rpm" --define "_prefix /usr" --define "myver $$(cat VERSION)" --define "myrel ${RELEASE}" rpm/SOURCES/jq-$$(cat VERSION).tar.gz
find rpm/RPMS/ -name "*.rpm" -exec mv {} ./ \;
rm -rf rpm
@@ -146,12 +119,5 @@ dist-clean-local:
# Not sure why this doesn't get cleaned up automatically, guess
# automake used to man pages which are hand coded?
# 'make clean' doesn't delete the manpage if it can't be rebuilt
.PHONY: clean-local-docs
clean-local-docs:
if ENABLE_DOCS
clean-local:
rm -f jq.1
endif
clean-local: clean-local-docs
rm -f version.h .remake-version-h
NEWS
-64
View File
@@ -1,64 +0,0 @@
Release history
* jq version 1.0 was released on Sun Oct 21 2012
* jq version 1.1 was released on Sun Oct 21 2012
* jq version 1.2 was released on Thu Dec 20 2012
* jq version 1.3 was released on Sun May 19 2013
* jq version 1.4 was released on Mon Jun 9 2014
As yet unreleased features in master branch, since 1.4:
- semantics
- multiple function argument numbers for any one function name,
dispatching on number of arguments in function calls
- misc
- env builting (to access the environment)
- prebuilt jq.1 man page
New features in 1.4 since 1.3:
- command-line arguments
- jq --arg-file variable file
- jq --unbuffered
- jq -e / --exit-status (set exit status based on outputs)
- jq -S / --sort-keys (now jq no longer sorts object keys by
default
- syntax
- .. -> like // in XPath (recursive traversal)
- question mark (e.g., .a?) to suppress errors
- ."foo" syntax (equivalent to .["foo"])
- better error handling for .foo
- added % operator (modulo)
- allow negation without requiring extra parenthesis
- more function arguments (up to six)
- filters:
- any, all
- iterables, arrays, objects, scalars, nulls, booleans, numbers,
strings, values
- string built-ins:
- split
- join (join an array of strings with a given separator string)
- ltrimstr, rtrimstr
- startswith, endswith
- explode, implode
- fromjson, tojson
- index, rindex, indices
- math functions
- floor, sqrt, cbrt, etetera (depends on what's available from libm)
- libjq -- a C API interface to jq's JSON representation and for
running jq programs from C applications
README.md
+7 -19
View File
@@ -6,30 +6,18 @@ jq is a command-line JSON processor.
If you want to learn to use jq, read the documentation at
[http://stedolan.github.io/jq](http://stedolan.github.io/jq). This
documentation is generated from the docs/ folder of this repository.
You can also try it online at [jqplay.org](http://jqplay.org).
If you want to hack on jq, feel free, but be warned that its internals
are not well-documented at the moment. Bring a hard hat and a
shovel. Also, read the wiki: http://github.com/stedolan/jq/wiki
shovel. Also, read the wiki: https://github.com/stedolan/jq/wiki
If you're building directly from the latest git, you'll need flex,
bison, libtool, make, autoconf and libonig installed. To build, run:
To build jq, run
autoreconf -i
./configure
make -j8
make check
make
sudo make install (optionally)
After make finishes, you'll be able to use `./jq`. You can also
install it using:
sudo make install
If you're not using the latest git version but instead building a
released tarball (available on the website), then you won't need to
run `autoreconf` (and shouldn't), and you won't need flex or bison.
To cross-compile for OS X and Windows, see docs/Rakefile's build task
and scripts/crosscompile. You'll need a cross-compilation environment,
such as Mingw for cross-compiling for Windows.
If you've just checked out the latest version from git (rather than
using a released source tarball) then you'll need to run this first:
autoreconf
builtin.c
+83 -766
View File
File diff suppressed because it is too large. Load diff
builtin.h
+14 -3
View File
@@ -1,10 +1,21 @@
#ifndef BUILTIN_H
#define BUILTIN_H
#include "jq.h"
#include "bytecode.h"
#include "compile.h"
int builtins_bind(jq_state *, block*);
block builtins_bind(block);
typedef void (*cfunction_ptr)(void);
struct cfunction {
cfunction_ptr fptr;
const char* name;
int nargs;
};
jv cfunction_invoke(struct cfunction* function, jv input[]);
#endif
bytecode.c
+20 -53
View File
@@ -3,64 +3,17 @@
#include <stdlib.h>
#include "bytecode.h"
#include "opcode.h"
#include "jv_alloc.h"
// flags, length
#define NONE 0, 1
#define CONSTANT OP_HAS_CONSTANT, 2
#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
#define GLOBAL (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
#define BRANCH OP_HAS_BRANCH, 2
#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
#define OP(name, imm, in, out) \
{name, #name, imm, in, out},
static const struct opcode_description opcode_descriptions[] = {
#include "opcode_list.h"
};
static const struct opcode_description invalid_opcode_description = {
-1, "#INVALID", 0, 0, 0, 0
};
const struct opcode_description* opcode_describe(opcode op) {
if ((int)op >= 0 && (int)op < NUM_OPCODES) {
return &opcode_descriptions[op];
} else {
return &invalid_opcode_description;
}
}
int bytecode_operation_length(uint16_t* codeptr) {
static int bytecode_operation_length(uint16_t* codeptr) {
int length = opcode_describe(*codeptr)->length;
if (*codeptr == CALL_JQ || *codeptr == TAIL_CALL_JQ) {
if (*codeptr == CALL_JQ) {
length += codeptr[1] * 2;
}
return length;
}
static void dump_code(int indent, struct bytecode* bc) {
int pc = 0;
while (pc < bc->codelen) {
printf("%*s", indent, "");
dump_operation(bc, bc->code + pc);
printf("\n");
pc += bytecode_operation_length(bc->code + pc);
}
}
static void symbol_table_free(struct symbol_table* syms) {
jv_mem_free(syms->cfunctions);
jv_free(syms->cfunc_names);
jv_mem_free(syms);
}
void dump_disassembly(int indent, struct bytecode* bc) {
if (bc->nclosures > 0) {
printf("%*s[params: ", indent, "");
@@ -84,6 +37,16 @@ void dump_disassembly(int indent, struct bytecode* bc) {
}
}
void dump_code(int indent, struct bytecode* bc) {
int pc = 0;
while (pc < bc->codelen) {
printf("%*s", indent, "");
dump_operation(bc, bc->code + pc);
printf("\n");
pc += bytecode_operation_length(bc->code + pc);
}
}
static struct bytecode* getlevel(struct bytecode* bc, int level) {
while (level > 0) {
bc = bc->parent;
@@ -99,7 +62,7 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
printf("%s", op->name);
if (op->length > 1) {
uint16_t imm = bc->code[pc++];
if (op->op == CALL_JQ || op->op == TAIL_CALL_JQ) {
if (op->op == CALL_JQ) {
for (int i=0; i<imm+1; i++) {
uint16_t level = bc->code[pc++];
uint16_t idx = bc->code[pc++];
@@ -146,9 +109,13 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
}
}
void symbol_table_free(struct symbol_table* syms) {
jv_mem_free(syms->cfunctions);
jv_free(syms->cfunc_names);
jv_mem_free(syms);
}
void bytecode_free(struct bytecode* bc) {
if (!bc)
return;
jv_mem_free(bc->code);
jv_free(bc->constants);
for (int i=0; i<bc->nsubfunctions; i++)
bytecode.h
+4 -48
View File
@@ -3,55 +3,10 @@
#include <stdint.h>
#include "jv.h"
typedef enum {
#define OP(name, imm, in, out) name,
#include "opcode_list.h"
#undef OP
} opcode;
enum {
NUM_OPCODES =
#define OP(name, imm, in, out) +1
#include "opcode_list.h"
#undef OP
};
enum {
OP_HAS_CONSTANT = 2,
OP_HAS_VARIABLE = 4,
OP_HAS_BRANCH = 8,
OP_HAS_CFUNC = 32,
OP_HAS_UFUNC = 64,
OP_IS_CALL_PSEUDO = 128,
OP_HAS_BINDING = 1024,
// NOTE: Not actually part of any op -- a pseudo-op flag for special
// handling of `break`.
OP_BIND_WILDCARD = 2048,
};
struct opcode_description {
opcode op;
const char* name;
int flags;
// length in 16-bit units
int length;
int stack_in, stack_out;
};
const struct opcode_description* opcode_describe(opcode op);
#include "opcode.h"
#include "builtin.h"
#define MAX_CFUNCTION_ARGS 10
typedef void (*cfunction_ptr)();
struct cfunction {
cfunction_ptr fptr;
const char* name;
int nargs;
};
struct symbol_table {
struct cfunction* cfunctions;
int ncfunctions;
@@ -84,9 +39,10 @@ struct bytecode {
};
void dump_disassembly(int, struct bytecode* code);
void dump_code(int, struct bytecode* code);
void dump_operation(struct bytecode* bc, uint16_t* op);
int bytecode_operation_length(uint16_t* codeptr);
void symbol_table_free(struct symbol_table* syms);
void bytecode_free(struct bytecode* bc);
#endif
compile-ios.sh
-102
View File
@@ -1,102 +0,0 @@
#!/bin/bash
# old values
OLD_CFLAGS=$CFLAGS
OLD_LDFLAGS=$LDFLAGS
ORIG_PWD=`pwd`
# For parallelism in make
NJOBS="-j`sysctl -n hw.ncpu || echo 1`"
# Get oniguruma
rm -rf $PWD/build/ios onig-5.9.5
echo "Downloading oniguruma 5.9.5"
curl http://www.geocities.jp/kosako3/oniguruma/archive/onig-5.9.5.tar.gz | tar xz
cd onig-5.9.5
# So, we need to remake the configure scripts so that the arm64 architecture
# exists in config.sub. In order to keep autoreconf from failing, create
# NEWS and ChangeLog.
touch NEWS ChangeLog
autoreconf -fi >/dev/null 2>&1
CC=`xcrun -f clang`
cd $ORIG_PWD
autoreconf -fi
for arch in i386 x86_64 armv7 armv7s arm64; do
# Some of the architectures are a bit different...
if [[ "$arch" = "i386" || "$arch" = "x86_64" ]]
then
SYSROOT=`xcrun -f --sdk iphonesimulator --show-sdk-path`
else
SYSROOT=`xcrun -f --sdk iphoneos --show-sdk-path`
fi
if [[ "$arch" = "arm64" ]]
then
HOST="aarch64-apple-darwin"
else
HOST="$arch-apple-darwin"
fi
CFLAGS="-arch $arch -miphoneos-version-min=6.0 -isysroot $SYSROOT $OLD_CFLAGS"
LDFLAGS="-arch $arch -miphoneos-version-min=6.0 -isysroot $SYSROOT $OLD_LDFLAGS"
# Build oniguruma for this architecture
cd onig-5.9.5
CC=$CC CFLAGS=$CFLAGS LDFLAGS=$LDFLAGS ./configure --disable-shared --enable-static --host=$HOST --prefix=$ORIG_PWD/build/ios/$arch
STATUS=$?
if [ $STATUS -ne 0 ]
then
echo "Failed to configure oniguruma for architecture $arch. Check `pwd`/config.log for details."
cd $PWD
exit $STATUS
fi
make clean
make $NJBOS install
STATUS=$?
if [ $STATUS -ne 0 ]
then
echo "Failed to make oniguruma for architecture $arch."
cd $PWD
exit $STATUS
fi
# Build jq for this architecture
cd $ORIG_PWD
CC=$CC CFLAGS=$CFLAGS LDFLAGS=$LDFLAGS ./configure --disable-shared --enable-static -host=$HOST --prefix=$ORIG_PWD/build/ios/$arch --with-oniguruma=$ORIG_PWD/build/ios/$arch
STATUS=$?
if [ $STATUS -ne 0 ]
then
echo "Failed to configure jq for architecture $arch"
exit $STATUS
fi
make clean
make $NJOBS install
STATUS=$?
if [ $STATUS -ne 0 ]
then
echo "Failed to make jq for architecture $arch"
exit $STATUS
fi
done
# lipo together the different architectures into a universal 'fat' file
lipo -create -output $ORIG_PWD/build/ios/libonig.a $ORIG_PWD/build/ios/{i386,x86_64,armv7,armv7s,arm64}/lib/libonig.a
lipo -create -output $ORIG_PWD/build/ios/libjq.a $ORIG_PWD/build/ios/{i386,x86_64,armv7,armv7s,arm64}/lib/libjq.a
# copy the products into the destination directory and clean up the single-architecture files.
cp $ORIG_PWD/build/ios/i386/include/*.h $ORIG_PWD/build/ios/
rm -rf $ORIG_PWD/build/ios/{i386,x86_64,armv7,armv7s,arm64}
echo "Products are in $ORIG_PWD/build/ios"
compile.c
+77 -559
View File
@@ -1,14 +1,12 @@
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // for strdup
#endif
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include "opcode.h"
#include "compile.h"
#include "bytecode.h"
#include "locfile.h"
#include "jv_alloc.h"
#include "linker.h"
/*
The intermediate representation for jq filters is as a sequence of
@@ -35,7 +33,6 @@ struct inst {
const struct cfunction* cfunc;
} imm;
struct locfile* locfile;
location source;
// Binding
@@ -51,9 +48,6 @@ struct inst {
struct inst* bound_by;
char* symbol;
int nformals;
int nactuals;
block subfn; // used by CLOSURE_CREATE (body of function)
block arglist; // used by CLOSURE_CREATE (formals) and CALL_JQ (arguments)
@@ -71,12 +65,9 @@ static inst* inst_new(opcode op) {
i->bytecode_pos = -1;
i->bound_by = 0;
i->symbol = 0;
i->nformals = -1;
i->nactuals = -1;
i->subfn = gen_noop();
i->arglist = gen_noop();
i->source = UNKNOWN_LOCATION;
i->locfile = 0;
return i;
}
@@ -84,8 +75,6 @@ static void inst_free(struct inst* i) {
jv_mem_free(i->symbol);
block_free(i->subfn);
block_free(i->arglist);
if (i->locfile)
locfile_free(i->locfile);
if (opcode_describe(i->op)->flags & OP_HAS_CONSTANT) {
jv_free(i->imm.constant);
}
@@ -97,7 +86,7 @@ static block inst_block(inst* i) {
return b;
}
int block_is_single(block b) {
static int block_is_single(block b) {
return b.first && b.first == b.last;
}
@@ -115,12 +104,11 @@ static inst* block_take(block* b) {
return i;
}
block gen_location(location loc, struct locfile* l, block b) {
block gen_location(location loc, block b) {
for (inst* i = b.first; i; i = i->next) {
if (i->source.start == UNKNOWN_LOCATION.start &&
i->source.end == UNKNOWN_LOCATION.end) {
i->source = loc;
i->locfile = locfile_retain(l);
}
}
return b;
@@ -131,10 +119,6 @@ block gen_noop() {
return b;
}
int block_is_noop(block b) {
return (b.first == 0 && b.last == 0);
}
block gen_op_simple(opcode op) {
assert(opcode_describe(op)->length == 1);
return inst_block(inst_new(op));
@@ -148,29 +132,6 @@ block gen_const(jv constant) {
return inst_block(i);
}
block gen_const_global(jv constant, const char *name) {
assert((opcode_describe(STORE_GLOBAL)->flags & (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING)) ==
(OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING));
inst* i = inst_new(STORE_GLOBAL);
i->imm.constant = constant;
i->symbol = strdup(name);
return inst_block(i);
}
int block_is_const(block b) {
return (block_is_single(b) && b.first->op == LOADK);
}
jv_kind block_const_kind(block b) {
assert(block_is_const(b));
return jv_get_kind(b.first->imm.constant);
}
jv block_const(block b) {
assert(block_is_const(b));
return jv_copy(b.first->imm.constant);
}
block gen_op_target(opcode op, block target) {
assert(opcode_describe(op)->flags & OP_HAS_BRANCH);
assert(target.last);
@@ -192,26 +153,33 @@ void inst_set_target(block b, block target) {
b.first->imm.target = target.last;
}
block gen_op_unbound(opcode op, const char* name) {
assert(opcode_describe(op)->flags & OP_HAS_BINDING);
block gen_op_var_unbound(opcode op, const char* name) {
assert(opcode_describe(op)->flags & OP_HAS_VARIABLE);
inst* i = inst_new(op);
i->symbol = strdup(name);
return inst_block(i);
}
block gen_op_var_fresh(opcode op, const char* name) {
assert(opcode_describe(op)->flags & OP_HAS_VARIABLE);
return block_bind(gen_op_unbound(op, name),
gen_noop(), OP_HAS_VARIABLE);
}
block gen_op_bound(opcode op, block binder) {
block gen_op_var_bound(opcode op, block binder) {
assert(block_is_single(binder));
block b = gen_op_unbound(op, binder.first->symbol);
block b = gen_op_var_unbound(op, binder.first->symbol);
b.first->bound_by = binder.first;
return b;
}
block gen_op_block_unbound(opcode op, const char* name) {
assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO);
inst* i = inst_new(op);
i->symbol = strdup(name);
return inst_block(i);
}
block gen_op_block_bound(opcode op, block binder) {
assert(block_is_single(binder));
block b = gen_op_block_unbound(op, binder.first->symbol);
b.first->bound_by = binder.first;
return b;
}
static void inst_join(inst* a, inst* b) {
assert(a && b);
@@ -238,122 +206,47 @@ block block_join(block a, block b) {
return c;
}
int block_has_only_binders_and_imports(block binders, int bindflags) {
bindflags |= OP_HAS_BINDING;
for (inst* curr = binders.first; curr; curr = curr->next) {
if ((opcode_describe(curr->op)->flags & bindflags) != bindflags && curr->op != DEPS && curr->op != MODULEMETA) {
return 0;
}
}
return 1;
}
static int inst_is_binder(inst *i, int bindflags) {
return !((opcode_describe(i->op)->flags & bindflags) != bindflags && i->op != MODULEMETA);
}
int block_has_only_binders(block binders, int bindflags) {
bindflags |= OP_HAS_BINDING;
bindflags &= ~OP_BIND_WILDCARD;
for (inst* curr = binders.first; curr; curr = curr->next) {
if ((opcode_describe(curr->op)->flags & bindflags) != bindflags && curr->op != MODULEMETA) {
if ((opcode_describe(curr->op)->flags & bindflags) != bindflags) {
return 0;
}
}
return 1;
}
// Count a binder's (function) formal params
static int block_count_formals(block b) {
int args = 0;
if (b.first->op == CLOSURE_CREATE_C)
return b.first->imm.cfunc->nargs - 1;
for (inst* i = b.first->arglist.first; i; i = i->next) {
assert(i->op == CLOSURE_PARAM);
args++;
}
return args;
}
// Count a call site's actual params
static int block_count_actuals(block b) {
int args = 0;
for (inst* i = b.first; i; i = i->next) {
switch (i->op) {
default: assert(0 && "Unknown function type"); break;
case CLOSURE_CREATE:
case CLOSURE_PARAM:
case CLOSURE_CREATE_C:
args++;
break;
}
}
return args;
}
static int block_count_refs(block binder, block body) {
int nrefs = 0;
for (inst* i = body.first; i; i = i->next) {
if (i != binder.first && i->bound_by == binder.first) {
nrefs++;
}
// counting recurses into closures
nrefs += block_count_refs(binder, i->subfn);
// counting recurses into argument list
nrefs += block_count_refs(binder, i->arglist);
}
return nrefs;
}
static int block_bind_subblock(block binder, block body, int bindflags, int break_distance) {
static int block_bind_subblock(block binder, block body, int bindflags) {
assert(block_is_single(binder));
assert((opcode_describe(binder.first->op)->flags & bindflags) == (bindflags & ~OP_BIND_WILDCARD));
assert((opcode_describe(binder.first->op)->flags & bindflags) == bindflags);
assert(binder.first->symbol);
assert(binder.first->bound_by == 0 || binder.first->bound_by == binder.first);
assert(break_distance >= 0);
binder.first->bound_by = binder.first;
if (binder.first->nformals == -1)
binder.first->nformals = block_count_formals(binder);
int nrefs = 0;
for (inst* i = body.first; i; i = i->next) {
int flags = opcode_describe(i->op)->flags;
if ((flags & bindflags) == (bindflags & ~OP_BIND_WILDCARD) && i->bound_by == 0 &&
(!strcmp(i->symbol, binder.first->symbol) ||
// Check for break/break2/break3; see parser.y
((bindflags & OP_BIND_WILDCARD) && i->symbol[0] == '*' &&
break_distance <= 3 && (i->symbol[1] == '1' + break_distance) &&
i->symbol[2] == '\0'))) {
if ((flags & bindflags) == bindflags &&
i->bound_by == 0 &&
!strcmp(i->symbol, binder.first->symbol)) {
// bind this instruction
if (i->op == CALL_JQ && i->nactuals == -1)
i->nactuals = block_count_actuals(i->arglist);
if (i->nactuals == -1 || i->nactuals == binder.first->nformals) {
i->bound_by = binder.first;
nrefs++;
}
} else if ((flags & bindflags) == (bindflags & ~OP_BIND_WILDCARD) && i->bound_by != 0 &&
!strncmp(binder.first->symbol, "*anonlabel", sizeof("*anonlabel") - 1) &&
!strncmp(i->symbol, "*anonlabel", sizeof("*anonlabel") - 1)) {
// Increment the break distance required for this binder to match
// a break whenever we come across a STOREV of *anonlabel...
break_distance++;
i->bound_by = binder.first;
nrefs++;
}
// binding recurses into closures
nrefs += block_bind_subblock(binder, i->subfn, bindflags, break_distance);
nrefs += block_bind_subblock(binder, i->subfn, bindflags);
// binding recurses into argument list
nrefs += block_bind_subblock(binder, i->arglist, bindflags, break_distance);
nrefs += block_bind_subblock(binder, i->arglist, bindflags);
}
return nrefs;
}
static int block_bind_each(block binder, block body, int bindflags) {
static void block_bind_each(block binder, block body, int bindflags) {
assert(block_has_only_binders(binder, bindflags));
bindflags |= OP_HAS_BINDING;
int nrefs = 0;
for (inst* curr = binder.first; curr; curr = curr->next) {
nrefs += block_bind_subblock(inst_block(curr), body, bindflags, 0);
block_bind_subblock(inst_block(curr), body, bindflags);
}
return nrefs;
}
block block_bind(block binder, block body, int bindflags) {
@@ -361,179 +254,38 @@ block block_bind(block binder, block body, int bindflags) {
return block_join(binder, body);
}
block block_bind_library(block binder, block body, int bindflags, const char* libname) {
bindflags |= OP_HAS_BINDING;
int nrefs = 0;
int matchlen = strlen(libname);
char* matchname = calloc(1,matchlen+2+1);
if (libname[0] != '\0') {
strcpy(matchname,libname);
strcpy(matchname+matchlen,"::");
matchlen += 2;
}
assert(block_has_only_binders(binder, bindflags));
for (inst *curr = binder.first; curr; curr = curr->next) {
int bindflags2 = bindflags;
char* cname = curr->symbol;
char* tname = malloc(strlen(curr->symbol)+matchlen+1);
strcpy(tname, matchname);
strcpy(tname+matchlen,cname);
// Ew
if ((opcode_describe(curr->op)->flags & (OP_HAS_VARIABLE | OP_HAS_CONSTANT)))
bindflags2 = OP_HAS_VARIABLE | OP_HAS_BINDING;
// This mutation is ugly, even if we undo it
curr->symbol = tname;
nrefs += block_bind_subblock(inst_block(curr), body, bindflags2, 0);
curr->symbol = cname;
free(tname);
}
free(matchname);
return body; // We don't return a join because we don't want those sticking around...
}
// Bind binder to body and throw away any defs in binder not referenced
// (directly or indirectly) from body.
block block_bind_referenced(block binder, block body, int bindflags) {
assert(block_has_only_binders(binder, bindflags));
bindflags |= OP_HAS_BINDING;
block refd = gen_noop();
block unrefd = gen_noop();
int nrefs;
for (int last_kept = 0, kept = 0; ; ) {
for (inst* curr; (curr = block_take(&binder));) {
block b = inst_block(curr);
nrefs = block_bind_each(b, body, bindflags);
// Check if this binder is referenced from any of the ones we
// already know are referenced by body.
nrefs += block_count_refs(b, refd);
nrefs += block_count_refs(b, body);
if (nrefs) {
refd = BLOCK(refd, b);
kept++;
} else {
unrefd = BLOCK(unrefd, b);
}
for (inst* curr; (curr = block_take(&binder));) {
block b = inst_block(curr);
if (block_bind_subblock(b, body, bindflags)) {
refd = BLOCK(refd, b);
} else {
block_free(b);
}
if (kept == last_kept)
break;
last_kept = kept;
binder = unrefd;
unrefd = gen_noop();
}
block_free(unrefd);
return block_join(refd, body);
}
block block_drop_unreferenced(block body) {
inst* curr;
block refd = gen_noop();
block unrefd = gen_noop();
int drop;
do {
drop = 0;
while ((curr = block_take(&body)) && curr->op != TOP) {
block b = inst_block(curr);
if (block_count_refs(b,refd) + block_count_refs(b,body) == 0) {
unrefd = BLOCK(unrefd, b);
drop++;
} else {
refd = BLOCK(refd, b);
}
}
if (curr && curr->op == TOP) {
body = BLOCK(inst_block(curr),body);
}
body = BLOCK(refd, body);
refd = gen_noop();
} while (drop != 0);
block_free(unrefd);
return body;
}
jv block_take_imports(block* body) {
jv imports = jv_array();
inst* top = NULL;
if (body->first && body->first->op == TOP) {
top = block_take(body);
}
while (body->first && (body->first->op == MODULEMETA || body->first->op == DEPS)) {
inst* dep = block_take(body);
if (dep->op == DEPS) {
imports = jv_array_append(imports, jv_copy(dep->imm.constant));
}
inst_free(dep);
}
if (top) {
*body = block_join(inst_block(top),*body);
}
return imports;
}
block gen_module(block metadata) {
inst* i = inst_new(MODULEMETA);
i->imm.constant = block_const(metadata);
if (jv_get_kind(i->imm.constant) != JV_KIND_OBJECT)
i->imm.constant = jv_object_set(jv_object(), jv_string("metadata"), i->imm.constant);
block_free(metadata);
return inst_block(i);
}
jv block_module_meta(block b) {
if (b.first != NULL && b.first->op == MODULEMETA)
return jv_copy(b.first->imm.constant);
return jv_null();
}
block gen_import(const char* name, block metadata, const char* as, int is_data) {
assert(metadata.first == NULL || block_is_const(metadata));
inst* i = inst_new(DEPS);
jv meta;
if (block_is_const(metadata))
meta = block_const(metadata);
else
meta = jv_object();
meta = jv_object_set(meta, jv_string("as"), jv_string(as));
meta = jv_object_set(meta, jv_string("is_data"), is_data ? jv_true() : jv_false());
meta = jv_object_set(meta, jv_string("relpath"), jv_string(name));
i->imm.constant = meta;
block_free(metadata);
return inst_block(i);
}
block gen_function(const char* name, block formals, block body) {
block_bind_each(formals, body, OP_IS_CALL_PSEUDO);
inst* i = inst_new(CLOSURE_CREATE);
for (inst* i = formals.last; i; i = i->prev) {
if (i->op == CLOSURE_PARAM_REGULAR) {
i->op = CLOSURE_PARAM;
body = gen_var_binding(gen_call(i->symbol, gen_noop()), i->symbol, body);
}
block_bind_subblock(inst_block(i), body, OP_IS_CALL_PSEUDO | OP_HAS_BINDING, 0);
}
i->subfn = body;
i->symbol = strdup(name);
i->arglist = formals;
block b = inst_block(i);
block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING, 0);
block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING);
return b;
}
block gen_param_regular(const char* name) {
return gen_op_unbound(CLOSURE_PARAM_REGULAR, name);
}
block gen_param(const char* name) {
return gen_op_unbound(CLOSURE_PARAM, name);
}
block gen_lambda(block body) {
return gen_function("@lambda", gen_noop(), body);
}
block gen_call(const char* name, block args) {
block b = gen_op_unbound(CALL_JQ, name);
block b = gen_op_block_unbound(CALL_JQ, name);
b.first->arglist = args;
return b;
}
@@ -552,137 +304,32 @@ block gen_both(block a, block b) {
return c;
}
block gen_const_object(block expr) {
int is_const = 1;
jv o = jv_object();
jv k = jv_null();
jv v = jv_null();
for (inst *i = expr.first; i; i = i->next) {
if (i->op != SUBEXP_BEGIN ||
i->next == NULL ||
i->next->op != LOADK ||
i->next->next == NULL ||
i->next->next->op != SUBEXP_END) {
is_const = 0;
break;
}
k = jv_copy(i->next->imm.constant);
i = i->next->next->next;
if (i == NULL ||
i->op != SUBEXP_BEGIN ||
i->next == NULL ||
i->next->op != LOADK ||
i->next->next == NULL ||
i->next->next->op != SUBEXP_END) {
is_const = 0;
break;
}
v = jv_copy(i->next->imm.constant);
i = i->next->next->next;
if (i == NULL || i->op != INSERT) {
is_const = 0;
break;
}
o = jv_object_set(o, k, v);
k = jv_null();
v = jv_null();
}
if (!is_const) {
jv_free(o);
jv_free(k);
jv_free(v);
block b = {0,0};
return b;
}
block_free(expr);
return gen_const(o);
}
static block gen_const_array(block expr) {
/*
* An expr of all constant elements looks like this:
*
* 0009 FORK 0027
* 0011 FORK 0023
* 0013 FORK 0019
* 0015 LOADK 1
* 0017 JUMP 0021
* 0019 LOADK 2
* 0021 JUMP 0025
* 0023 LOADK 3
* 0025 JUMP 0029
* 0027 LOADK 4
*
* That's: N-1 commas for N elements, N LOADKs, and a JUMP between
* every LOADK. The sequence ends in a LOADK. Any deviation and it's
* not a list of constants.
*
* Here we check for this pattern almost exactly. We don't check that
* the targets of the FORK and JUMP instructions are in the right
* sequence.
*/
int all_const = 1;
int commas = 0;
int normal = 1;
jv a = jv_array();
for (inst *i = expr.first; i; i = i->next) {
if (i->op == FORK) {
commas++;
if (i->imm.target == NULL || i->imm.target->op != JUMP ||
jv_array_length(jv_copy(a)) > 0) {
normal = 0;
break;
}
} else if (all_const && i->op == LOADK) {
if (i->next != NULL && i->next->op != JUMP) {
normal = 0;
break;
}
a = jv_array_append(a, jv_copy(i->imm.constant));
} else if (i->op != JUMP || i->imm.target == NULL ||
i->imm.target->op != LOADK) {
all_const = 0;
}
}
if (all_const && normal &&
(expr.last == NULL || expr.last->op == LOADK) &&
jv_array_length(jv_copy(a)) == commas + 1) {
block_free(expr);
return gen_const(a);
}
jv_free(a);
block b = {0,0};
return b;
}
block gen_collect(block expr) {
block const_array = gen_const_array(expr);
if (const_array.first != NULL)
return const_array;
block array_var = gen_op_var_fresh(STOREV, "collect");
block array_var = block_bind(gen_op_var_unbound(STOREV, "collect"),
gen_noop(), OP_HAS_VARIABLE);
block c = BLOCK(gen_op_simple(DUP), gen_const(jv_array()), array_var);
block tail = BLOCK(gen_op_bound(APPEND, array_var),
block tail = BLOCK(gen_op_var_bound(APPEND, array_var),
gen_op_simple(BACKTRACK));
return BLOCK(c,
gen_op_target(FORK, tail),
expr,
expr,
tail,
gen_op_bound(LOADVN, array_var));
gen_op_var_bound(LOADVN, array_var));
}
block gen_reduce(const char* varname, block source, block init, block body) {
block res_var = gen_op_var_fresh(STOREV, "reduce");
block loop = BLOCK(gen_op_simple(DUPN),
block res_var = block_bind(gen_op_var_unbound(STOREV, "reduce"),
gen_noop(), OP_HAS_VARIABLE);
block loop = BLOCK(gen_op_simple(DUP),
source,
block_bind(gen_op_unbound(STOREV, varname),
BLOCK(gen_op_bound(LOADVN, res_var),
block_bind(gen_op_var_unbound(STOREV, varname),
BLOCK(gen_op_var_bound(LOADVN, res_var),
body,
gen_op_bound(STOREV, res_var)),
gen_op_var_bound(STOREV, res_var)),
OP_HAS_VARIABLE),
gen_op_simple(BACKTRACK));
return BLOCK(gen_op_simple(DUP),
@@ -690,62 +337,19 @@ block gen_reduce(const char* varname, block source, block init, block body) {
res_var,
gen_op_target(FORK, loop),
loop,
gen_op_bound(LOADVN, res_var));
}
block gen_foreach(const char* varname, block source, block init, block update, block extract) {
block output = gen_op_targetlater(JUMP);
block state_var = gen_op_var_fresh(STOREV, "foreach");
block loop = BLOCK(gen_op_simple(DUPN),
// get a value from the source expression:
source,
// bind the $varname to that value for all the code in
// this block_bind() to see:
block_bind(gen_op_unbound(STOREV, varname),
// load the loop state variable
BLOCK(gen_op_bound(LOADVN, state_var),
// generate updated state
update,
// save the updated state for value extraction
gen_op_simple(DUP),
// save new state
gen_op_bound(STOREV, state_var),
// extract an output...
extract,
// ...and output it by jumping
// past the BACKTRACK that comes
// right after the loop body,
// which in turn is there
// because...
//
// (Incidentally, extract can also
// backtrack, e.g., if it calls
// empty, in which case we don't
// get here.)
output),
OP_HAS_VARIABLE));
block foreach = BLOCK(gen_op_simple(DUP),
init,
state_var,
gen_op_target(FORK, loop),
loop,
// ...at this point `foreach`'s original input
// will be on top of the stack, and we don't
// want to output it, so we backtrack.
gen_op_simple(BACKTRACK));
inst_set_target(output, foreach); // make that JUMP go bast the BACKTRACK at the end of the loop
return foreach;
gen_op_var_bound(LOADVN, res_var));
}
block gen_definedor(block a, block b) {
// var found := false
block found_var = gen_op_var_fresh(STOREV, "found");
block found_var = block_bind(gen_op_var_unbound(STOREV, "found"),
gen_noop(), OP_HAS_VARIABLE);
block init = BLOCK(gen_op_simple(DUP), gen_const(jv_false()), found_var);
// if found, backtrack. Otherwise execute b
block backtrack = gen_op_simple(BACKTRACK);
block tail = BLOCK(gen_op_simple(DUP),
gen_op_bound(LOADV, found_var),
gen_op_var_bound(LOADV, found_var),
gen_op_target(JUMP_F, backtrack),
backtrack,
gen_op_simple(POP),
@@ -757,7 +361,7 @@ block gen_definedor(block a, block b) {
// found := true, produce result
block if_found = BLOCK(gen_op_simple(DUP),
gen_const(jv_true()),
gen_op_bound(STOREV, found_var),
gen_op_var_bound(STOREV, found_var),
gen_op_target(JUMP, tail));
return BLOCK(init,
@@ -769,20 +373,6 @@ block gen_definedor(block a, block b) {
tail);
}
int block_has_main(block top) {
for (inst *c = top.first; c; c = c->next) {
if (c->op == TOP)
return 1;
}
return 0;
}
int block_is_funcdef(block b) {
if (b.first != NULL && b.first->op == CLOSURE_CREATE)
return 1;
return 0;
}
block gen_condbranch(block iftrue, block iffalse) {
iftrue = BLOCK(iftrue, gen_op_target(JUMP, iffalse));
return BLOCK(gen_op_target(JUMP_F, iftrue), iftrue, iffalse);
@@ -809,20 +399,9 @@ block gen_or(block a, block b) {
}
block gen_var_binding(block var, const char* name, block body) {
// var bindings can be added after coding the program; leave the TOP first.
block top = gen_noop();
if (body.first && body.first->op == TOP)
top = inst_block(block_take(&body));
return BLOCK(top, gen_op_simple(DUP), var,
block_bind(gen_op_unbound(STOREV, name),
body, OP_HAS_VARIABLE));
}
// Like gen_var_binding(), but bind `break`'s wildcard unbound variable
static block gen_wildvar_binding(block var, const char* name, block body) {
return BLOCK(gen_op_simple(DUP), var,
block_bind(gen_op_unbound(STOREV, name),
body, OP_HAS_VARIABLE | OP_BIND_WILDCARD));
block_bind(gen_op_var_unbound(STOREV, name),
body, OP_HAS_VARIABLE));
}
block gen_cond(block cond, block iftrue, block iffalse) {
@@ -831,64 +410,6 @@ block gen_cond(block cond, block iftrue, block iffalse) {
BLOCK(gen_op_simple(POP), iffalse)));
}
block gen_try_handler(block handler) {
// Quite a pain just to hide jq's internal errors.
return gen_cond(// `if type=="object" and .__jq
gen_and(gen_call("_equal",
BLOCK(gen_lambda(gen_const(jv_string("object"))),
gen_lambda(gen_noop()))),
BLOCK(gen_subexp(gen_const(jv_string("__jq"))),
gen_noop(),
gen_op_simple(INDEX))),
// `then error`
gen_call("error", gen_noop()),
// `else HANDLER end`
handler);
}
block gen_try(block exp, block handler) {
/*
* Produce something like:
* FORK_OPT <address of handler>
* <exp>
* JUMP <end of handler>
* <handler>
*
* If this is not an internal try/catch, then catch and re-raise
* internal errors to prevent them from leaking.
*
* The handler will only execute if we backtrack to the FORK_OPT with
* an error (exception). If <exp> produces no value then FORK_OPT
* will backtrack (propagate the `empty`, as it were. If <exp>
* produces a value then we'll execute whatever bytecode follows this
* sequence.
*/
if (!handler.first && !handler.last)
// A hack to deal with `.` as the handler; we could use a real NOOP here
handler = BLOCK(gen_op_simple(DUP), gen_op_simple(POP), handler);
exp = BLOCK(exp, gen_op_target(JUMP, handler));
return BLOCK(gen_op_target(FORK_OPT, exp), exp, handler);
}
block gen_label(const char *label, block exp) {
block cond = gen_call("_equal",
BLOCK(gen_lambda(gen_noop()),
gen_lambda(gen_op_unbound(LOADV, label))));
return gen_wildvar_binding(gen_op_simple(GENLABEL), label,
BLOCK(gen_op_simple(POP),
// try exp catch if . == $label
// then empty
// else error end
//
// Can't use gen_binop(), as that's firmly
// stuck in parser.y as it refers to things
// like EQ.
gen_try(exp,
gen_cond(cond,
gen_op_simple(BACKTRACK),
gen_call("error", gen_noop())))));
}
block gen_cbinding(const struct cfunction* cfunctions, int ncfunctions, block code) {
for (int cfunc=0; cfunc<ncfunctions; cfunc++) {
inst* i = inst_new(CLOSURE_CREATE_C);
@@ -921,16 +442,13 @@ static int count_cfunctions(block b) {
// Expands call instructions into a calling sequence
static int expand_call_arglist(block* b) {
static int expand_call_arglist(struct locfile* locations, block* b) {
int errors = 0;
block ret = gen_noop();
for (inst* curr; (curr = block_take(b));) {
if (opcode_describe(curr->op)->flags & OP_HAS_BINDING) {
if (!curr->bound_by) {
if (curr->symbol[0] == '*' && curr->symbol[1] >= '1' && curr->symbol[1] <= '3' && curr->symbol[2] == '\0')
locfile_locate(curr->locfile, curr->source, "jq: error: break used outside labeled control structure");
else
locfile_locate(curr->locfile, curr->source, "jq: error: %s/%d is not defined", curr->symbol, block_count_actuals(curr->arglist));
locfile_locate(locations, curr->source, "error: %s is not defined", curr->symbol);
errors++;
// don't process this instruction if it's not well-defined
ret = BLOCK(ret, inst_block(curr));
@@ -957,7 +475,7 @@ static int expand_call_arglist(block* b) {
break;
case CLOSURE_CREATE:
block_append(&prelude, b);
block_append(&callargs, gen_op_bound(CLOSURE_REF, b));
block_append(&callargs, gen_op_block_bound(CLOSURE_REF, b));
break;
}
actual_args++;
@@ -981,7 +499,7 @@ static int expand_call_arglist(block* b) {
i->subfn = gen_noop();
inst_free(i);
// arguments should be pushed in reverse order, prepend them to prelude
errors += expand_call_arglist(&body);
errors += expand_call_arglist(locations, &body);
prelude = BLOCK(gen_subexp(body), prelude);
actual_args++;
}
@@ -995,7 +513,14 @@ static int expand_call_arglist(block* b) {
}
}
assert(actual_args == desired_args); // because now handle this above
if (actual_args != desired_args) {
locfile_locate(locations, curr->source,
"error: %s arguments to %s (expected %d but got %d)",
actual_args > desired_args ? "too many" : "too few",
curr->symbol, desired_args, actual_args);
errors++;
}
}
ret = BLOCK(ret, prelude, inst_block(curr));
}
@@ -1003,12 +528,12 @@ static int expand_call_arglist(block* b) {
return errors;
}
static int compile(struct bytecode* bc, block b) {
static int compile(struct locfile* locations, struct bytecode* bc, block b) {
int errors = 0;
int pos = 0;
int var_frame_idx = 0;
bc->nsubfunctions = 0;
errors += expand_call_arglist(&b);
errors += expand_call_arglist(locations, &b);
b = BLOCK(b, gen_op_simple(RET));
jv localnames = jv_array();
for (inst* curr = b.first; curr; curr = curr->next) {
@@ -1064,7 +589,7 @@ static int compile(struct bytecode* bc, block b) {
params = jv_array_append(params, jv_string(param->symbol));
}
subfn->debuginfo = jv_object_set(subfn->debuginfo, jv_string("params"), params);
errors += compile(subfn, curr->subfn);
errors += compile(locations, subfn, curr->subfn);
curr->subfn = gen_noop();
}
}
@@ -1100,13 +625,6 @@ static int compile(struct bytecode* bc, block b) {
code[pos++] = nesting_level(bc, arg->bound_by);
code[pos++] = arg->bound_by->imm.intval | ARG_NEWCLOSURE;
}
} else if ((op->flags & OP_HAS_CONSTANT) && (op->flags & OP_HAS_VARIABLE)) {
// STORE_GLOBAL: constant global, basically
code[pos++] = jv_array_length(jv_copy(constant_pool));
constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));
code[pos++] = nesting_level(bc, curr->bound_by);
uint16_t var = (uint16_t)curr->bound_by->imm.intval;
code[pos++] = var;
} else if (op->flags & OP_HAS_CONSTANT) {
code[pos++] = jv_array_length(jv_copy(constant_pool));
constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));
@@ -1130,7 +648,7 @@ static int compile(struct bytecode* bc, block b) {
return errors;
}
int block_compile(block b, struct bytecode** out) {
int block_compile(block b, struct locfile* locations, struct bytecode** out) {
struct bytecode* bc = jv_mem_alloc(sizeof(struct bytecode));
bc->parent = 0;
bc->nclosures = 0;
@@ -1140,7 +658,7 @@ int block_compile(block b, struct bytecode** out) {
bc->globals->cfunctions = jv_mem_alloc(sizeof(struct cfunction) * ncfunc);
bc->globals->cfunc_names = jv_array();
bc->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_null());
int nerrors = compile(bc, b);
int nerrors = compile(locations, bc, b);
assert(bc->globals->ncfunctions == ncfunc);
if (nerrors > 0) {
bytecode_free(bc);
compile.h
+12 -32
View File
@@ -2,9 +2,13 @@
#define COMPILE_H
#include <stdint.h>
#include "jv.h"
#include "bytecode.h"
#include "opcode.h"
#include "locfile.h"
struct bytecode;
struct symbol_table;
struct cfunction;
struct inst;
typedef struct inst inst;
@@ -14,35 +18,23 @@ typedef struct block {
inst* last;
} block;
block gen_location(location, struct locfile*, block);
block gen_location(location, block);
block gen_noop();
int block_is_noop(block b);
block gen_op_simple(opcode op);
block gen_const(jv constant);
block gen_const_global(jv constant, const char *name);
int block_is_const(block b);
jv_kind block_const_kind(block b);
jv block_const(block b);
block gen_op_target(opcode op, block target);
block gen_op_unbound(opcode op, const char* name);
block gen_op_bound(opcode op, block binder);
block gen_op_var_fresh(opcode op, const char* name);
block gen_op_var_unbound(opcode op, const char* name);
block gen_op_var_bound(opcode op, block binder);
block gen_op_block_unbound(opcode op, const char* name);
block gen_module(block metadata);
jv block_module_meta(block b);
block gen_import(const char* name, block metadata, const char *as, int is_data);
block gen_function(const char* name, block formals, block body);
block gen_param_regular(const char* name);
block gen_param(const char* name);
block gen_lambda(block body);
block gen_call(const char* name, block body);
block gen_subexp(block a);
block gen_both(block a, block b);
block gen_const_object(block expr);
block gen_collect(block expr);
block gen_reduce(const char* varname, block source, block init, block body);
block gen_foreach(const char* varname, block source, block init, block update, block extract);
block gen_definedor(block a, block b);
block gen_condbranch(block iftrue, block iffalse);
block gen_and(block a, block b);
@@ -51,27 +43,16 @@ block gen_or(block a, block b);
block gen_var_binding(block var, const char* name, block body);
block gen_cond(block cond, block iftrue, block iffalse);
block gen_try_handler(block handler);
block gen_try(block exp, block handler);
block gen_label(const char *label, block exp);
block gen_cbinding(const struct cfunction* functions, int nfunctions, block b);
void block_append(block* b, block b2);
block block_join(block a, block b);
int block_has_only_binders_and_imports(block, int bindflags);
int block_has_only_binders(block, int bindflags);
int block_has_main(block);
int block_is_funcdef(block b);
int block_is_single(block b);
block block_bind(block binder, block body, int bindflags);
block block_bind_library(block binder, block body, int bindflags, const char* libname);
block block_bind_referenced(block binder, block body, int bindflags);
block block_drop_unreferenced(block body);
jv block_take_imports(block* body);
int block_compile(block, struct bytecode**);
int block_compile(block, struct locfile*, struct bytecode**);
void block_free(block);
@@ -87,11 +68,10 @@ void block_free(block);
#define BLOCK_5(b1,b2,b3,b4,b5) (block_join(BLOCK_4(b1,b2,b3,b4),(b5)))
#define BLOCK_6(b1,b2,b3,b4,b5,b6) (block_join(BLOCK_5(b1,b2,b3,b4,b5),(b6)))
#define BLOCK_7(b1,b2,b3,b4,b5,b6,b7) (block_join(BLOCK_6(b1,b2,b3,b4,b5,b6),(b7)))
#define BLOCK_8(b1,b2,b3,b4,b5,b6,b7,b8) (block_join(BLOCK_7(b1,b2,b3,b4,b5,b6,b7),(b8)))
#define BLOCK_IDX(_1,_2,_3,_4,_5,_6,_7,_8,NAME,...) NAME
#define BLOCK_IDX(_1,_2,_3,_4,_5,_6,_7,NAME,...) NAME
#define BLOCK(...) \
BLOCK_IDX(__VA_ARGS__, BLOCK_8, BLOCK_7, BLOCK_6, BLOCK_5, BLOCK_4, BLOCK_3, BLOCK_2, BLOCK_1)(__VA_ARGS__)
BLOCK_IDX(__VA_ARGS__, BLOCK_7, BLOCK_6, BLOCK_5, BLOCK_4, BLOCK_3, BLOCK_2, BLOCK_1)(__VA_ARGS__)
#endif
config.h.in
+32
View File
@@ -0,0 +1,32 @@
/* config.h.in. Generated from configure.ac by autoheader. */
/* Define to 1 if your C compiler doesn't accept -c and -o together. */
#undef NO_MINUS_C_MINUS_O
/* Name of package */
#undef PACKAGE
/* Define to the address where bug reports for this package should be sent. */
#undef PACKAGE_BUGREPORT
/* Define to the full name of this package. */
#undef PACKAGE_NAME
/* Define to the full name and version of this package. */
#undef PACKAGE_STRING
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
/* Define to the home page for this package. */
#undef PACKAGE_URL
/* Define to the version of this package. */
#undef PACKAGE_VERSION
/* Version number of package */
#undef VERSION
/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
`char[]'. */
#undef YYTEXT_POINTER
config/.gitignore
-10
View File
@@ -1,10 +0,0 @@
config.guess
config.sub
compile
depcomp
install-sh
ltmain.sh
missing
ylwrap
test-driver
ar-lib
config/compile
Executable
+343
View File
@@ -0,0 +1,343 @@
#! /bin/sh
# Wrapper for compilers which do not understand '-c -o'.
scriptversion=2012-03-05.13; # UTC
# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2009, 2010, 2012 Free
# Software Foundation, Inc.
# Written by Tom Tromey <tromey@cygnus.com>.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
# This file is maintained in Automake, please report
# bugs to <bug-automake@gnu.org> or send patches to
# <automake-patches@gnu.org>.
nl='
'
# We need space, tab and new line, in precisely that order. Quoting is
# there to prevent tools from complaining about whitespace usage.
IFS=" "" $nl"
file_conv=
# func_file_conv build_file lazy
# Convert a $build file to $host form and store it in $file
# Currently only supports Windows hosts. If the determined conversion
# type is listed in (the comma separated) LAZY, no conversion will
# take place.
func_file_conv ()
{
file=$1
case $file in
/ | /[!/]*) # absolute file, and not a UNC file
if test -z "$file_conv"; then
# lazily determine how to convert abs files
case `uname -s` in
MINGW*)
file_conv=mingw
;;
CYGWIN*)
file_conv=cygwin
;;
*)
file_conv=wine
;;
esac
fi
case $file_conv/,$2, in
*,$file_conv,*)
;;
mingw/*)
file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'`
;;
cygwin/*)
file=`cygpath -m "$file" || echo "$file"`
;;
wine/*)
file=`winepath -w "$file" || echo "$file"`
;;
esac
;;
esac
}
# func_cl_dashL linkdir
# Make cl look for libraries in LINKDIR
func_cl_dashL ()
{
func_file_conv "$1"
if test -z "$lib_path"; then
lib_path=$file
else
lib_path="$lib_path;$file"
fi
linker_opts="$linker_opts -LIBPATH:$file"
}
# func_cl_dashl library
# Do a library search-path lookup for cl
func_cl_dashl ()
{
lib=$1
found=no
save_IFS=$IFS
IFS=';'
for dir in $lib_path $LIB
do
IFS=$save_IFS
if $shared && test -f "$dir/$lib.dll.lib"; then
found=yes
lib=$dir/$lib.dll.lib
break
fi
if test -f "$dir/$lib.lib"; then
found=yes
lib=$dir/$lib.lib
break
fi
done
IFS=$save_IFS
if test "$found" != yes; then
lib=$lib.lib
fi
}
# func_cl_wrapper cl arg...
# Adjust compile command to suit cl
func_cl_wrapper ()
{
# Assume a capable shell
lib_path=
shared=:
linker_opts=
for arg
do
if test -n "$eat"; then
eat=
else
case $1 in
-o)
# configure might choose to run compile as 'compile cc -o foo foo.c'.
eat=1
case $2 in
*.o | *.[oO][bB][jJ])
func_file_conv "$2"
set x "$@" -Fo"$file"
shift
;;
*)
func_file_conv "$2"
set x "$@" -Fe"$file"
shift
;;
esac
;;
-I)
eat=1
func_file_conv "$2" mingw
set x "$@" -I"$file"
shift
;;
-I*)
func_file_conv "${1#-I}" mingw
set x "$@" -I"$file"
shift
;;
-l)
eat=1
func_cl_dashl "$2"
set x "$@" "$lib"
shift
;;
-l*)
func_cl_dashl "${1#-l}"
set x "$@" "$lib"
shift
;;
-L)
eat=1
func_cl_dashL "$2"
;;
-L*)
func_cl_dashL "${1#-L}"
;;
-static)
shared=false
;;
-Wl,*)
arg=${1#-Wl,}
save_ifs="$IFS"; IFS=','
for flag in $arg; do
IFS="$save_ifs"
linker_opts="$linker_opts $flag"
done
IFS="$save_ifs"
;;
-Xlinker)
eat=1
linker_opts="$linker_opts $2"
;;
-*)
set x "$@" "$1"
shift
;;
*.cc | *.CC | *.cxx | *.CXX | *.[cC]++)
func_file_conv "$1"
set x "$@" -Tp"$file"
shift
;;
*.c | *.cpp | *.CPP | *.lib | *.LIB | *.Lib | *.OBJ | *.obj | *.[oO])
func_file_conv "$1" mingw
set x "$@" "$file"
shift
;;
*)
set x "$@" "$1"
shift
;;
esac
fi
shift
done
if test -n "$linker_opts"; then
linker_opts="-link$linker_opts"
fi
exec "$@" $linker_opts
exit 1
}
eat=
case $1 in
'')
echo "$0: No command. Try '$0 --help' for more information." 1>&2
exit 1;
;;
-h | --h*)
cat <<\EOF
Usage: compile [--help] [--version] PROGRAM [ARGS]
Wrapper for compilers which do not understand '-c -o'.
Remove '-o dest.o' from ARGS, run PROGRAM with the remaining
arguments, and rename the output as expected.
If you are trying to build a whole package this is not the
right script to run: please start by reading the file 'INSTALL'.
Report bugs to <bug-automake@gnu.org>.
EOF
exit $?
;;
-v | --v*)
echo "compile $scriptversion"
exit $?
;;
cl | *[/\\]cl | cl.exe | *[/\\]cl.exe )
func_cl_wrapper "$@" # Doesn't return...
;;
esac
ofile=
cfile=
for arg
do
if test -n "$eat"; then
eat=
else
case $1 in
-o)
# configure might choose to run compile as 'compile cc -o foo foo.c'.
# So we strip '-o arg' only if arg is an object.
eat=1
case $2 in
*.o | *.obj)
ofile=$2
;;
*)
set x "$@" -o "$2"
shift
;;
esac
;;
*.c)
cfile=$1
set x "$@" "$1"
shift
;;
*)
set x "$@" "$1"
shift
;;
esac
fi
shift
done
if test -z "$ofile" || test -z "$cfile"; then
# If no '-o' option was seen then we might have been invoked from a
# pattern rule where we don't need one. That is ok -- this is a
# normal compilation that the losing compiler can handle. If no
# '.c' file was seen then we are probably linking. That is also
# ok.
exec "$@"
fi
# Name of file we expect compiler to create.
cofile=`echo "$cfile" | sed 's|^.*[\\/]||; s|^[a-zA-Z]:||; s/\.c$/.o/'`
# Create the lock directory.
# Note: use '[/\\:.-]' here to ensure that we don't use the same name
# that we are using for the .o file. Also, base the name on the expected
# object file name, since that is what matters with a parallel build.
lockdir=`echo "$cofile" | sed -e 's|[/\\:.-]|_|g'`.d
while true; do
if mkdir "$lockdir" >/dev/null 2>&1; then
break
fi
sleep 1
done
# FIXME: race condition here if user kills between mkdir and trap.
trap "rmdir '$lockdir'; exit 1" 1 2 15
# Run the compile.
"$@"
ret=$?
if test -f "$cofile"; then
test "$cofile" = "$ofile" || mv "$cofile" "$ofile"
elif test -f "${cofile}bj"; then
test "${cofile}bj" = "$ofile" || mv "${cofile}bj" "$ofile"
fi
rmdir "$lockdir"
exit $ret
# Local Variables:
# mode: shell-script
# sh-indentation: 2
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:
config/depcomp
Executable
+708
View File
@@ -0,0 +1,708 @@
#! /bin/sh
# depcomp - compile a program generating dependencies as side-effects
scriptversion=2012-03-27.16; # UTC
# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2006, 2007, 2009, 2010,
# 2011, 2012 Free Software Foundation, Inc.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
# Originally written by Alexandre Oliva <oliva@dcc.unicamp.br>.
case $1 in
'')
echo "$0: No command. Try '$0 --help' for more information." 1>&2
exit 1;
;;
-h | --h*)
cat <<\EOF
Usage: depcomp [--help] [--version] PROGRAM [ARGS]
Run PROGRAMS ARGS to compile a file, generating dependencies
as side-effects.
Environment variables:
depmode Dependency tracking mode.
source Source file read by 'PROGRAMS ARGS'.
object Object file output by 'PROGRAMS ARGS'.
DEPDIR directory where to store dependencies.
depfile Dependency file to output.
tmpdepfile Temporary file to use when outputting dependencies.
libtool Whether libtool is used (yes/no).
Report bugs to <bug-automake@gnu.org>.
EOF
exit $?
;;
-v | --v*)
echo "depcomp $scriptversion"
exit $?
;;
esac
# A tabulation character.
tab=' '
# A newline character.
nl='
'
if test -z "$depmode" || test -z "$source" || test -z "$object"; then
echo "depcomp: Variables source, object and depmode must be set" 1>&2
exit 1
fi
# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po.
depfile=${depfile-`echo "$object" |
sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`}
tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`}
rm -f "$tmpdepfile"
# Some modes work just like other modes, but use different flags. We
# parameterize here, but still list the modes in the big case below,
# to make depend.m4 easier to write. Note that we *cannot* use a case
# here, because this file can only contain one case statement.
if test "$depmode" = hp; then
# HP compiler uses -M and no extra arg.
gccflag=-M
depmode=gcc
fi
if test "$depmode" = dashXmstdout; then
# This is just like dashmstdout with a different argument.
dashmflag=-xM
depmode=dashmstdout
fi
cygpath_u="cygpath -u -f -"
if test "$depmode" = msvcmsys; then
# This is just like msvisualcpp but w/o cygpath translation.
# Just convert the backslash-escaped backslashes to single forward
# slashes to satisfy depend.m4
cygpath_u='sed s,\\\\,/,g'
depmode=msvisualcpp
fi
if test "$depmode" = msvc7msys; then
# This is just like msvc7 but w/o cygpath translation.
# Just convert the backslash-escaped backslashes to single forward
# slashes to satisfy depend.m4
cygpath_u='sed s,\\\\,/,g'
depmode=msvc7
fi
if test "$depmode" = xlc; then
# IBM C/C++ Compilers xlc/xlC can output gcc-like dependency informations.
gccflag=-qmakedep=gcc,-MF
depmode=gcc
fi
case "$depmode" in
gcc3)
## gcc 3 implements dependency tracking that does exactly what
## we want. Yay! Note: for some reason libtool 1.4 doesn't like
## it if -MD -MP comes after the -MF stuff. Hmm.
## Unfortunately, FreeBSD c89 acceptance of flags depends upon
## the command line argument order; so add the flags where they
## appear in depend2.am. Note that the slowdown incurred here
## affects only configure: in makefiles, %FASTDEP% shortcuts this.
for arg
do
case $arg in
-c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;;
*) set fnord "$@" "$arg" ;;
esac
shift # fnord
shift # $arg
done
"$@"
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
mv "$tmpdepfile" "$depfile"
;;
gcc)
## There are various ways to get dependency output from gcc. Here's
## why we pick this rather obscure method:
## - Don't want to use -MD because we'd like the dependencies to end
## up in a subdir. Having to rename by hand is ugly.
## (We might end up doing this anyway to support other compilers.)
## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like
## -MM, not -M (despite what the docs say).
## - Using -M directly means running the compiler twice (even worse
## than renaming).
if test -z "$gccflag"; then
gccflag=-MD,
fi
"$@" -Wp,"$gccflag$tmpdepfile"
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
rm -f "$depfile"
echo "$object : \\" > "$depfile"
alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
## The second -e expression handles DOS-style file names with drive letters.
sed -e 's/^[^:]*: / /' \
-e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile"
## This next piece of magic avoids the "deleted header file" problem.
## The problem is that when a header file which appears in a .P file
## is deleted, the dependency causes make to die (because there is
## typically no way to rebuild the header). We avoid this by adding
## dummy dependencies for each header file. Too bad gcc doesn't do
## this for us directly.
tr ' ' "$nl" < "$tmpdepfile" |
## Some versions of gcc put a space before the ':'. On the theory
## that the space means something, we add a space to the output as
## well. hp depmode also adds that space, but also prefixes the VPATH
## to the object. Take care to not repeat it in the output.
## Some versions of the HPUX 10.20 sed can't process this invocation
## correctly. Breaking it into two sed invocations is a workaround.
sed -e 's/^\\$//' -e '/^$/d' -e "s|.*$object$||" -e '/:$/d' \
| sed -e 's/$/ :/' >> "$depfile"
rm -f "$tmpdepfile"
;;
hp)
# This case exists only to let depend.m4 do its work. It works by
# looking at the text of this script. This case will never be run,
# since it is checked for above.
exit 1
;;
sgi)
if test "$libtool" = yes; then
"$@" "-Wp,-MDupdate,$tmpdepfile"
else
"$@" -MDupdate "$tmpdepfile"
fi
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
rm -f "$depfile"
if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files
echo "$object : \\" > "$depfile"
# Clip off the initial element (the dependent). Don't try to be
# clever and replace this with sed code, as IRIX sed won't handle
# lines with more than a fixed number of characters (4096 in
# IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines;
# the IRIX cc adds comments like '#:fec' to the end of the
# dependency line.
tr ' ' "$nl" < "$tmpdepfile" \
| sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \
tr "$nl" ' ' >> "$depfile"
echo >> "$depfile"
# The second pass generates a dummy entry for each header file.
tr ' ' "$nl" < "$tmpdepfile" \
| sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \
>> "$depfile"
else
# The sourcefile does not contain any dependencies, so just
# store a dummy comment line, to avoid errors with the Makefile
# "include basename.Plo" scheme.
echo "#dummy" > "$depfile"
fi
rm -f "$tmpdepfile"
;;
xlc)
# This case exists only to let depend.m4 do its work. It works by
# looking at the text of this script. This case will never be run,
# since it is checked for above.
exit 1
;;
aix)
# The C for AIX Compiler uses -M and outputs the dependencies
# in a .u file. In older versions, this file always lives in the
# current directory. Also, the AIX compiler puts '$object:' at the
# start of each line; $object doesn't have directory information.
# Version 6 uses the directory in both cases.
dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
test "x$dir" = "x$object" && dir=
base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
if test "$libtool" = yes; then
tmpdepfile1=$dir$base.u
tmpdepfile2=$base.u
tmpdepfile3=$dir.libs/$base.u
"$@" -Wc,-M
else
tmpdepfile1=$dir$base.u
tmpdepfile2=$dir$base.u
tmpdepfile3=$dir$base.u
"$@" -M
fi
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
exit $stat
fi
for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
do
test -f "$tmpdepfile" && break
done
if test -f "$tmpdepfile"; then
# Each line is of the form 'foo.o: dependent.h'.
# Do two passes, one to just change these to
# '$object: dependent.h' and one to simply 'dependent.h:'.
sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
else
# The sourcefile does not contain any dependencies, so just
# store a dummy comment line, to avoid errors with the Makefile
# "include basename.Plo" scheme.
echo "#dummy" > "$depfile"
fi
rm -f "$tmpdepfile"
;;
icc)
# Intel's C compiler anf tcc (Tiny C Compiler) understand '-MD -MF file'.
# However on
# $CC -MD -MF foo.d -c -o sub/foo.o sub/foo.c
# ICC 7.0 will fill foo.d with something like
# foo.o: sub/foo.c
# foo.o: sub/foo.h
# which is wrong. We want
# sub/foo.o: sub/foo.c
# sub/foo.o: sub/foo.h
# sub/foo.c:
# sub/foo.h:
# ICC 7.1 will output
# foo.o: sub/foo.c sub/foo.h
# and will wrap long lines using '\':
# foo.o: sub/foo.c ... \
# sub/foo.h ... \
# ...
# tcc 0.9.26 (FIXME still under development at the moment of writing)
# will emit a similar output, but also prepend the continuation lines
# with horizontal tabulation characters.
"$@" -MD -MF "$tmpdepfile"
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
rm -f "$depfile"
# Each line is of the form 'foo.o: dependent.h',
# or 'foo.o: dep1.h dep2.h \', or ' dep3.h dep4.h \'.
# Do two passes, one to just change these to
# '$object: dependent.h' and one to simply 'dependent.h:'.
sed -e "s/^[ $tab][ $tab]*/ /" -e "s,^[^:]*:,$object :," \
< "$tmpdepfile" > "$depfile"
sed '
s/[ '"$tab"'][ '"$tab"']*/ /g
s/^ *//
s/ *\\*$//
s/^[^:]*: *//
/^$/d
/:$/d
s/$/ :/
' < "$tmpdepfile" >> "$depfile"
rm -f "$tmpdepfile"
;;
hp2)
# The "hp" stanza above does not work with aCC (C++) and HP's ia64
# compilers, which have integrated preprocessors. The correct option
# to use with these is +Maked; it writes dependencies to a file named
# 'foo.d', which lands next to the object file, wherever that
# happens to be.
# Much of this is similar to the tru64 case; see comments there.
dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
test "x$dir" = "x$object" && dir=
base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
if test "$libtool" = yes; then
tmpdepfile1=$dir$base.d
tmpdepfile2=$dir.libs/$base.d
"$@" -Wc,+Maked
else
tmpdepfile1=$dir$base.d
tmpdepfile2=$dir$base.d
"$@" +Maked
fi
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile1" "$tmpdepfile2"
exit $stat
fi
for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2"
do
test -f "$tmpdepfile" && break
done
if test -f "$tmpdepfile"; then
sed -e "s,^.*\.[a-z]*:,$object:," "$tmpdepfile" > "$depfile"
# Add 'dependent.h:' lines.
sed -ne '2,${
s/^ *//
s/ \\*$//
s/$/:/
p
}' "$tmpdepfile" >> "$depfile"
else
echo "#dummy" > "$depfile"
fi
rm -f "$tmpdepfile" "$tmpdepfile2"
;;
tru64)
# The Tru64 compiler uses -MD to generate dependencies as a side
# effect. 'cc -MD -o foo.o ...' puts the dependencies into 'foo.o.d'.
# At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put
# dependencies in 'foo.d' instead, so we check for that too.
# Subdirectories are respected.
dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
test "x$dir" = "x$object" && dir=
base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
if test "$libtool" = yes; then
# With Tru64 cc, shared objects can also be used to make a
# static library. This mechanism is used in libtool 1.4 series to
# handle both shared and static libraries in a single compilation.
# With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d.
#
# With libtool 1.5 this exception was removed, and libtool now
# generates 2 separate objects for the 2 libraries. These two
# compilations output dependencies in $dir.libs/$base.o.d and
# in $dir$base.o.d. We have to check for both files, because
# one of the two compilations can be disabled. We should prefer
# $dir$base.o.d over $dir.libs/$base.o.d because the latter is
# automatically cleaned when .libs/ is deleted, while ignoring
# the former would cause a distcleancheck panic.
tmpdepfile1=$dir.libs/$base.lo.d # libtool 1.4
tmpdepfile2=$dir$base.o.d # libtool 1.5
tmpdepfile3=$dir.libs/$base.o.d # libtool 1.5
tmpdepfile4=$dir.libs/$base.d # Compaq CCC V6.2-504
"$@" -Wc,-MD
else
tmpdepfile1=$dir$base.o.d
tmpdepfile2=$dir$base.d
tmpdepfile3=$dir$base.d
tmpdepfile4=$dir$base.d
"$@" -MD
fi
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
exit $stat
fi
for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
do
test -f "$tmpdepfile" && break
done
if test -f "$tmpdepfile"; then
sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
else
echo "#dummy" > "$depfile"
fi
rm -f "$tmpdepfile"
;;
msvc7)
if test "$libtool" = yes; then
showIncludes=-Wc,-showIncludes
else
showIncludes=-showIncludes
fi
"$@" $showIncludes > "$tmpdepfile"
stat=$?
grep -v '^Note: including file: ' "$tmpdepfile"
if test "$stat" = 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
rm -f "$depfile"
echo "$object : \\" > "$depfile"
# The first sed program below extracts the file names and escapes
# backslashes for cygpath. The second sed program outputs the file
# name when reading, but also accumulates all include files in the
# hold buffer in order to output them again at the end. This only
# works with sed implementations that can handle large buffers.
sed < "$tmpdepfile" -n '
/^Note: including file: *\(.*\)/ {
s//\1/
s/\\/\\\\/g
p
}' | $cygpath_u | sort -u | sed -n '
s/ /\\ /g
s/\(.*\)/'"$tab"'\1 \\/p
s/.\(.*\) \\/\1:/
H
$ {
s/.*/'"$tab"'/
G
p
}' >> "$depfile"
rm -f "$tmpdepfile"
;;
msvc7msys)
# This case exists only to let depend.m4 do its work. It works by
# looking at the text of this script. This case will never be run,
# since it is checked for above.
exit 1
;;
#nosideeffect)
# This comment above is used by automake to tell side-effect
# dependency tracking mechanisms from slower ones.
dashmstdout)
# Important note: in order to support this mode, a compiler *must*
# always write the preprocessed file to stdout, regardless of -o.
"$@" || exit $?
# Remove the call to Libtool.
if test "$libtool" = yes; then
while test "X$1" != 'X--mode=compile'; do
shift
done
shift
fi
# Remove '-o $object'.
IFS=" "
for arg
do
case $arg in
-o)
shift
;;
$object)
shift
;;
*)
set fnord "$@" "$arg"
shift # fnord
shift # $arg
;;
esac
done
test -z "$dashmflag" && dashmflag=-M
# Require at least two characters before searching for ':'
# in the target name. This is to cope with DOS-style filenames:
# a dependency such as 'c:/foo/bar' could be seen as target 'c' otherwise.
"$@" $dashmflag |
sed 's:^['"$tab"' ]*[^:'"$tab"' ][^:][^:]*\:['"$tab"' ]*:'"$object"'\: :' > "$tmpdepfile"
rm -f "$depfile"
cat < "$tmpdepfile" > "$depfile"
tr ' ' "$nl" < "$tmpdepfile" | \
## Some versions of the HPUX 10.20 sed can't process this invocation
## correctly. Breaking it into two sed invocations is a workaround.
sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
rm -f "$tmpdepfile"
;;
dashXmstdout)
# This case only exists to satisfy depend.m4. It is never actually
# run, as this mode is specially recognized in the preamble.
exit 1
;;
makedepend)
"$@" || exit $?
# Remove any Libtool call
if test "$libtool" = yes; then
while test "X$1" != 'X--mode=compile'; do
shift
done
shift
fi
# X makedepend
shift
cleared=no eat=no
for arg
do
case $cleared in
no)
set ""; shift
cleared=yes ;;
esac
if test $eat = yes; then
eat=no
continue
fi
case "$arg" in
-D*|-I*)
set fnord "$@" "$arg"; shift ;;
# Strip any option that makedepend may not understand. Remove
# the object too, otherwise makedepend will parse it as a source file.
-arch)
eat=yes ;;
-*|$object)
;;
*)
set fnord "$@" "$arg"; shift ;;
esac
done
obj_suffix=`echo "$object" | sed 's/^.*\././'`
touch "$tmpdepfile"
${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@"
rm -f "$depfile"
# makedepend may prepend the VPATH from the source file name to the object.
# No need to regex-escape $object, excess matching of '.' is harmless.
sed "s|^.*\($object *:\)|\1|" "$tmpdepfile" > "$depfile"
sed '1,2d' "$tmpdepfile" | tr ' ' "$nl" | \
## Some versions of the HPUX 10.20 sed can't process this invocation
## correctly. Breaking it into two sed invocations is a workaround.
sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
rm -f "$tmpdepfile" "$tmpdepfile".bak
;;
cpp)
# Important note: in order to support this mode, a compiler *must*
# always write the preprocessed file to stdout.
"$@" || exit $?
# Remove the call to Libtool.
if test "$libtool" = yes; then
while test "X$1" != 'X--mode=compile'; do
shift
done
shift
fi
# Remove '-o $object'.
IFS=" "
for arg
do
case $arg in
-o)
shift
;;
$object)
shift
;;
*)
set fnord "$@" "$arg"
shift # fnord
shift # $arg
;;
esac
done
"$@" -E |
sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \
-e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' |
sed '$ s: \\$::' > "$tmpdepfile"
rm -f "$depfile"
echo "$object : \\" > "$depfile"
cat < "$tmpdepfile" >> "$depfile"
sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile"
rm -f "$tmpdepfile"
;;
msvisualcpp)
# Important note: in order to support this mode, a compiler *must*
# always write the preprocessed file to stdout.
"$@" || exit $?
# Remove the call to Libtool.
if test "$libtool" = yes; then
while test "X$1" != 'X--mode=compile'; do
shift
done
shift
fi
IFS=" "
for arg
do
case "$arg" in
-o)
shift
;;
$object)
shift
;;
"-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI")
set fnord "$@"
shift
shift
;;
*)
set fnord "$@" "$arg"
shift
shift
;;
esac
done
"$@" -E 2>/dev/null |
sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile"
rm -f "$depfile"
echo "$object : \\" > "$depfile"
sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::'"$tab"'\1 \\:p' >> "$depfile"
echo "$tab" >> "$depfile"
sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile"
rm -f "$tmpdepfile"
;;
msvcmsys)
# This case exists only to let depend.m4 do its work. It works by
# looking at the text of this script. This case will never be run,
# since it is checked for above.
exit 1
;;
none)
exec "$@"
;;
*)
echo "Unknown depmode $depmode" 1>&2
exit 1
;;
esac
exit 0
# Local Variables:
# mode: shell-script
# sh-indentation: 2
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:
config/install-sh
Executable
+527
View File
@@ -0,0 +1,527 @@
#!/bin/sh
# install - install a program, script, or datafile
scriptversion=2011-01-19.21; # UTC
# This originates from X11R5 (mit/util/scripts/install.sh), which was
# later released in X11R6 (xc/config/util/install.sh) with the
# following copyright and license.
#
# Copyright (C) 1994 X Consortium
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# Except as contained in this notice, the name of the X Consortium shall not
# be used in advertising or otherwise to promote the sale, use or other deal-
# ings in this Software without prior written authorization from the X Consor-
# tium.
#
#
# FSF changes to this file are in the public domain.
#
# Calling this script install-sh is preferred over install.sh, to prevent
# `make' implicit rules from creating a file called install from it
# when there is no Makefile.
#
# This script is compatible with the BSD install script, but was written
# from scratch.
nl='
'
IFS=" "" $nl"
# set DOITPROG to echo to test this script
# Don't use :- since 4.3BSD and earlier shells don't like it.
doit=${DOITPROG-}
if test -z "$doit"; then
doit_exec=exec
else
doit_exec=$doit
fi
# Put in absolute file names if you don't have them in your path;
# or use environment vars.
chgrpprog=${CHGRPPROG-chgrp}
chmodprog=${CHMODPROG-chmod}
chownprog=${CHOWNPROG-chown}
cmpprog=${CMPPROG-cmp}
cpprog=${CPPROG-cp}
mkdirprog=${MKDIRPROG-mkdir}
mvprog=${MVPROG-mv}
rmprog=${RMPROG-rm}
stripprog=${STRIPPROG-strip}
posix_glob='?'
initialize_posix_glob='
test "$posix_glob" != "?" || {
if (set -f) 2>/dev/null; then
posix_glob=
else
posix_glob=:
fi
}
'
posix_mkdir=
# Desired mode of installed file.
mode=0755
chgrpcmd=
chmodcmd=$chmodprog
chowncmd=
mvcmd=$mvprog
rmcmd="$rmprog -f"
stripcmd=
src=
dst=
dir_arg=
dst_arg=
copy_on_change=false
no_target_directory=
usage="\
Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
or: $0 [OPTION]... SRCFILES... DIRECTORY
or: $0 [OPTION]... -t DIRECTORY SRCFILES...
or: $0 [OPTION]... -d DIRECTORIES...
In the 1st form, copy SRCFILE to DSTFILE.
In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
In the 4th, create DIRECTORIES.
Options:
--help display this help and exit.
--version display version info and exit.
-c (ignored)
-C install only if different (preserve the last data modification time)
-d create directories instead of installing files.
-g GROUP $chgrpprog installed files to GROUP.
-m MODE $chmodprog installed files to MODE.
-o USER $chownprog installed files to USER.
-s $stripprog installed files.
-t DIRECTORY install into DIRECTORY.
-T report an error if DSTFILE is a directory.
Environment variables override the default commands:
CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
RMPROG STRIPPROG
"
while test $# -ne 0; do
case $1 in
-c) ;;
-C) copy_on_change=true;;
-d) dir_arg=true;;
-g) chgrpcmd="$chgrpprog $2"
shift;;
--help) echo "$usage"; exit $?;;
-m) mode=$2
case $mode in
*' '* | *' '* | *'
'* | *'*'* | *'?'* | *'['*)
echo "$0: invalid mode: $mode" >&2
exit 1;;
esac
shift;;
-o) chowncmd="$chownprog $2"
shift;;
-s) stripcmd=$stripprog;;
-t) dst_arg=$2
# Protect names problematic for `test' and other utilities.
case $dst_arg in
-* | [=\(\)!]) dst_arg=./$dst_arg;;
esac
shift;;
-T) no_target_directory=true;;
--version) echo "$0 $scriptversion"; exit $?;;
--) shift
break;;
-*) echo "$0: invalid option: $1" >&2
exit 1;;
*) break;;
esac
shift
done
if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
# When -d is used, all remaining arguments are directories to create.
# When -t is used, the destination is already specified.
# Otherwise, the last argument is the destination. Remove it from $@.
for arg
do
if test -n "$dst_arg"; then
# $@ is not empty: it contains at least $arg.
set fnord "$@" "$dst_arg"
shift # fnord
fi
shift # arg
dst_arg=$arg
# Protect names problematic for `test' and other utilities.
case $dst_arg in
-* | [=\(\)!]) dst_arg=./$dst_arg;;
esac
done
fi
if test $# -eq 0; then
if test -z "$dir_arg"; then
echo "$0: no input file specified." >&2
exit 1
fi
# It's OK to call `install-sh -d' without argument.
# This can happen when creating conditional directories.
exit 0
fi
if test -z "$dir_arg"; then
do_exit='(exit $ret); exit $ret'
trap "ret=129; $do_exit" 1
trap "ret=130; $do_exit" 2
trap "ret=141; $do_exit" 13
trap "ret=143; $do_exit" 15
# Set umask so as not to create temps with too-generous modes.
# However, 'strip' requires both read and write access to temps.
case $mode in
# Optimize common cases.
*644) cp_umask=133;;
*755) cp_umask=22;;
*[0-7])
if test -z "$stripcmd"; then
u_plus_rw=
else
u_plus_rw='% 200'
fi
cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
*)
if test -z "$stripcmd"; then
u_plus_rw=
else
u_plus_rw=,u+rw
fi
cp_umask=$mode$u_plus_rw;;
esac
fi
for src
do
# Protect names problematic for `test' and other utilities.
case $src in
-* | [=\(\)!]) src=./$src;;
esac
if test -n "$dir_arg"; then
dst=$src
dstdir=$dst
test -d "$dstdir"
dstdir_status=$?
else
# Waiting for this to be detected by the "$cpprog $src $dsttmp" command
# might cause directories to be created, which would be especially bad
# if $src (and thus $dsttmp) contains '*'.
if test ! -f "$src" && test ! -d "$src"; then
echo "$0: $src does not exist." >&2
exit 1
fi
if test -z "$dst_arg"; then
echo "$0: no destination specified." >&2
exit 1
fi
dst=$dst_arg
# If destination is a directory, append the input filename; won't work
# if double slashes aren't ignored.
if test -d "$dst"; then
if test -n "$no_target_directory"; then
echo "$0: $dst_arg: Is a directory" >&2
exit 1
fi
dstdir=$dst
dst=$dstdir/`basename "$src"`
dstdir_status=0
else
# Prefer dirname, but fall back on a substitute if dirname fails.
dstdir=`
(dirname "$dst") 2>/dev/null ||
expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
X"$dst" : 'X\(//\)[^/]' \| \
X"$dst" : 'X\(//\)$' \| \
X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
echo X"$dst" |
sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
s//\1/
q
}
/^X\(\/\/\)[^/].*/{
s//\1/
q
}
/^X\(\/\/\)$/{
s//\1/
q
}
/^X\(\/\).*/{
s//\1/
q
}
s/.*/./; q'
`
test -d "$dstdir"
dstdir_status=$?
fi
fi
obsolete_mkdir_used=false
if test $dstdir_status != 0; then
case $posix_mkdir in
'')
# Create intermediate dirs using mode 755 as modified by the umask.
# This is like FreeBSD 'install' as of 1997-10-28.
umask=`umask`
case $stripcmd.$umask in
# Optimize common cases.
*[2367][2367]) mkdir_umask=$umask;;
.*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
*[0-7])
mkdir_umask=`expr $umask + 22 \
- $umask % 100 % 40 + $umask % 20 \
- $umask % 10 % 4 + $umask % 2
`;;
*) mkdir_umask=$umask,go-w;;
esac
# With -d, create the new directory with the user-specified mode.
# Otherwise, rely on $mkdir_umask.
if test -n "$dir_arg"; then
mkdir_mode=-m$mode
else
mkdir_mode=
fi
posix_mkdir=false
case $umask in
*[123567][0-7][0-7])
# POSIX mkdir -p sets u+wx bits regardless of umask, which
# is incompatible with FreeBSD 'install' when (umask & 300) != 0.
;;
*)
tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
if (umask $mkdir_umask &&
exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
then
if test -z "$dir_arg" || {
# Check for POSIX incompatibilities with -m.
# HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
# other-writeable bit of parent directory when it shouldn't.
# FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
ls_ld_tmpdir=`ls -ld "$tmpdir"`
case $ls_ld_tmpdir in
d????-?r-*) different_mode=700;;
d????-?--*) different_mode=755;;
*) false;;
esac &&
$mkdirprog -m$different_mode -p -- "$tmpdir" && {
ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
}
}
then posix_mkdir=:
fi
rmdir "$tmpdir/d" "$tmpdir"
else
# Remove any dirs left behind by ancient mkdir implementations.
rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
fi
trap '' 0;;
esac;;
esac
if
$posix_mkdir && (
umask $mkdir_umask &&
$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
)
then :
else
# The umask is ridiculous, or mkdir does not conform to POSIX,
# or it failed possibly due to a race condition. Create the
# directory the slow way, step by step, checking for races as we go.
case $dstdir in
/*) prefix='/';;
[-=\(\)!]*) prefix='./';;
*) prefix='';;
esac
eval "$initialize_posix_glob"
oIFS=$IFS
IFS=/
$posix_glob set -f
set fnord $dstdir
shift
$posix_glob set +f
IFS=$oIFS
prefixes=
for d
do
test X"$d" = X && continue
prefix=$prefix$d
if test -d "$prefix"; then
prefixes=
else
if $posix_mkdir; then
(umask=$mkdir_umask &&
$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
# Don't fail if two instances are running concurrently.
test -d "$prefix" || exit 1
else
case $prefix in
*\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
*) qprefix=$prefix;;
esac
prefixes="$prefixes '$qprefix'"
fi
fi
prefix=$prefix/
done
if test -n "$prefixes"; then
# Don't fail if two instances are running concurrently.
(umask $mkdir_umask &&
eval "\$doit_exec \$mkdirprog $prefixes") ||
test -d "$dstdir" || exit 1
obsolete_mkdir_used=true
fi
fi
fi
if test -n "$dir_arg"; then
{ test -z "$chowncmd" || $doit $chowncmd "$dst"; } &&
{ test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } &&
{ test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false ||
test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1
else
# Make a couple of temp file names in the proper directory.
dsttmp=$dstdir/_inst.$$_
rmtmp=$dstdir/_rm.$$_
# Trap to clean up those temp files at exit.
trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
# Copy the file name to the temp name.
(umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
# and set any options; do chmod last to preserve setuid bits.
#
# If any of these fail, we abort the whole thing. If we want to
# ignore errors from any of these, just make sure not to ignore
# errors from the above "$doit $cpprog $src $dsttmp" command.
#
{ test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
{ test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
{ test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
{ test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
# If -C, don't bother to copy if it wouldn't change the file.
if $copy_on_change &&
old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` &&
new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` &&
eval "$initialize_posix_glob" &&
$posix_glob set -f &&
set X $old && old=:$2:$4:$5:$6 &&
set X $new && new=:$2:$4:$5:$6 &&
$posix_glob set +f &&
test "$old" = "$new" &&
$cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
then
rm -f "$dsttmp"
else
# Rename the file to the real destination.
$doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
# The rename failed, perhaps because mv can't rename something else
# to itself, or perhaps because mv is so ancient that it does not
# support -f.
{
# Now remove or move aside any old file at destination location.
# We try this two ways since rm can't unlink itself on some
# systems and the destination file might be busy for other
# reasons. In this case, the final cleanup might fail but the new
# file should still install successfully.
{
test ! -f "$dst" ||
$doit $rmcmd -f "$dst" 2>/dev/null ||
{ $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
{ $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
} ||
{ echo "$0: cannot unlink or rename $dst" >&2
(exit 1); exit 1
}
} &&
# Now rename the file to the real destination.
$doit $mvcmd "$dsttmp" "$dst"
}
fi || exit 1
trap '' 0
fi
done
# Local variables:
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:
config/m4/.gitignore
-2
View File
@@ -1,2 +0,0 @@
libtool.m4
lt*.m4
config/m4/find-func-no-libs.m4
-8
View File
@@ -1,8 +0,0 @@
dnl Shamelessly stolen from Heimdal
dnl
dnl Look for function in any of the specified libraries
dnl
dnl AC_FIND_FUNC_NO_LIBS(func, libraries, includes, arguments, extra libs, extra args)
AC_DEFUN([AC_FIND_FUNC_NO_LIBS], [
AC_FIND_FUNC_NO_LIBS2([$1], ["" $2], [$3], [$4], [$5], [$6])])
config/m4/find-func-no-libs2.m4
-62
View File
@@ -1,62 +0,0 @@
dnl Shamelessly stolen from Heimdal
dnl
dnl Look for function in any of the specified libraries
dnl
dnl AC_FIND_FUNC_NO_LIBS2(func, libraries, includes, arguments, extra libs, extra args)
AC_DEFUN([AC_FIND_FUNC_NO_LIBS2], [
AC_MSG_CHECKING([for $1])
AC_CACHE_VAL(ac_cv_funclib_$1,
[
if eval "test \"\$ac_cv_func_$1\" != yes" ; then
ac_save_LIBS="$LIBS"
for ac_lib in $2; do
case "$ac_lib" in
"") ;;
yes) ac_lib="" ;;
no) continue ;;
-l*) ;;
*) ac_lib="-l$ac_lib" ;;
esac
LIBS="$6 $ac_lib $5 $ac_save_LIBS"
AC_LINK_IFELSE([AC_LANG_PROGRAM([[$3]],[[$1($4)]])],[eval "if test -n \"$ac_lib\";then ac_cv_funclib_$1=$ac_lib; else ac_cv_funclib_$1=yes; fi";break])
done
eval "ac_cv_funclib_$1=\${ac_cv_funclib_$1-no}"
LIBS="$ac_save_LIBS"
fi
])
eval "ac_res=\$ac_cv_funclib_$1"
if false; then
AC_CHECK_FUNCS($1)
dnl AC_CHECK_LIBS($2, foo)
fi
# $1
eval "ac_tr_func=HAVE_[]upcase($1)"
eval "ac_tr_lib=HAVE_LIB[]upcase($ac_res | sed -e 's/-l//')"
eval "LIB_$1=$ac_res"
case "$ac_res" in
yes)
eval "ac_cv_func_$1=yes"
eval "LIB_$1="
AC_DEFINE_UNQUOTED($ac_tr_func)
AC_MSG_RESULT([yes])
;;
no)
eval "ac_cv_func_$1=no"
eval "LIB_$1="
AC_MSG_RESULT([no])
;;
*)
eval "ac_cv_func_$1=yes"
eval "ac_cv_lib_`echo "$ac_res" | sed 's/-l//'`=yes"
AC_DEFINE_UNQUOTED($ac_tr_func)
AC_DEFINE_UNQUOTED($ac_tr_lib)
AC_MSG_RESULT([yes, in $ac_res])
;;
esac
AC_SUBST(LIB_$1)
])
config/m4/find-func.m4
-9
View File
@@ -1,9 +0,0 @@
dnl Shamelessly stolen from Heimdal
dnl
dnl AC_FIND_FUNC(func, libraries, includes, arguments)
AC_DEFUN([AC_FIND_FUNC], [
AC_FIND_FUNC_NO_LIBS([$1], [$2], [$3], [$4])
if test -n "$LIB_$1"; then
LIBS="$LIB_$1 $LIBS"
fi
])
config/m4/misc.m4
-3
View File
@@ -1,3 +0,0 @@
dnl Shamelessly stolen from Heimdal
AC_DEFUN([upcase],[`echo $1 | tr abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ`])dnl
config/missing
Executable
+331
View File
@@ -0,0 +1,331 @@
#! /bin/sh
# Common stub for a few missing GNU programs while installing.
scriptversion=2012-01-06.13; # UTC
# Copyright (C) 1996, 1997, 1999, 2000, 2002, 2003, 2004, 2005, 2006,
# 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
# Originally by Fran,cois Pinard <pinard@iro.umontreal.ca>, 1996.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
if test $# -eq 0; then
echo 1>&2 "Try \`$0 --help' for more information"
exit 1
fi
run=:
sed_output='s/.* --output[ =]\([^ ]*\).*/\1/p'
sed_minuso='s/.* -o \([^ ]*\).*/\1/p'
# In the cases where this matters, `missing' is being run in the
# srcdir already.
if test -f configure.ac; then
configure_ac=configure.ac
else
configure_ac=configure.in
fi
msg="missing on your system"
case $1 in
--run)
# Try to run requested program, and just exit if it succeeds.
run=
shift
"$@" && exit 0
# Exit code 63 means version mismatch. This often happens
# when the user try to use an ancient version of a tool on
# a file that requires a minimum version. In this case we
# we should proceed has if the program had been absent, or
# if --run hadn't been passed.
if test $? = 63; then
run=:
msg="probably too old"
fi
;;
-h|--h|--he|--hel|--help)
echo "\
$0 [OPTION]... PROGRAM [ARGUMENT]...
Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an
error status if there is no known handling for PROGRAM.
Options:
-h, --help display this help and exit
-v, --version output version information and exit
--run try to run the given command, and emulate it if it fails
Supported PROGRAM values:
aclocal touch file \`aclocal.m4'
autoconf touch file \`configure'
autoheader touch file \`config.h.in'
autom4te touch the output file, or create a stub one
automake touch all \`Makefile.in' files
bison create \`y.tab.[ch]', if possible, from existing .[ch]
flex create \`lex.yy.c', if possible, from existing .c
help2man touch the output file
lex create \`lex.yy.c', if possible, from existing .c
makeinfo touch the output file
yacc create \`y.tab.[ch]', if possible, from existing .[ch]
Version suffixes to PROGRAM as well as the prefixes \`gnu-', \`gnu', and
\`g' are ignored when checking the name.
Send bug reports to <bug-automake@gnu.org>."
exit $?
;;
-v|--v|--ve|--ver|--vers|--versi|--versio|--version)
echo "missing $scriptversion (GNU Automake)"
exit $?
;;
-*)
echo 1>&2 "$0: Unknown \`$1' option"
echo 1>&2 "Try \`$0 --help' for more information"
exit 1
;;
esac
# normalize program name to check for.
program=`echo "$1" | sed '
s/^gnu-//; t
s/^gnu//; t
s/^g//; t'`
# Now exit if we have it, but it failed. Also exit now if we
# don't have it and --version was passed (most likely to detect
# the program). This is about non-GNU programs, so use $1 not
# $program.
case $1 in
lex*|yacc*)
# Not GNU programs, they don't have --version.
;;
*)
if test -z "$run" && ($1 --version) > /dev/null 2>&1; then
# We have it, but it failed.
exit 1
elif test "x$2" = "x--version" || test "x$2" = "x--help"; then
# Could not run --version or --help. This is probably someone
# running `$TOOL --version' or `$TOOL --help' to check whether
# $TOOL exists and not knowing $TOOL uses missing.
exit 1
fi
;;
esac
# If it does not exist, or fails to run (possibly an outdated version),
# try to emulate it.
case $program in
aclocal*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified \`acinclude.m4' or \`${configure_ac}'. You might want
to install the \`Automake' and \`Perl' packages. Grab them from
any GNU archive site."
touch aclocal.m4
;;
autoconf*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified \`${configure_ac}'. You might want to install the
\`Autoconf' and \`GNU m4' packages. Grab them from any GNU
archive site."
touch configure
;;
autoheader*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified \`acconfig.h' or \`${configure_ac}'. You might want
to install the \`Autoconf' and \`GNU m4' packages. Grab them
from any GNU archive site."
files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' ${configure_ac}`
test -z "$files" && files="config.h"
touch_files=
for f in $files; do
case $f in
*:*) touch_files="$touch_files "`echo "$f" |
sed -e 's/^[^:]*://' -e 's/:.*//'`;;
*) touch_files="$touch_files $f.in";;
esac
done
touch $touch_files
;;
automake*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified \`Makefile.am', \`acinclude.m4' or \`${configure_ac}'.
You might want to install the \`Automake' and \`Perl' packages.
Grab them from any GNU archive site."
find . -type f -name Makefile.am -print |
sed 's/\.am$/.in/' |
while read f; do touch "$f"; done
;;
autom4te*)
echo 1>&2 "\
WARNING: \`$1' is needed, but is $msg.
You might have modified some files without having the
proper tools for further handling them.
You can get \`$1' as part of \`Autoconf' from any GNU
archive site."
file=`echo "$*" | sed -n "$sed_output"`
test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
if test -f "$file"; then
touch $file
else
test -z "$file" || exec >$file
echo "#! /bin/sh"
echo "# Created by GNU Automake missing as a replacement of"
echo "# $ $@"
echo "exit 0"
chmod +x $file
exit 1
fi
;;
bison*|yacc*)
echo 1>&2 "\
WARNING: \`$1' $msg. You should only need it if
you modified a \`.y' file. You may need the \`Bison' package
in order for those modifications to take effect. You can get
\`Bison' from any GNU archive site."
rm -f y.tab.c y.tab.h
if test $# -ne 1; then
eval LASTARG=\${$#}
case $LASTARG in
*.y)
SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'`
if test -f "$SRCFILE"; then
cp "$SRCFILE" y.tab.c
fi
SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'`
if test -f "$SRCFILE"; then
cp "$SRCFILE" y.tab.h
fi
;;
esac
fi
if test ! -f y.tab.h; then
echo >y.tab.h
fi
if test ! -f y.tab.c; then
echo 'main() { return 0; }' >y.tab.c
fi
;;
lex*|flex*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified a \`.l' file. You may need the \`Flex' package
in order for those modifications to take effect. You can get
\`Flex' from any GNU archive site."
rm -f lex.yy.c
if test $# -ne 1; then
eval LASTARG=\${$#}
case $LASTARG in
*.l)
SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'`
if test -f "$SRCFILE"; then
cp "$SRCFILE" lex.yy.c
fi
;;
esac
fi
if test ! -f lex.yy.c; then
echo 'main() { return 0; }' >lex.yy.c
fi
;;
help2man*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified a dependency of a manual page. You may need the
\`Help2man' package in order for those modifications to take
effect. You can get \`Help2man' from any GNU archive site."
file=`echo "$*" | sed -n "$sed_output"`
test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
if test -f "$file"; then
touch $file
else
test -z "$file" || exec >$file
echo ".ab help2man is required to generate this page"
exit $?
fi
;;
makeinfo*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified a \`.texi' or \`.texinfo' file, or any other file
indirectly affecting the aspect of the manual. The spurious
call might also be the consequence of using a buggy \`make' (AIX,
DU, IRIX). You might want to install the \`Texinfo' package or
the \`GNU make' package. Grab either from any GNU archive site."
# The file to touch is that specified with -o ...
file=`echo "$*" | sed -n "$sed_output"`
test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
if test -z "$file"; then
# ... or it is the one specified with @setfilename ...
infile=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'`
file=`sed -n '
/^@setfilename/{
s/.* \([^ ]*\) *$/\1/
p
q
}' $infile`
# ... or it is derived from the source name (dir/f.texi becomes f.info)
test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info
fi
# If the file does not exist, the user really needs makeinfo;
# let's fail without touching anything.
test -f $file || exit 1
touch $file
;;
*)
echo 1>&2 "\
WARNING: \`$1' is needed, and is $msg.
You might have modified some files without having the
proper tools for further handling them. Check the \`README' file,
it often tells you about the needed prerequisites for installing
this package. You may also peek at any GNU archive site, in case
some other package would contain this missing \`$1' program."
exit 1
;;
esac
exit 0
# Local variables:
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:
config/ylwrap
Executable
+226
View File
@@ -0,0 +1,226 @@
#! /bin/sh
# ylwrap - wrapper for lex/yacc invocations.
scriptversion=2011-08-25.18; # UTC
# Copyright (C) 1996, 1997, 1998, 1999, 2001, 2002, 2003, 2004, 2005,
# 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
#
# Written by Tom Tromey <tromey@cygnus.com>.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
# This file is maintained in Automake, please report
# bugs to <bug-automake@gnu.org> or send patches to
# <automake-patches@gnu.org>.
case "$1" in
'')
echo "$0: No files given. Try \`$0 --help' for more information." 1>&2
exit 1
;;
--basedir)
basedir=$2
shift 2
;;
-h|--h*)
cat <<\EOF
Usage: ylwrap [--help|--version] INPUT [OUTPUT DESIRED]... -- PROGRAM [ARGS]...
Wrapper for lex/yacc invocations, renaming files as desired.
INPUT is the input file
OUTPUT is one file PROG generates
DESIRED is the file we actually want instead of OUTPUT
PROGRAM is program to run
ARGS are passed to PROG
Any number of OUTPUT,DESIRED pairs may be used.
Report bugs to <bug-automake@gnu.org>.
EOF
exit $?
;;
-v|--v*)
echo "ylwrap $scriptversion"
exit $?
;;
esac
# The input.
input="$1"
shift
case "$input" in
[\\/]* | ?:[\\/]*)
# Absolute path; do nothing.
;;
*)
# Relative path. Make it absolute.
input="`pwd`/$input"
;;
esac
pairlist=
while test "$#" -ne 0; do
if test "$1" = "--"; then
shift
break
fi
pairlist="$pairlist $1"
shift
done
# The program to run.
prog="$1"
shift
# Make any relative path in $prog absolute.
case "$prog" in
[\\/]* | ?:[\\/]*) ;;
*[\\/]*) prog="`pwd`/$prog" ;;
esac
# FIXME: add hostname here for parallel makes that run commands on
# other machines. But that might take us over the 14-char limit.
dirname=ylwrap$$
do_exit="cd '`pwd`' && rm -rf $dirname > /dev/null 2>&1;"' (exit $ret); exit $ret'
trap "ret=129; $do_exit" 1
trap "ret=130; $do_exit" 2
trap "ret=141; $do_exit" 13
trap "ret=143; $do_exit" 15
mkdir $dirname || exit 1
cd $dirname
case $# in
0) "$prog" "$input" ;;
*) "$prog" "$@" "$input" ;;
esac
ret=$?
if test $ret -eq 0; then
set X $pairlist
shift
first=yes
# Since DOS filename conventions don't allow two dots,
# the DOS version of Bison writes out y_tab.c instead of y.tab.c
# and y_tab.h instead of y.tab.h. Test to see if this is the case.
y_tab_nodot="no"
if test -f y_tab.c || test -f y_tab.h; then
y_tab_nodot="yes"
fi
# The directory holding the input.
input_dir=`echo "$input" | sed -e 's,\([\\/]\)[^\\/]*$,\1,'`
# Quote $INPUT_DIR so we can use it in a regexp.
# FIXME: really we should care about more than `.' and `\'.
input_rx=`echo "$input_dir" | sed 's,\\\\,\\\\\\\\,g;s,\\.,\\\\.,g'`
while test "$#" -ne 0; do
from="$1"
# Handle y_tab.c and y_tab.h output by DOS
if test $y_tab_nodot = "yes"; then
if test $from = "y.tab.c"; then
from="y_tab.c"
else
if test $from = "y.tab.h"; then
from="y_tab.h"
fi
fi
fi
if test -f "$from"; then
# If $2 is an absolute path name, then just use that,
# otherwise prepend `../'.
case "$2" in
[\\/]* | ?:[\\/]*) target="$2";;
*) target="../$2";;
esac
# We do not want to overwrite a header file if it hasn't
# changed. This avoid useless recompilations. However the
# parser itself (the first file) should always be updated,
# because it is the destination of the .y.c rule in the
# Makefile. Divert the output of all other files to a temporary
# file so we can compare them to existing versions.
if test $first = no; then
realtarget="$target"
target="tmp-`echo $target | sed s/.*[\\/]//g`"
fi
# Edit out `#line' or `#' directives.
#
# We don't want the resulting debug information to point at
# an absolute srcdir; it is better for it to just mention the
# .y file with no path.
#
# We want to use the real output file name, not yy.lex.c for
# instance.
#
# We want the include guards to be adjusted too.
FROM=`echo "$from" | sed \
-e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'\
-e 's/[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]/_/g'`
TARGET=`echo "$2" | sed \
-e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'\
-e 's/[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]/_/g'`
sed -e "/^#/!b" -e "s,$input_rx,," -e "s,$from,$2," \
-e "s,$FROM,$TARGET," "$from" >"$target" || ret=$?
# Check whether header files must be updated.
if test $first = no; then
if test -f "$realtarget" && cmp -s "$realtarget" "$target"; then
echo "$2" is unchanged
rm -f "$target"
else
echo updating "$2"
mv -f "$target" "$realtarget"
fi
fi
else
# A missing file is only an error for the first file. This
# is a blatant hack to let us support using "yacc -d". If -d
# is not specified, we don't want an error when the header
# file is "missing".
if test $first = yes; then
ret=1
fi
fi
shift
shift
first=no
done
else
ret=$?
fi
# Remove the directory.
cd ..
rm -rf $dirname
exit $ret
# Local Variables:
# mode: shell-script
# sh-indentation: 2
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:
configure.ac
+12 -108
View File
@@ -1,47 +1,21 @@
m4_define([jq_version], m4_esyscmd_s([git describe --tags --dirty --match 'jq-*'|sed 's/^jq-//']))
AC_INIT([jq], [jq_version], [mu@netsoc.tcd.ie],
AC_INIT([jq], [1.3], [mu@netsoc.tcd.ie],
[jq], [http://stedolan.github.com/jq/])
m4_include([m4/ax_compare_version.m4])
m4_include([m4/ax_prog_bison_version.m4])
dnl Created autoconf implementation thompson@dtosolutions, 26NOV12
AC_PREREQ([2.61])
AC_CONFIG_AUX_DIR([config])
AM_INIT_AUTOMAKE([parallel-tests foreign -Wall])
AM_INIT_AUTOMAKE([parallel-tests foreign])
AM_SILENT_RULES([yes])
AM_PROG_AR
AC_PROG_CC
AC_PROG_CC_STDC
AC_PROG_CPP_WERROR
AC_PROG_YACC
AC_OBJEXT
AC_EXEEXT
LT_INIT([shared static win32-dll])
AM_PROG_CC_C_O
AM_PROG_CC_C_O
dnl couldn't use AM_PROG_LEX as it doesn't support header files like the
dnl AC_PROG_YACC macros...
dnl check bison version
if test "$YACC" != "bison -y"; then
AC_MSG_CHECKING([bison version])
AC_MSG_RESULT([not bison])
else
AX_PROG_BISON_VERSION([3],
[],
[AC_MSG_ERROR([You need bison version 3.0 or greater.])])
dnl These gross square brackets keep autotools from eating the perl script...
# yacc_version=[$($YACC --version 2>1 | perl -pe '($_)=/([0-9]+([.][0-9]+)+)/')]
# AX_COMPARE_VERSION([$yacc_version], [gt], [3],
# [AC_MSG_RESULT([ok])],
# [AC_MSG_RESULT([no])
# AC_MSG_ERROR([You need bison version 3.0 or greater.])])
fi
dnl
dnl these program checks should probably be deleted
dnl
@@ -52,41 +26,11 @@ if test "x$LEX" != xflex; then
fi
##########################################################################
# check for ONIGURUMA library
##########################################################################
AC_ARG_WITH([oniguruma],
[AS_HELP_STRING([--with-oniguruma=prefix],
[try this for a non-standard install prefix of the oniguruma library])],
[ONIGURUMAPATHSET=1],
[ONIGURUMAPATHSET=0])
if test $ONIGURUMAPATHSET == 1; then
CFLAGS="$CFLAGS -I${with_oniguruma}/include"
LDFLAGS="$LDFLAGS -L${with_oniguruma}/lib"
fi
# check for ONIGURUMA library
HAVE_ONIGURUMA=1
AC_CHECK_HEADER("oniguruma.h",
AC_CHECK_LIB([onig],[onig_version],[LIBS="$LIBS -lonig"; HAVE_ONIGURUMA=1;]))
# handle check results
if test $HAVE_ONIGURUMA != 1; then
AC_MSG_NOTICE([Oniguruma was not found.])
AC_MSG_NOTICE([ Try setting the location using '--with-oniguruma=PREFIX' ])
AC_MSG_ERROR([ oniguruma is required to build jq.])
fi
dnl Check for valgrind
AC_CHECK_PROGS(valgrind_cmd, valgrind)
if test "x$valgrind_cmd" = "x" ; then
AC_MSG_WARN([valgrind is required to test jq.])
fi
AC_CHECK_FUNCS(memmem)
AC_CHECK_FUNCS(mkstemp)
dnl Don't attempt to build docs if there's no Ruby lying around
@@ -95,13 +39,10 @@ AC_ARG_ENABLE([docs],
AS_IF([test "x$enable_docs" != "xno"],[
AC_CHECK_PROGS(bundle_cmd, bundle)
AC_CACHE_CHECK([for Ruby dependencies], [jq_cv_ruby_deps],
[jq_cv_ruby_deps=yes;
AS_IF([test "x$bundle_cmd" = "x" || \
! bmsg="`cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null`"],[
AC_MSG_WARN([$bmsg])
cat <<EOF
AS_IF([test "x$bundle_cmd" = "x" || \
! ( cd ${srcdir}/docs; "$bundle_cmd" check 2>/dev/null )],[
AC_MSG_WARN([no bundler])
cat <<EOF
*****************************************************************
* Ruby dependencies for building jq documentation not found. *
* You can still build, install and hack on jq, but the manpage *
@@ -109,53 +50,16 @@ AS_IF([test "x$enable_docs" != "xno"],[
* See docs/README.md for how to install the docs dependencies. *
*****************************************************************
EOF
jq_cv_ruby_deps=no
])])
if test "x$jq_cv_ruby_deps" != "xyes"; then
enable_docs=no
fi
enable_docs=no
])
])
AM_CONDITIONAL([ENABLE_DOCS], [test "x$enable_docs" != xno])
AC_FIND_FUNC([isatty], [c], [#include <unistd.h>], [0])
AC_FIND_FUNC([_isatty], [c], [#include <io.h>], [0])
AC_ARG_ENABLE([pthread-tls],
[AC_HELP_STRING([--enable-pthread-tls],
[Enable use of pthread thread local storage])],
[],
[enable_pthread_tls=no])
if test $enable_pthread_tls = yes; then
AC_FIND_FUNC([pthread_key_create], [pthread], [#include <pthread.h>], [NULL, NULL])
AC_FIND_FUNC([pthread_once], [pthread], [#include <pthread.h>], [NULL, NULL])
AC_FIND_FUNC([atexit], [pthread], [#include <stdlib.h>], [NULL])
fi
dnl Thread local storage
have___thread=no
AC_MSG_CHECKING(for thread-local storage)
AC_LINK_IFELSE([AC_LANG_SOURCE([
static __thread int x ;
int main () { x = 123; return x; }
])], have___thread=yes)
if test $have___thread = yes; then
AC_DEFINE([HAVE___THREAD],1,[Define to 1 if the system supports __thread])
fi
AC_MSG_RESULT($have___thread)
AC_C_BIGENDIAN(
AC_DEFINE([IEEE_MC68k], 1, [machine is bigendian]),
AC_DEFINE([IEEE_8087], 1, [machine is littleendian]),
AC_MSG_ERROR(unknown endianess),
AC_MSG_ERROR(universial endianess not supported)
)
AC_SUBST([BUNDLER], ["$bundle_cmd"])
AC_CONFIG_MACRO_DIR([config/m4])
dnl AC_CONFIG_MACRO_DIR([m4])
AC_CONFIG_HEADERS(config.h)
AM_INIT_AUTOMAKE([-Wall])
AC_CONFIG_FILES([Makefile])
AC_OUTPUT
docs/Rakefile
+12 -4
View File
@@ -75,7 +75,7 @@ $BINARIES = {
$BINARIES.each do |name, args|
file "../build/#{name}" do |t|
sh "../scripts/crosscompile -j4 #{name} #{args}"
sh "MAKEFLAGS=-j4 ../scripts/crosscompile #{name} #{args}"
end
task :binaries => ["../build/#{name}"]
end
@@ -91,6 +91,10 @@ def load_manual
YAML::load(File.open("content/3.manual/manual.yml"))
end
def jq_version
YAML::load(File.open("site.yml"))["jq_version"]
end
task :manpage do
Tempfile.open "manpage" do |f|
manual = load_manual
@@ -137,10 +141,14 @@ task :mantests do
end
end
$TARBALL = "../jq-#{jq_version}.tar.gz"
file $TARBALL do
sh "cd ..; ./configure && make dist && make distclean"
end
directory "output/download/source"
task :tarball => ["output/download/source"] do
sh "cd ..; ./configure && make dist && make distclean"
sh "cp ../jq-*.tar.gz output/download/source"
task :tarball => ["output/download/source", $TARBALL] do
sh "cp #{$TARBALL} output/download/source"
end
task :dist => [:build, :binaries, :tarball]
docs/content/1.tutorial/default.yml
+160 -241
View File
@@ -2,205 +2,123 @@ headline: Tutorial
body:
- text: |
GitHub has a JSON API, so let's play with that. This URL gets
us the last 5 commits from the jq repo:
- command: "curl 'https://api.github.com/repos/stedolan/jq/commits?per_page=5'"
Twitter has a JSON API, so let's play with that. This URL gets
us the last 5 tweets about JSON:
- command: "curl 'http://search.twitter.com/search.json?q=json&rpp=5&include_entities=true'"
result: |
[
{
"sha": "d25341478381063d1c76e81b3a52e0592a7c997f",
"commit": {
"author": {
"name": "Stephen Dolan",
"email": "mu@netsoc.tcd.ie",
"date": "2013-06-22T16:30:59Z"
},
"committer": {
"name": "Stephen Dolan",
"email": "mu@netsoc.tcd.ie",
"date": "2013-06-22T16:30:59Z"
},
"message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161",
"tree": {
"sha": "6ab697a8dfb5a96e124666bf6d6213822599fb40",
"url": "https://api.github.com/repos/stedolan/jq/git/trees/6ab697a8dfb5a96e124666bf6d6213822599fb40"
},
"url": "https://api.github.com/repos/stedolan/jq/git/commits/d25341478381063d1c76e81b3a52e0592a7c997f",
"comment_count": 0
},
"url": "https://api.github.com/repos/stedolan/jq/commits/d25341478381063d1c76e81b3a52e0592a7c997f",
"html_url": "https://github.com/stedolan/jq/commit/d25341478381063d1c76e81b3a52e0592a7c997f",
"comments_url": "https://api.github.com/repos/stedolan/jq/commits/d25341478381063d1c76e81b3a52e0592a7c997f/comments",
"author": {
"login": "stedolan",
...
- text: |
Github returns a ton of info here, and it's relatively hard to
read as it is. To syntax highlight it and order the attributes we pipe
it through jq, telling jq to just spit the input back at us using
the expression `.`:
- command: "curl 'https://api.github.com/repos/stedolan/jq/commits?per_page=5' | jq '.'"
result: |
[
{
"parents": [
{
"html_url": "https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
"url": "https://api.github.com/repos/stedolan/jq/commits/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
"sha": "54b9c9bdb225af5d886466d72f47eafc51acb4f7"
},
{
"html_url": "https://github.com/stedolan/jq/commit/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
"url": "https://api.github.com/repos/stedolan/jq/commits/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
"sha": "8b1b503609c161fea4b003a7179b3fbb2dd4345a"
}
],
"committer": {
"type": "User",
"received_events_url": "https://api.github.com/users/stedolan/received_events",
"events_url": "https://api.github.com/users/stedolan/events{/privacy}",
"repos_url": "https://api.github.com/users/stedolan/repos",
"organizations_url": "https://api.github.com/users/stedolan/orgs",
...
{"completed_in":0.108,"max_id":247677287108067328,"max_id_str":"247677287108067328","next_page":"?page=2&max_id=247677287108067328&q=json&rpp=5&include_entities=1","page":1,"query":"json","refresh_url":"?since_id=247677287108067328&q=json&include_entities=1","results":[{"created_at":"Mon, 17 Sep 2012 12:44:01 +0000","entities":{"hashtags":[],"urls":[{"url":"http:\/\/t.co\/XRvh1ZVw","expanded_url":"http:\/\/jase.im\/Ri7I0M","display_url":"jase.im\/Ri7I0M","indices":[112,132]}],"user_mentions":[{"screen_name":"imagemechanics","name":"Jason Cotterell","id":57271393,"id_str":"57271393","indices":[3,18]}]},"from_user":"_AaronNg","from_user_id":79771704,"from_user_id_str":"79771704","from_user_name":"NgChenChong","geo":null,"id":247677287108067328,"id_str":"247677287108067328","iso_language_code":"en","metadata":{"result_type":"recent"},"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/2523558403\/ek8mo4j4beq84iw28gjl_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/2523558403\/ek8mo4j4beq84iw28gjl_normal.jpeg","source":"&lt;a href=&quot;http:\/\/twitter.com\/&quot;&gt;web&lt;\/a&gt;","text":"RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt;.\nhttp:\/\/t.co\/XRvh1ZVw","to_user":null,"to_user_id":0,"to_user_id_str":"0","to_user_name":null}, ...
- text: |
There's still far too much info for our purposes, so we can try
just grab the first commit:
There's lots of info and no whitespace, so to make it a bit more
legible we pipe it through jq, telling jq to just spit the input
back at us using the expression `.`:
- command: "curl 'https://api.github.com/repos/stedolan/jq/commits?per_page=5' | jq '.[0]'"
- command: "curl 'http://search.twitter.com/search.json?q=json&rpp=5&include_entities=true' | jq '.'"
result: |
{
"parents": [
"since_id_str": "0",
"since_id": 0,
"results_per_page": 5,
"completed_in": 0.108,
"max_id": 247677287108067330,
"max_id_str": "247677287108067328",
"next_page": "?page=2&max_id=247677287108067328&q=json&rpp=5&include_entities=1",
"page": 1,
"query": "json",
"refresh_url": "?since_id=247677287108067328&q=json&include_entities=1",
"results": [
{
"html_url": "https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
"url": "https://api.github.com/repos/stedolan/jq/commits/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
"sha": "54b9c9bdb225af5d886466d72f47eafc51acb4f7"
"from_user_name": "NgChenChong",
"from_user_id_str": "79771704",
"from_user_id": 79771704,
"from_user": "_AaronNg",
"iso_language_code": "en",
"text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt;.\nhttp://t.co/XRvh1ZVw",
"to_user": null
/* lots more fields... */
},
{
"html_url": "https://github.com/stedolan/jq/commit/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
"url": "https://api.github.com/repos/stedolan/jq/commits/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
"sha": "8b1b503609c161fea4b003a7179b3fbb2dd4345a"
}
],
"committer": {
"type": "User",
"received_events_url": "https://api.github.com/users/stedolan/received_events",
"events_url": "https://api.github.com/users/stedolan/events{/privacy}",
"repos_url": "https://api.github.com/users/stedolan/repos",
"organizations_url": "https://api.github.com/users/stedolan/orgs",
"subscriptions_url": "https://api.github.com/users/stedolan/subscriptions",
"starred_url": "https://api.github.com/users/stedolan/starred{/owner}{/repo}",
"gists_url": "https://api.github.com/users/stedolan/gists{/gist_id}",
"login": "stedolan",
"id": 79765,
"avatar_url": "https://1.gravatar.com/avatar/31de909d8e55dd07ed782d92ece59842?d=https%3A%2F%2Fidenticons.github.com%2Ffc5b6765b1c9cfaecea48ae71df4d279.png",
"gravatar_id": "31de909d8e55dd07ed782d92ece59842",
"url": "https://api.github.com/users/stedolan",
"html_url": "https://github.com/stedolan",
"followers_url": "https://api.github.com/users/stedolan/followers",
"following_url": "https://api.github.com/users/stedolan/following{/other_user}"
},
"author": {
"type": "User",
"received_events_url": "https://api.github.com/users/stedolan/received_events",
"events_url": "https://api.github.com/users/stedolan/events{/privacy}",
"repos_url": "https://api.github.com/users/stedolan/repos",
"organizations_url": "https://api.github.com/users/stedolan/orgs",
"subscriptions_url": "https://api.github.com/users/stedolan/subscriptions",
"starred_url": "https://api.github.com/users/stedolan/starred{/owner}{/repo}",
"gists_url": "https://api.github.com/users/stedolan/gists{/gist_id}",
"login": "stedolan",
"id": 79765,
"avatar_url": "https://1.gravatar.com/avatar/31de909d8e55dd07ed782d92ece59842?d=https%3A%2F%2Fidenticons.github.com%2Ffc5b6765b1c9cfaecea48ae71df4d279.png",
"gravatar_id": "31de909d8e55dd07ed782d92ece59842",
"url": "https://api.github.com/users/stedolan",
"html_url": "https://github.com/stedolan",
"followers_url": "https://api.github.com/users/stedolan/followers",
"following_url": "https://api.github.com/users/stedolan/following{/other_user}"
},
"comments_url": "https://api.github.com/repos/stedolan/jq/commits/d25341478381063d1c76e81b3a52e0592a7c997f/comments",
"html_url": "https://github.com/stedolan/jq/commit/d25341478381063d1c76e81b3a52e0592a7c997f",
"url": "https://api.github.com/repos/stedolan/jq/commits/d25341478381063d1c76e81b3a52e0592a7c997f",
"commit": {
"comment_count": 0,
"url": "https://api.github.com/repos/stedolan/jq/git/commits/d25341478381063d1c76e81b3a52e0592a7c997f",
"tree": {
"url": "https://api.github.com/repos/stedolan/jq/git/trees/6ab697a8dfb5a96e124666bf6d6213822599fb40",
"sha": "6ab697a8dfb5a96e124666bf6d6213822599fb40"
},
"message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161",
"committer": {
"date": "2013-06-22T16:30:59Z",
"email": "mu@netsoc.tcd.ie",
"name": "Stephen Dolan"
},
"author": {
"date": "2013-06-22T16:30:59Z",
"email": "mu@netsoc.tcd.ie",
"name": "Stephen Dolan"
}
},
"sha": "d25341478381063d1c76e81b3a52e0592a7c997f"
/* lots more results... */
]
}
- text: |
Let's pull out the first tweet:
- command: "curl 'http://search.twitter.com/search.json?q=json&rpp=5&include_entities=true' | jq '.results[0]'"
result: |
{
"from_user_name": "NgChenChong",
"from_user_id_str": "79771704",
"from_user_id": 79771704,
"from_user": "_AaronNg",
"iso_language_code": "en",
"text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt;.\nhttp://t.co/XRvh1ZVw",
"to_user": null
/* lots more fields... */
},
- text: |
For the rest of the examples, I'll leave out the `curl` command - it's not going to change.
There's still a lot of info we don't care about there, so we'll
restrict it down to the most interesting fields.
- command: "jq '.[0] | {message: .commit.message, name: .commit.committer.name}'"
- command: "jq '.results[0] | {from_user, text}'"
result: |
{
"name": "Stephen Dolan",
"message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161"
"text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt;.\nhttp://t.co/XRvh1ZVw",
"from_user": "_AaronNg"
}
- text: |
The `|` operator in jq feeds the output of one filter
(`.[0]` which gets the first element of the array in the response)
into the input of another (`{...}` which
builds an object out of those fields). You can access nested attributes,
such as `.commit.message`:
(`.results[0]` which gets the first element of the results
array) into the input of another (`{from_user, text}` which
builds an object of those two fields).
Now let's get the rest of the tweets:
Now let's get the rest of the commits:
- command: "jq '.[] | {message: .commit.message, name: .commit.committer.name}'"
- command: "jq '.results[] | {from_user, text}'"
result: |
{
"name": "Stephen Dolan",
"message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161"
"text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt;.\nhttp://t.co/XRvh1ZVw",
"from_user": "_AaronNg"
}
{
"name": "Stephen Dolan",
"message": "Reject all overlong UTF8 sequences."
"text": "RT @_kud: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt; -- http://t.co/Lhp92IqD",
"from_user": "blouerat"
}
{
"name": "Stephen Dolan",
"message": "Fix various UTF8 parsing bugs.\n\nIn particular, parse bad UTF8 by replacing the broken bits with U+FFFD\nand resychronise correctly after broken sequences."
"text": "Dynamic Forms Mobile App by pmadiset: Few Forms details are hosted on our server in the form of JSON. This JSON ... http://t.co/7KALdWaX",
"from_user": "cebu_iphone"
}
{
"name": "Stephen Dolan",
"message": "Fix example in manual for `floor`. See #155."
"text": "iPhone 5 website insanity - Video compressing using JPEG, JSON, and &lt;canvas&gt; http://t.co/28Jesbio (Oh #Apple, U So #Awesome...)",
"from_user": "dieswaytoofast"
}
{
"name": "Nicolas Williams",
"message": "Document floor"
"text": "RT @umutm: A very nice web-based JSON editor - http://t.co/M70snaIf",
"from_user": "Leolik"
}
- text: |
`.[]` returns each element of the array returned in the response,
one at a time, which are all fed into `{message: .commit.message, name: .commit.committer.name}`.
`.results[]` returns each element of the results array, one at a
time, which are all fed into `{from_user, text}`.
Data in jq is represented as streams of JSON values - every jq
@@ -210,108 +128,109 @@ body:
Streams are serialised by just separating JSON values with
whitespace. This is a `cat`-friendly format - you can just join
two JSON streams together and get a valid JSON stream.
If you want to get the output as a single array, you can tell jq to
"collect" all of the answers by wrapping the filter in square
brackets:
- command: "jq '[.[] | {message: .commit.message, name: .commit.committer.name}]'"
- command: "jq '[.results[] | {from_user, text}]'"
result: |
[
{
"name": "Stephen Dolan",
"message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161"
"text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt;.\nhttp://t.co/XRvh1ZVw",
"from_user": "_AaronNg"
},
{
"name": "Stephen Dolan",
"message": "Reject all overlong UTF8 sequences."
"text": "RT @_kud: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt; -- http://t.co/Lhp92IqD",
"from_user": "blouerat"
},
{
"name": "Stephen Dolan",
"message": "Fix various UTF8 parsing bugs.\n\nIn particular, parse bad UTF8 by replacing the broken bits with U+FFFD\nand resychronise correctly after broken sequences."
"text": "Dynamic Forms Mobile App by pmadiset: Few Forms details are hosted on our server in the form of JSON. This JSON ... http://t.co/7KALdWaX",
"from_user": "cebu_iphone"
},
{
"name": "Stephen Dolan",
"message": "Fix example in manual for `floor`. See #155."
"text": "iPhone 5 website insanity - Video compressing using JPEG, JSON, and &lt;canvas&gt; http://t.co/28Jesbio (Oh #Apple, U So #Awesome...)",
"from_user": "dieswaytoofast"
},
{
"name": "Nicolas Williams",
"message": "Document floor"
"text": "RT @umutm: A very nice web-based JSON editor - http://t.co/M70snaIf",
"from_user": "Leolik"
}
]
- text: |
- - -
Next, let's try getting the URLs out of those API results as
well. In each tweet, the Twitter API includes a field called
"entities" which looks like this:
Next, let's try getting the URLs of the parent commits out of the
API results as well. In each commit, the GitHub API includes information
about "parent" commits. There can be one or many.
{
"user_mentions": [],
"urls": [{
"indices": [83, 103],
"display_url": "bit.ly/StniqT",
"expanded_url": "http://bit.ly/StniqT",
"url": "http://t.co/28Jesbio"
}],
"hashtags": [
{"indices": [108, 114], "text": "Apple" },
{"indices": [121, 129], "text": "Awesome"}
]
}
"parents": [
{
"html_url": "https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
"url": "https://api.github.com/repos/stedolan/jq/commits/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
"sha": "54b9c9bdb225af5d886466d72f47eafc51acb4f7"
},
{
"html_url": "https://github.com/stedolan/jq/commit/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
"url": "https://api.github.com/repos/stedolan/jq/commits/8b1b503609c161fea4b003a7179b3fbb2dd4345a",
"sha": "8b1b503609c161fea4b003a7179b3fbb2dd4345a"
}
]
We want to pull out all of the "url" fields inside that array of url
objects, and make a simple list of strings to go along with the
"from_user" and "text" fields we already have.
We want to pull out all of the "html_url" fields inside that array of parent
commits and make a simple list of strings to go along with the
"message" and "author" fields we already have.
- command: "jq '[.[] | {message: .commit.message, name: .commit.committer.name, parents: [.parents[].html_url]}]'"
- command: "jq '.results[] | {from_user, text, urls: [.entities.urls[].url]}'"
result: |
[
{
"parents": [
"https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7",
"https://github.com/stedolan/jq/commit/8b1b503609c161fea4b003a7179b3fbb2dd4345a"
],
"name": "Stephen Dolan",
"message": "Merge pull request #162 from stedolan/utf8-fixes\n\nUtf8 fixes. Closes #161"
},
{
"parents": [
"https://github.com/stedolan/jq/commit/ff48bd6ec538b01d1057be8e93b94eef6914e9ef"
],
"name": "Stephen Dolan",
"message": "Reject all overlong UTF8 sequences."
},
{
"parents": [
"https://github.com/stedolan/jq/commit/54b9c9bdb225af5d886466d72f47eafc51acb4f7"
],
"name": "Stephen Dolan",
"message": "Fix various UTF8 parsing bugs.\n\nIn particular, parse bad UTF8 by replacing the broken bits with U+FFFD\nand resychronise correctly after broken sequences."
},
{
"parents": [
"https://github.com/stedolan/jq/commit/3dcdc582ea993afea3f5503a78a77675967ecdfa"
],
"name": "Stephen Dolan",
"message": "Fix example in manual for `floor`. See #155."
},
{
"parents": [
"https://github.com/stedolan/jq/commit/7c4171d414f647ab08bcd20c76a4d8ed68d9c602"
],
"name": "Nicolas Williams",
"message": "Document floor"
}
]
{
"urls": [
"http://t.co/XRvh1ZVw"
],
"text": "RT @imagemechanics: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt;.\nhttp://t.co/XRvh1ZVw",
"from_user": "_AaronNg"
}
{
"urls": [
"http://t.co/Lhp92IqD"
],
"text": "RT @_kud: iPhone 5 website teardown: How Apple compresses video using JPEG, JSON, and &lt;canvas&gt; -- http://t.co/Lhp92IqD",
"from_user": "blouerat"
}
{
"urls": [
"http://t.co/7KALdWaX"
],
"text": "Dynamic Forms Mobile App by pmadiset: Few Forms details are hosted on our server in the form of JSON. This JSON ... http://t.co/7KALdWaX",
"from_user": "cebu_iphone"
}
{
"urls": [
"http://t.co/28Jesbio"
],
"text": "iPhone 5 website insanity - Video compressing using JPEG, JSON, and &lt;canvas&gt; http://t.co/28Jesbio (Oh #Apple, U So #Awesome...)",
"from_user": "dieswaytoofast"
}
{
"urls": [
"http://t.co/M70snaIf"
],
"text": "RT @umutm: A very nice web-based JSON editor - http://t.co/M70snaIf",
"from_user": "Leolik"
}
- text: |
Here we're making an object as before, but this time the `parents`
field is being set to `[.parents[].html_url]}]`, which collects
all of the parent commit URLs defined in the parents object.
Here we're making an object as before, but this time the urls
field is being set to `[.entities.urls[].url]`, which collects
all of the URLs defined in the entities object.
- text: |
@@ -319,4 +238,4 @@ body:
Here endeth the tutorial! There's lots more to play with, go
read [the manual](../manual/) if you're interested, and [download
jq](../download/) if you haven't already.
jq](../download/) if you haven't already.
docs/content/2.download/default.yml
+13 -27
View File
@@ -16,8 +16,7 @@ body:
### Linux
* jq 1.4 binaries for [64-bit](linux64/jq) or [32-bit](linux32/jq) systems.
* jq 1.3 binaries for [64-bit](linux64/jq-1.3/jq) or [32-bit](linux32/jq-1.3/jq) systems.
* Binaries for [64-bit](linux64/jq) or [32-bit](linux32/jq) systems.
* For Arch users, a PKGBUILD for jq-1.1 is in the
[AUR](https://aur.archlinux.org/packages.php?ID=63849), as
@@ -28,39 +27,31 @@ body:
manually by following instructions on [Arch Linux's
Wiki](https://wiki.archlinux.org/index.php/Arch_UseRepository).
* jq 1.4 is [in Debian](http://packages.debian.org/jq)
* jq 1.2 is [in Debian](http://packages.debian.org/jq)
### OS X
* `brew install jq` using [homebrew](http://mxcl.github.com/homebrew/)
* `brew install jq` using [homebrew](http://mxcl.github.io/homebrew/)
* Or, grab prebuilt jq 1.4 [64-bit binaries](osx64/jq) or [32-bit
* Or, grab prebuilt [64-bit binaries](osx64/jq) or [32-bit
binaries](osx32/jq)
* Or, grab prebuilt jq 1.3 [64-bit binaries](osx64/jq-1.3/jq) or [32-bit
binaries](osx32/jq-1.3/jq)
### Solaris 11
* jq 1.4 executables for [64-bit](solaris11-64/jq.exe) or [32-bit](solaris11-32/jq.exe)
### Windows
* jq 1.4 executables for [64-bit](win64/jq.exe) or [32-bit](win32/jq.exe)
* jq 1.3 executables for [64-bit](win64/jq-1.3/jq.exe) or [32-bit](win32/jq-1.3/jq.exe)
* Executables for [64-bit](win64/jq.exe) or [32-bit](win32/jq.exe)
### From source on Linux, OS X, Cygwin, and other POSIX-like operating systems
### From source on Linux or OS X
* [Source tarball for jq $JQ_VERSION](source/jq-$JQ_VERSION.tar.gz)
You can build it using the usual `./configure && make && sudo
make install` rigmarole.
If you're interested in using the lastest development version, try:
If you're interested in using the latest development version, try:
git clone https://github.com/stedolan/jq.git
cd jq
autoreconf -i
autoreconf
./configure
make
sudo make install
@@ -81,7 +72,7 @@ body:
On OS X, these are all included in Apple's command line tools, which
can be installed from [Xcode](http://developer.apple.com/technologies/tools/).
However, you may find that you need a newer version of Bison than the one provided
by Apple. This can be found in [Homebrew](http://mxcl.github.com/homebrew/)
by Apple. This can be found in [Homebrew](http://mxcl.github.io/homebrew/)
or [MacPorts](http://macports.org/).
`flex` and `bison` are used to generate the lexer and parser for
@@ -91,14 +82,9 @@ body:
#### Building the documentation
jq's documentation is compiled into static HTML using
[Bonsai](http://www.tinytree.info). To view the documentation
locally, run `rake serve` (or `bundle exec rake serve`) from the
docs/ subdirectory. To build the docs just `rake build` from the
docs subdirectory. You'll need a few Ruby dependencies, which can
be installed by following the instructions in `docs/README.md`.
The man page is built by `make jq.1`, or just `make`, also from
the YAML docs, and you'll still need the Ruby dependencies to
build the manpage.
[Bonsai](http://www.tinytree.info). To view the documentation
locally, run `rake serve` from the docs/ subdirectory. You'll
need a few Ruby dependencies, which can be installed by
following the instructions in `docs/README.md`.
docs/content/3.manual/manual.yml
+190 -1522
View File
File diff suppressed because it is too large. Load diff
docs/content/index/index.yml
+3 -9
View File
@@ -2,8 +2,7 @@ headline: jq
blurb: |
jq is a lightweight and flexible command-line JSON processor with
support for regular expressions.
jq is a lightweight and flexible command-line JSON processor.
body1: |
@@ -15,7 +14,7 @@ body2: |
jq is written in portable C, and it has zero runtime
dependencies. You can download a single binary, `scp` it to a far away
machine of the same type, and expect it to work.
machine, and expect it to work.
body3: |
@@ -28,13 +27,8 @@ tail: |
for *way* more.
news:
- date: 2014-06-08
body: |
jq 1.4 (finally) released! Get it on the [download](download/) page.
- date: 2013-05-19
body: |
jq 1.3 released.
jq 1.3 (finally) released! Get it on the [download](download/) page.
docs/site.yml
+1 -1
View File
@@ -4,7 +4,7 @@
# This line is modified by the Makefile. To change the version number,
# edit the Autoconf version number at the top of configure.ac
jq_version: "1.4-2-g15c4a7f-dirty"
jq_version: "1.3"
root: '/jq'
footer: |
docs/templates/index.liquid
Vendored
+1 -10
View File
@@ -10,7 +10,6 @@
<img src="{{root}}/jq.png" style="width: 400px; height: 220px; float: left; margin-right: 60px">
<div id="blurb">
{{blurb | markdownify}}
<div class="btn-group" style="float:right">
<a class="btn btn-primary dropdown-toggle" data-toggle="dropdown" href="#">
Download jq-{{jq_version}}
@@ -19,18 +18,10 @@
<ul class="dropdown-menu">
<li><a href="{{root}}/download/linux64/jq">Linux (64-bit)</a></li>
<li><a href="{{root}}/download/osx64/jq">OS X (64-bit)</a></li>
<li><a href="{{root}}/download/solaris11-64/jq">Solaris (64-bit)</a></li>
<li><a href="{{root}}/download/win64/jq.exe">Windows (64-bit)</a></li>
<li><a href="{{root}}/download/">Other platforms, older versions, and source</a></li>
<li><a href="{{root}}/download/">Other platforms and source</a></li>
</ul>
</div>
<div class="btn-group" style="float:right">
<a class="btn btn-primary" href="http://jqplay.org">
Try online!
</a>
</div>
</div>
<div style="clear:both;"></div>
</div>
docs/templates/manual.liquid
Vendored
+3 -3
View File
@@ -70,8 +70,8 @@
{% for example in entry.examples %}
<table class="manual-example">
<tr><th></th><td class="jqprogram">jq '{{example.program | escape}}'</td></tr>
<tr><th>Input</th><td>{{example.input | escape}}</td></tr>
<tr><th></th><td class="jqprogram">jq '{{example.program}}'</td></tr>
<tr><th>Input</th><td>{{example.input}}</td></tr>
{% unless example.output[0] %}
<tr>
<th>Output</th>
@@ -85,7 +85,7 @@
{% else %}
<th></th>
{% endif %}
<td>{{output | escape}}</td>
<td>{{output}}</td>
</tr>
{% endfor %}
docs/templates/shared/_header.liquid
Vendored
-2
View File
@@ -17,8 +17,6 @@
{% endfor %}
<li><a href="https://github.com/stedolan/jq/issues">Issues</a></li>
<li><a href="https://github.com/stedolan/jq">Source</a></li>
<li><a href="http://jqplay.org">Try online!</a></li>
<li><a href="https://raw.githubusercontent.com/stedolan/jq/master/NEWS">NEWS</a></li>
</ul>
</div>
</div>
exec_stack.h
-83
View File
@@ -1,83 +0,0 @@
#ifndef EXEC_STACK_H
#define EXEC_STACK_H
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "jv_alloc.h"
struct determine_alignment {
char x;
union { int i; double d; uint64_t u64; size_t sz; void* ptr; } u;
};
enum {ALIGNMENT = offsetof(struct determine_alignment, u)};
static size_t align_round_up(size_t sz) {
return ((sz + (ALIGNMENT - 1)) / ALIGNMENT) * ALIGNMENT;
}
typedef int stack_ptr;
struct stack {
char* mem_end; // one-past-the-end of allocated region
stack_ptr bound;
stack_ptr limit; // 0 - stack is empty
};
static void stack_init(struct stack* s) {
s->mem_end = 0;
s->bound = ALIGNMENT;
s->limit = 0;
}
static void stack_reset(struct stack* s) {
assert(s->limit == 0 && "stack freed while not empty");
char* mem_start = s->mem_end - ( -s->bound + ALIGNMENT);
free(mem_start);
stack_init(s);
}
static int stack_pop_will_free(struct stack* s, stack_ptr p) {
return p == s->limit;
}
static void* stack_block(struct stack* s, stack_ptr p) {
return (void*)(s->mem_end + p);
}
static stack_ptr* stack_block_next(struct stack* s, stack_ptr p) {
return &((stack_ptr*)stack_block(s, p))[-1];
}
static void stack_reallocate(struct stack* s, size_t sz) {
int old_mem_length = -(s->bound) + ALIGNMENT;
char* old_mem_start = s->mem_end - old_mem_length;
int new_mem_length = align_round_up((old_mem_length + sz + 256) * 2);
char* new_mem_start = jv_mem_realloc(old_mem_start, new_mem_length);
memmove(new_mem_start + (new_mem_length - old_mem_length),
new_mem_start, old_mem_length);
s->mem_end = new_mem_start + new_mem_length;
s->bound = -(new_mem_length - ALIGNMENT);
}
static stack_ptr stack_push_block(struct stack* s, stack_ptr p, size_t sz) {
int alloc_sz = align_round_up(sz) + ALIGNMENT;
stack_ptr r = s->limit - alloc_sz;
if (r < s->bound) {
stack_reallocate(s, alloc_sz);
}
s->limit = r;
*stack_block_next(s, r) = p;
return r;
}
static stack_ptr stack_pop_block(struct stack* s, stack_ptr p, size_t sz) {
stack_ptr r = *stack_block_next(s, p);
if (p == s->limit) {
int alloc_sz = align_round_up(sz) + ALIGNMENT;
s->limit += alloc_sz;
}
return r;
}
#endif
execute.c
+182 -649
View File
File diff suppressed because it is too large. Load diff
execute.h
+20
View File
@@ -0,0 +1,20 @@
#ifndef EXECUTE_H
#define EXECUTE_H
#include "bytecode.h"
struct bytecode* jq_compile(const char* str);
/* args must be an array of the form [{name:"foo", value:"thing"}, {name:"bar",value:3}] */
struct bytecode* jq_compile_args(const char* str, jv args);
typedef struct jq_state jq_state;
enum {JQ_DEBUG_TRACE = 1};
void jq_init(struct bytecode* bc, jv value, jq_state **, int flags);
jv jq_next(jq_state *);
void jq_teardown(jq_state **);
#endif
forkable_stack.h
+172
View File
@@ -0,0 +1,172 @@
#ifndef FORKABLE_STACK_H
#define FORKABLE_STACK_H
#include <stddef.h>
#include <assert.h>
#include <string.h>
#include "jv_alloc.h"
struct forkable_stack_header {
/* distance in bytes between this header and the header
of the next object on the stack */
int next_delta;
};
#define FORKABLE_STACK_HEADER struct forkable_stack_header fk_header_
struct forkable_stack {
// Stack grows down from stk+length
char* stk;
// stk+length is just past end of allocated area
// stk = jv_mem_alloc(length)
int length;
// stk+pos is header of top object, or stk+length if empty
int pos;
// everything after-or-including stk+savedlimit must be preserved
int savedlimit;
};
static void forkable_stack_check(struct forkable_stack* s) {
assert(s->stk);
assert(s->length > 0);
assert(s->pos >= 0 && s->pos <= s->length);
assert(s->savedlimit >= 0 && s->savedlimit <= s->length);
}
static int forkable_stack_empty(struct forkable_stack* s) {
return s->pos == s->length;
}
static void forkable_stack_init(struct forkable_stack* s, size_t sz) {
s->stk = jv_mem_alloc(sz);
s->length = sz;
s->pos = s->length;
s->savedlimit = s->length;
forkable_stack_check(s);
}
static void forkable_stack_free(struct forkable_stack* s) {
assert(forkable_stack_empty(s));
assert(s->savedlimit == s->length);
jv_mem_free(s->stk);
s->stk = 0;
}
static void* forkable_stack_push(struct forkable_stack* s, size_t sz_size) {
assert(sz_size > 0 && sz_size % sizeof(struct forkable_stack_header) == 0);
int size = (int)sz_size;
forkable_stack_check(s);
int curr = s->pos < s->savedlimit ? s->pos : s->savedlimit;
if (curr - size < 0) {
int oldlen = s->length;
s->length = (size + oldlen + 1024) * 2;
s->stk = jv_mem_realloc(s->stk, s->length);
int shift = s->length - oldlen;
memmove(s->stk + shift, s->stk, oldlen);
s->pos += shift;
s->savedlimit += shift;
curr += shift;
}
int newpos = curr - size;
assert(newpos < s->pos);
void* ret = (void*)(s->stk + newpos);
((struct forkable_stack_header*)ret)->next_delta = s->pos - newpos;
s->pos = newpos;
forkable_stack_check(s);
return ret;
}
static void* forkable_stack_peek(struct forkable_stack* s) {
assert(!forkable_stack_empty(s));
forkable_stack_check(s);
return (void*)(s->stk + s->pos);
}
static void* forkable_stack_peek_next(struct forkable_stack* s, void* top) {
forkable_stack_check(s);
struct forkable_stack_header* elem = top;
int elempos = (char*)top - s->stk;
assert(elempos >= 0 && elempos < s->length);
assert(elempos + elem->next_delta <= s->length);
if (elempos + elem->next_delta < s->length) {
return (void*)(s->stk + elempos + elem->next_delta);
} else {
return 0;
}
}
// Returns 1 if the next forkable_stack_pop will permanently remove an
// object from the stack (i.e. the top object was not saved with a fork)
static int forkable_stack_pop_will_free(struct forkable_stack* s) {
return s->pos < s->savedlimit;
}
static void forkable_stack_pop(struct forkable_stack* s) {
forkable_stack_check(s);
struct forkable_stack_header* elem = forkable_stack_peek(s);
int reclaim_upto = s->pos + elem->next_delta < s->savedlimit ?
s->pos + elem->next_delta : s->savedlimit;
assert(reclaim_upto <= s->length);
int reclaimed = reclaim_upto > s->pos ? reclaim_upto - s->pos : 0;
assert(0 <= reclaimed && s->pos + reclaimed <= s->length);
// s->pos <= i < reclaim_upto means that position i is to be freed
assert((reclaimed > 0) == forkable_stack_pop_will_free(s));
int new_pos = s->pos + elem->next_delta;
jv_mem_invalidate(elem, reclaimed);
s->pos = new_pos;
}
struct forkable_stack_state {
// We save the previous pos, savedlimit as
// length-pos, length-savedlimit since these
// values are stable across stack reallocations.
int prev_pos_delta, prev_limit_delta;
};
static void forkable_stack_save(struct forkable_stack* s, struct forkable_stack_state* state) {
forkable_stack_check(s);
state->prev_pos_delta = s->length - s->pos;
state->prev_limit_delta = s->length - s->savedlimit;
if (s->pos < s->savedlimit) s->savedlimit = s->pos;
}
static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stack_state* state) {
forkable_stack_check(s);
int curr_pos = s->pos;
s->pos = s->length - state->prev_pos_delta;
state->prev_pos_delta = s->length - curr_pos;
int curr_limit = s->savedlimit;
if (curr_pos < curr_limit) s->savedlimit = curr_pos;
//state->prevlimit = curr_limit; FIXME clean up
forkable_stack_check(s);
}
static void forkable_stack_restore(struct forkable_stack* s, struct forkable_stack_state* state) {
forkable_stack_check(s);
assert(s->savedlimit <= s->length - state->prev_pos_delta);
assert(s->savedlimit <= s->length - state->prev_limit_delta);
s->pos = s->length - state->prev_pos_delta;
s->savedlimit = s->length - state->prev_limit_delta;
forkable_stack_check(s);
}
typedef int stack_idx;
static stack_idx forkable_stack_to_idx(struct forkable_stack* s, void* ptr) {
char* item = ptr;
int pos = item - s->stk;
assert(pos >= 0 && pos < s->length);
return s->length - pos;
}
static void* forkable_stack_from_idx(struct forkable_stack* s, stack_idx idx) {
assert(idx >= 1 && idx <= s->length);
return &s->stk[s->length - idx];
}
#endif
frame_layout.h
+104
View File
@@ -0,0 +1,104 @@
#ifndef FRAME_LAYOUT_H
#include "forkable_stack.h"
#include "bytecode.h"
struct closure {
struct bytecode* bc;
stack_idx env;
};
struct continuation {
struct bytecode* bc;
stack_idx env;
uint16_t* retaddr;
};
typedef union frame_elem {
FORKABLE_STACK_HEADER;
struct continuation cont;
struct closure closure;
jv jsonval;
} *frame_ptr;
/*
* Frame layout
* fr[0] - FORKABLE_STACK_HEADER (next pointer)
* fr[1] - self (used to store return addresses, etc)
* fr[2...nclosures+2] - closure params
* fr[nclosures+2..nclosures+nlocals+2] - local variables
*/
static int frame_size(struct bytecode* bc) {
return sizeof(union frame_elem) * (bc->nclosures + bc->nlocals + 2);
}
static struct continuation* frame_self(frame_ptr fr) {
return &fr[1].cont;
}
static struct closure* frame_closure_arg(frame_ptr fr, int closure) {
assert(closure >= 0);
assert(closure < frame_self(fr)->bc->nclosures);
return &fr[2+closure].closure;
}
static jv* frame_local_var(frame_ptr fr, int var) {
assert(var >= 0);
assert(var < frame_self(fr)->bc->nlocals);
return &fr[2 + frame_self(fr)->bc->nclosures + var].jsonval;
}
static frame_ptr frame_current(struct forkable_stack* stk) {
frame_ptr fp = forkable_stack_peek(stk);
if (forkable_stack_peek_next(stk, fp)) {
struct bytecode* bc = frame_self(forkable_stack_peek_next(stk, fp))->bc;
assert(frame_self(fp)->retaddr >= bc->code && frame_self(fp)->retaddr < bc->code + bc->codelen);
} else {
assert(frame_self(fp)->retaddr == 0);
}
return fp;
}
static struct bytecode* frame_current_bytecode(struct forkable_stack* stk) {
return frame_self(frame_current(stk))->bc;
}
static uint16_t** frame_current_retaddr(struct forkable_stack* stk) {
return &frame_self(frame_current(stk))->retaddr;
}
static frame_ptr frame_get_parent(struct forkable_stack* stk, frame_ptr fr) {
return forkable_stack_from_idx(stk, frame_self(fr)->env);
}
static frame_ptr frame_get_level(struct forkable_stack* stk, frame_ptr fr, int level) {
for (int i=0; i<level; i++) {
fr = frame_get_parent(stk, fr);
}
return fr;
}
static frame_ptr frame_push(struct forkable_stack* stk, struct closure cl, uint16_t* retaddr) {
frame_ptr fp = forkable_stack_push(stk, frame_size(cl.bc));
struct continuation* cc = frame_self(fp);
cc->bc = cl.bc;
cc->env = cl.env;
cc->retaddr = retaddr;
for (int i=0; i<cl.bc->nlocals; i++) {
*frame_local_var(fp, i) = jv_invalid();
}
return fp;
}
static void frame_pop(struct forkable_stack* stk) {
frame_ptr fp = frame_current(stk);
if (forkable_stack_pop_will_free(stk)) {
int nlocals = frame_self(fp)->bc->nlocals;
for (int i=0; i<nlocals; i++) {
jv_free(*frame_local_var(fp, i));
}
}
forkable_stack_pop(stk);
}
#endif
scripts/gen_utf8_tables.py → gen_utf8_tables.py
+2 -2
View File
@@ -6,12 +6,13 @@ mask = lambda n: (1 << n) - 1
def print_table(type, name, t):
assert len(t) == 256
print("static const %s %s[] =" % (type, name))
first = True
for i in range(0,len(t),16):
print ((" {" if i == 0 else " ") +
", ".join("0x%02x"%n for n in t[i:i+16]) +
("," if i + 16 < len(t) else "};"))
("," if i + 16 < 256 else "};"))
def utf8info(c):
@@ -29,4 +30,3 @@ print("#define UTF8_CONTINUATION_BYTE ((unsigned char)255)")
print_table("unsigned char", "utf8_coding_length", table(0))
print_table("unsigned char", "utf8_coding_bits", table(1))
print_table("int", "utf8_first_codepoint", [0, 0x0, 0x80, 0x800, 0x10000])
jq.1.prebuilt
-2012
View File
File diff suppressed because it is too large. Load diff
jq.h
-40
View File
@@ -1,40 +0,0 @@
#ifndef _JQ_H_
#define _JQ_H_
#include <stdio.h>
#include <jv.h>
enum {JQ_DEBUG_TRACE = 1};
typedef struct jq_state jq_state;
typedef void (*jq_err_cb)(void *, jv);
jq_state *jq_init(void);
void jq_set_error_cb(jq_state *, jq_err_cb, void *);
void jq_get_error_cb(jq_state *, jq_err_cb *, void **);
void jq_set_nomem_handler(jq_state *, void (*)(void *), void *);
jv jq_format_error(jv msg);
void jq_report_error(jq_state *, jv);
int jq_compile(jq_state *, const char*);
int jq_compile_args(jq_state *, const char*, jv);
void jq_dump_disassembly(jq_state *, int);
void jq_start(jq_state *, jv value, int);
jv jq_next(jq_state *);
void jq_teardown(jq_state **);
typedef jv (*jq_input_cb)(jq_state *, void *);
void jq_set_input_cb(jq_state *, jq_input_cb, void *);
void jq_get_input_cb(jq_state *, jq_input_cb *, void **);
typedef void (*jq_debug_cb)(jq_state *, void *, jv);
void jq_set_debug_cb(jq_state *, jq_debug_cb, void *);
void jq_get_debug_cb(jq_state *, jq_debug_cb *, void **);
void jq_set_attrs(jq_state *, jv);
jv jq_get_attrs(jq_state *);
jv jq_get_jq_origin(jq_state *);
jv jq_get_prog_origin(jq_state *);
jv jq_get_lib_dirs(jq_state *);
void jq_set_attr(jq_state *, jv, jv);
jv jq_get_attr(jq_state *, jv);
#endif /* !_JQ_H_ */
jq.spec
+2 -7
View File
@@ -52,16 +52,11 @@ rm -rf %{buildroot}
%endif
%{_datadir}/doc/%{name}/AUTHORS
%{_datadir}/doc/%{name}/COPYING
%{_datadir}/doc/%{name}/INSTALL
%{_datadir}/doc/%{name}/NEWS
%{_datadir}/doc/%{name}/README
%{_datadir}/doc/%{name}/README.md
%{_datadir}/man/man1/jq.1
%{_includedir}/jq.h
%{_includedir}/jv.h
%{_prefix}/lib/libjq.a
%{_prefix}/lib/libjq.la
%{_prefix}/lib/libjq.so
%{_prefix}/lib/libjq.so.1
%{_prefix}/lib/libjq.so.1.0.4
%changelog
jq_parser.h
-2
View File
@@ -1,7 +1,5 @@
#ifndef JQ_PARSER_H
#define JQ_PARSER_H
#include "locfile.h"
#include "compile.h"
int jq_parse(struct locfile* source, block* answer);
int jq_parse_library(struct locfile* locations, block* answer);
jq_test.c
+27 -120
View File
@@ -3,23 +3,15 @@
#include <string.h>
#include <stdlib.h>
#include "jv.h"
#include "jq.h"
#include "execute.h"
static void jv_test();
static void run_jq_tests();
int jq_testsuite(jv libdirs, int argc, char* argv[]) {
FILE *testdata = stdin;
int jq_testsuite(int argc, char* argv[]) {
jv_test();
if (argc > 2) {
testdata = fopen(argv[2], "r");
if (!testdata) {
perror("fopen");
exit(1);
}
}
run_jq_tests(libdirs, testdata);
run_jq_tests(stdin);
return 0;
}
@@ -30,103 +22,36 @@ static int skipline(const char* buf) {
return 0;
}
static int checkfail(const char* buf) {
return !strcmp(buf, "%%FAIL\n");
}
struct err_data {
static void run_jq_tests(FILE *testdata) {
char buf[4096];
};
static void test_err_cb(void *data, jv e) {
struct err_data *err_data = data;
if (jv_get_kind(e) != JV_KIND_STRING)
e = jv_dump_string(e, JV_PRINT_INVALID);
if (!strncmp(jv_string_value(e), "jq: error", sizeof("jq: error") - 1))
snprintf(err_data->buf, sizeof(err_data->buf), "%s", jv_string_value(e));
if (strchr(err_data->buf, '\n'))
*(strchr(err_data->buf, '\n')) = '\0';
jv_free(e);
}
static void run_jq_tests(jv lib_dirs, FILE *testdata) {
char prog[4096];
char buf[4096];
struct err_data err_msg;
int tests = 0, passed = 0, invalid = 0;
unsigned int lineno = 0;
int must_fail = 0;
jq_state *jq = NULL;
jq = jq_init();
assert(jq);
if (jv_get_kind(lib_dirs) == JV_KIND_NULL)
lib_dirs = jv_array();
jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_dirs);
while (1) {
if (!fgets(prog, sizeof(prog), testdata)) break;
lineno++;
if (skipline(prog)) continue;
if (checkfail(prog)) {
must_fail = 1;
jq_set_error_cb(jq, test_err_cb, &err_msg);
continue;
}
if (prog[strlen(prog)-1] == '\n') prog[strlen(prog)-1] = 0;
printf("Testing '%s' at line number %u\n", prog, lineno);
if (!fgets(buf, sizeof(buf), testdata)) break;
if (skipline(buf)) continue;
if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0;
printf("Testing %s\n", buf);
int pass = 1;
tests++;
int compiled = jq_compile(jq, prog);
if (must_fail) {
jq_set_error_cb(jq, NULL, NULL);
must_fail = 0;
if (!fgets(buf, sizeof(buf), testdata)) { invalid++; break; }
lineno++;
if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0;
if (compiled) {
printf("*** Test program compiled that should not have at line %u: %s\n", lineno, prog);
invalid++; continue;
}
if (strcmp(buf, err_msg.buf)) {
printf("*** Erroneous test program failed with wrong message (%s) at line %u: %s\n", err_msg.buf, lineno, prog);
invalid++;
} else {
passed++;
}
continue;
}
if (!compiled) {
printf("*** Test program failed to compile at line %u: %s\n", lineno, prog);
invalid++;
// skip past test data
while (fgets(buf, sizeof(buf), testdata)) {
lineno++;
if (buf[0] == '\n' || (buf[0] == '\r' && buf[1] == '\n'))
break;
}
continue;
}
struct bytecode* bc = jq_compile(buf);
if (!bc) {invalid++; continue;}
printf("Disassembly:\n");
jq_dump_disassembly(jq, 2);
dump_disassembly(2, bc);
printf("\n");
if (!fgets(buf, sizeof(buf), testdata)) { invalid++; break; }
lineno++;
fgets(buf, sizeof(buf), testdata);
jv input = jv_parse(buf);
if (!jv_is_valid(input)){ invalid++; continue; }
jq_start(jq, input, JQ_DEBUG_TRACE);
jq_init(bc, input, &jq, JQ_DEBUG_TRACE);
while (fgets(buf, sizeof(buf), testdata)) {
lineno++;
if (skipline(buf)) break;
jv expected = jv_parse(buf);
if (!jv_is_valid(expected)){ invalid++; continue; }
jv actual = jq_next(jq);
if (!jv_is_valid(actual)) {
jv_free(actual);
printf("*** Insufficient results for test at line number %u: %s\n", lineno, prog);
printf("*** Insufficient results\n");
pass = 0;
break;
} else if (!jv_equal(jv_copy(expected), jv_copy(actual))) {
@@ -134,10 +59,10 @@ static void run_jq_tests(jv lib_dirs, FILE *testdata) {
jv_dump(jv_copy(expected), 0);
printf(", but got ");
jv_dump(jv_copy(actual), 0);
printf(" for test at line number %u: %s\n", lineno, prog);
printf("\n");
pass = 0;
}
jv as_string = jv_dump_string(jv_copy(expected), rand() & ~(JV_PRINT_COLOUR|JV_PRINT_REFCOUNT));
jv as_string = jv_dump_string(jv_copy(expected), rand() & ~JV_PRINT_COLOUR);
jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length_bytes(jv_copy(as_string)));
assert(jv_equal(jv_copy(expected), jv_copy(reparsed)));
jv_free(as_string);
@@ -150,29 +75,22 @@ static void run_jq_tests(jv lib_dirs, FILE *testdata) {
if (jv_is_valid(extra)) {
printf("*** Superfluous result: ");
jv_dump(extra, 0);
printf(" for test at line number %u, %s\n", lineno, prog);
printf("\n");
pass = 0;
} else {
jv_free(extra);
}
}
jq_teardown(&jq);
bytecode_free(bc);
passed+=pass;
}
jq_teardown(&jq);
printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
if (passed != tests) exit(1);
}
static void jv_test() {
/// JSON parser regression tests
{
jv v = jv_parse("{\"a':\"12\"}");
assert(jv_get_kind(v) == JV_KIND_INVALID);
v = jv_invalid_get_msg(v);
assert(strcmp(jv_string_value(v), "Expected separator between values at line 1, column 9 (while parsing '{\"a':\"12\"}')") == 0);
jv_free(v);
}
/// Arrays and numbers
{
jv a = jv_array();
@@ -197,9 +115,9 @@ static void jv_test() {
jv_free(a2);
assert(jv_get_refcnt(a) == 1);
assert(a.val.nontrivial.ptr->count == 1);
a = jv_array_append(a, jv_copy(a));
assert(jv_get_refcnt(a) == 1);
assert(a.val.nontrivial.ptr->count == 1);
assert(jv_array_length(jv_copy(a)) == 2);
assert(jv_number_value(jv_array_get(jv_copy(a), 0)) == 42);
@@ -232,9 +150,9 @@ static void jv_test() {
jv_free(subarray);
void* before = sub2.u.ptr;
void* before = sub2.val.nontrivial.ptr;
sub2 = jv_array_append(sub2, jv_number(200));
void* after = sub2.u.ptr;
void* after = sub2.val.nontrivial.ptr;
assert(before == after);
jv_free(sub2);
@@ -254,10 +172,8 @@ static void jv_test() {
a4 = jv_array_slice(a4, 0, 1);
assert(jv_array_length(jv_copy(a4)) == 1);
a4 = jv_array_append(a4, jv_number(4));
assert(jv_array_length(jv_copy(a4)) == 2);
assert(jv_array_length(jv_copy(a5)) == 2);
jv_free(a4);
jv_free(a5);
assert(jv_array_length(a4) == 2);
assert(jv_array_length(a5) == 2);
assert(jv_array_length(jv_copy(a)) == 2);
@@ -275,10 +191,8 @@ static void jv_test() {
assert(jv_equal(jv_string("foo"), jv_string_sized("foo", 3)));
char nasty[] = "foo\0";
jv shortstr = jv_string(nasty), longstr = jv_string_sized(nasty, sizeof(nasty));
assert(jv_string_length_bytes(jv_copy(shortstr)) == (int)strlen(nasty));
assert(jv_string_length_bytes(jv_copy(longstr)) == (int)sizeof(nasty));
jv_free(shortstr);
jv_free(longstr);
assert(jv_string_length_bytes(shortstr) == (int)strlen(nasty));
assert(jv_string_length_bytes(longstr) == (int)sizeof(nasty));
char a1s[] = "hello", a2s[] = "hello", bs[] = "goodbye";
@@ -324,11 +238,4 @@ static void jv_test() {
jv_free(o2);
}
/// Compile errors
{
jq_state *jq = jq_init();
jq_compile_args(jq, "}", jv_array());
jq_teardown(&jq);
}
}
jv.c
+270 -512
View File
File diff suppressed because it is too large. Load diff
jv.h
+29 -100
View File
@@ -1,9 +1,11 @@
#ifndef JV_H
#define JV_H
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <assert.h>
#include <stddef.h>
typedef enum {
JV_KIND_INVALID,
@@ -16,19 +18,21 @@ typedef enum {
JV_KIND_OBJECT
} jv_kind;
struct jv_refcnt;
/* All of the fields of this struct are private.
Really. Do not play with them. */
typedef struct {
unsigned char kind_flags;
unsigned char pad_;
unsigned short offset; /* array offsets */
int size;
size_t count;
} jv_refcnt;
typedef struct{
jv_refcnt* ptr;
int i[2];
} jv_nontrivial;
typedef struct {
jv_kind kind;
union {
struct jv_refcnt* ptr;
double number;
} u;
jv_nontrivial nontrivial;
} val;
} jv;
/*
@@ -43,27 +47,24 @@ static int jv_is_valid(jv x) { return jv_get_kind(x) != JV_KIND_INVALID; }
jv jv_copy(jv);
void jv_free(jv);
int jv_get_refcnt(jv);
int jv_equal(jv, jv);
int jv_contains(jv, jv);
jv jv_invalid(void);
jv jv_invalid();
jv jv_invalid_with_msg(jv);
jv jv_invalid_get_msg(jv);
int jv_invalid_has_msg(jv);
jv jv_null(void);
jv jv_true(void);
jv jv_false(void);
jv jv_null();
jv jv_true();
jv jv_false();
jv jv_bool(int);
jv jv_number(double);
double jv_number_value(jv);
int jv_is_integer(jv);
jv jv_array(void);
jv jv_array();
jv jv_array_sized(int);
int jv_array_length(jv);
jv jv_array_get(jv, int);
@@ -71,7 +72,6 @@ jv jv_array_set(jv, int, jv);
jv jv_array_append(jv, jv);
jv jv_array_concat(jv, jv);
jv jv_array_slice(jv, int, int);
jv jv_array_indexes(jv, jv);
#define jv_array_foreach(a, i, x) \
for (int jv_len__ = jv_array_length(jv_copy(a)), i=0, jv_j__ = 1; \
jv_j__; jv_j__ = 0) \
@@ -80,51 +80,23 @@ jv jv_array_indexes(jv, jv);
(x = jv_array_get(jv_copy(a), i), 1) : 0; \
i++)
#define JV_ARRAY_1(e) (jv_array_append(jv_array(),e))
#define JV_ARRAY_2(e1,e2) (jv_array_append(JV_ARRAY_1(e1),e2))
#define JV_ARRAY_3(e1,e2,e3) (jv_array_append(JV_ARRAY_2(e1,e2),e3))
#define JV_ARRAY_4(e1,e2,e3,e4) (jv_array_append(JV_ARRAY_3(e1,e2,e3),e4))
#define JV_ARRAY_5(e1,e2,e3,e4,e5) (jv_array_append(JV_ARRAY_4(e1,e2,e3,e4),e5))
#define JV_ARRAY_6(e1,e2,e3,e4,e5,e6) (jv_array_append(JV_ARRAY_5(e1,e2,e3,e4,e5),e6))
#define JV_ARRAY_7(e1,e2,e3,e4,e5,e6,e7) (jv_array_append(JV_ARRAY_6(e1,e2,e3,e4,e5,e6),e7))
#define JV_ARRAY_IDX(_1,_2,_3,_4,_5,_6,_7,NAME,...) NAME
#define JV_ARRAY(...) \
JV_ARRAY_IDX(__VA_ARGS__, JV_ARRAY_7, JV_ARRAY_6, JV_ARRAY_5, JV_ARRAY_4, JV_ARRAY_3, JV_ARRAY_2, JV_ARRAY_1)(__VA_ARGS__)
#ifdef __GNUC__
#define JV_PRINTF_LIKE(fmt_arg_num, args_num) \
__attribute__ ((__format__( __printf__, fmt_arg_num, args_num)))
#define JV_VPRINTF_LIKE(fmt_arg_num) \
__attribute__ ((__format__( __printf__, fmt_arg_num, 0)))
#endif
jv jv_string(const char*);
jv jv_string_sized(const char*, int);
jv jv_string_empty(int len);
int jv_string_length_bytes(jv);
int jv_string_length_codepoints(jv);
unsigned long jv_string_hash(jv);
uint32_t jv_string_hash(jv);
const char* jv_string_value(jv);
jv jv_string_indexes(jv j, jv k);
jv jv_string_slice(jv j, int start, int end);
jv jv_string_concat(jv, jv);
jv jv_string_vfmt(const char*, va_list) JV_VPRINTF_LIKE(1);
jv jv_string_fmt(const char*, ...) JV_PRINTF_LIKE(1, 2);
jv jv_string_append_codepoint(jv a, uint32_t c);
jv jv_string_fmt(const char*, ...);
jv jv_string_append_buf(jv a, const char* buf, int len);
jv jv_string_append_str(jv a, const char* str);
jv jv_string_split(jv j, jv sep);
jv jv_string_explode(jv j);
jv jv_string_implode(jv j);
jv jv_object(void);
jv jv_object();
jv jv_object_get(jv object, jv key);
jv jv_object_set(jv object, jv key, jv value);
jv jv_object_delete(jv object, jv key);
int jv_object_length(jv object);
jv jv_object_merge(jv, jv);
jv jv_object_merge_recursive(jv, jv);
int jv_object_iter(jv);
int jv_object_iter_next(jv, int);
@@ -140,68 +112,25 @@ jv jv_object_iter_value(jv, int);
1) \
: 0; \
jv_i__ = jv_object_iter_next(t, jv_i__)) \
#define JV_OBJECT_1(k) (jv_object_set(jv_object(),(k),jv_null()))
#define JV_OBJECT_2(k1,v1) (jv_object_set(jv_object(),(k1),(v1)))
#define JV_OBJECT_3(k1,v1,k2) (jv_object_set(JV_OBJECT_2(k1,v1),k2,jv_null()))
#define JV_OBJECT_4(k1,v1,k2,v2) (jv_object_set(JV_OBJECT_2(k1,v1),k2,v2))
#define JV_OBJECT_5(k1,v1,k2,v2,k3) (jv_object_set(JV_OBJECT_4(k1,v1,k2,v2),k3,jv_null))
#define JV_OBJECT_6(k1,v1,k2,v2,k3,v3) (jv_object_set(JV_OBJECT_4(k1,v1,k2,v2),k3,v3))
#define JV_OBJECT_7(k1,v1,k2,v2,k3,v3,k4) (jv_object_set(JV_OBJECT_6(k1,v1,k2,v2,k3,v3),k4,jv_null()))
#define JV_OBJECT_8(k1,v1,k2,v2,k3,v3,k4,v4) (jv_object_set(JV_OBJECT_6(k1,v1,k2,v2,k3,v3),k4,v4))
#define JV_OBJECT_IDX(_1,_2,_3,_4,_5,_6,_7,_8,NAME,...) NAME
#define JV_OBJECT(...) \
JV_OBJECT_IDX(__VA_ARGS__, JV_OBJECT_8, JV_OBJECT_7, JV_OBJECT_6, JV_OBJECT_5, JV_OBJECT_4, JV_OBJECT_3, JV_OBJECT_2, JV_OBJECT_1)(__VA_ARGS__)
int jv_get_refcnt(jv);
enum {
JV_PRINT_PRETTY = 1,
JV_PRINT_ASCII = 2,
JV_PRINT_COLOUR = 4,
JV_PRINT_SORTED = 8,
JV_PRINT_INVALID = 16,
JV_PRINT_REFCOUNT = 32,
};
void jv_dumpf(jv, FILE *f, int flags);
enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4 };
void jv_dump(jv, int flags);
void jv_show(jv, int flags);
jv jv_dump_string(jv, int flags);
enum {
JV_PARSE_SEQ = 1,
JV_PARSE_STREAMING = 2,
JV_PARSE_STREAM_ERRORS = 4,
};
jv jv_parse(const char* string);
jv jv_parse_sized(const char* string, int length);
typedef void (*jv_nomem_handler_f)(void *);
void jv_nomem_handler(jv_nomem_handler_f, void *);
jv jv_load_file(const char *, int);
struct jv_parser;
struct jv_parser* jv_parser_new(int);
void jv_parser_set_buf(struct jv_parser*, const char*, int, int);
int jv_parser_remaining(struct jv_parser*);
jv jv_parser_next(struct jv_parser*);
void jv_parser_free(struct jv_parser*);
jv jv_get(jv, jv);
jv jv_set(jv, jv, jv);
jv jv_has(jv, jv);
jv jv_setpath(jv, jv, jv);
jv jv_getpath(jv, jv);
jv jv_delpaths(jv, jv);
jv jv_keys(jv /*object or array*/);
jv jv_keys_unsorted(jv /*object or array*/);
int jv_cmp(jv, jv);
jv jv_group(jv, jv);
jv jv_sort(jv, jv);
#endif
jv_alloc.c
-114
View File
@@ -2,121 +2,11 @@
#include <stdio.h>
#include "jv_alloc.h"
struct nomem_handler {
jv_nomem_handler_f handler;
void *data;
};
#if !defined(HAVE_PTHREAD_KEY_CREATE) || \
!defined(HAVE_PTHREAD_ONCE) || \
!defined(HAVE_ATEXIT)
/* Try thread-local storage? */
#ifdef _MSC_VER
/* Visual C++: yes */
static __declspec(thread) struct nomem_handler nomem_handler;
#define USE_TLS
#else
#ifdef HAVE___THREAD
/* GCC and friends: yes */
static __thread struct nomem_handler nomem_handler;
#define USE_TLS
#endif /* HAVE___THREAD */
#endif /* _MSC_VER */
#endif /* !HAVE_PTHREAD_KEY_CREATE */
#ifdef USE_TLS
void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
nomem_handler.handler = handler;
}
static void memory_exhausted() {
if (nomem_handler.handler)
nomem_handler.handler(nomem_handler.data); // Maybe handler() will longjmp() to safety
// Or not
fprintf(stderr, "error: cannot allocate memory\n");
abort();
}
#else /* USE_TLS */
#ifdef HAVE_PTHREAD_KEY_CREATE
#include <pthread.h>
pthread_key_t nomem_handler_key;
pthread_once_t mem_once = PTHREAD_ONCE_INIT;
static void tsd_fini(void) {
struct nomem_handler *nomem_handler;
nomem_handler = pthread_getspecific(nomem_handler_key);
if (nomem_handler) {
(void) pthread_setspecific(nomem_handler_key, NULL);
free(nomem_handler);
}
}
static void tsd_init(void) {
if (pthread_key_create(&nomem_handler_key, NULL) != 0) {
fprintf(stderr, "error: cannot create thread specific key");
abort();
}
if (atexit(tsd_fini) != 0) {
fprintf(stderr, "error: cannot set an exit handler");
abort();
}
struct nomem_handler *nomem_handler = calloc(1, sizeof(struct nomem_handler));
if (pthread_setspecific(nomem_handler_key, nomem_handler) != 0) {
fprintf(stderr, "error: cannot set thread specific data");
abort();
}
}
void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
pthread_once(&mem_once, tsd_init); // cannot fail
struct nomem_handler *nomem_handler;
nomem_handler = pthread_getspecific(nomem_handler_key);
if (nomem_handler == NULL) {
handler(data);
fprintf(stderr, "error: cannot allocate memory\n");
abort();
}
nomem_handler->handler = handler;
nomem_handler->data = data;
}
static void memory_exhausted() {
struct nomem_handler *nomem_handler;
pthread_once(&mem_once, tsd_init);
nomem_handler = pthread_getspecific(nomem_handler_key);
if (nomem_handler)
nomem_handler->handler(nomem_handler->data); // Maybe handler() will longjmp() to safety
// Or not
fprintf(stderr, "error: cannot allocate memory\n");
abort();
}
#else
/* No thread-local storage of any kind that we know how to handle */
static struct nomem_handler nomem_handler;
void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
nomem_handler.handler = handler;
nomem_handler.data = data;
}
static void memory_exhausted() {
fprintf(stderr, "error: cannot allocate memory\n");
abort();
}
#endif /* HAVE_PTHREAD_KEY_CREATE */
#endif /* USE_TLS */
void* jv_mem_alloc(size_t sz) {
void* p = malloc(sz);
if (!p) {
@@ -125,10 +15,6 @@ void* jv_mem_alloc(size_t sz) {
return p;
}
void* jv_mem_alloc_unguarded(size_t sz) {
return malloc(sz);
}
void jv_mem_free(void* p) {
free(p);
}
jv_alloc.h
-2
View File
@@ -2,7 +2,6 @@
#define JV_ALLOC_H
#include <stddef.h>
#include "jv.h"
#ifndef NDEBUG
extern volatile char jv_mem_uninitialised;
@@ -16,7 +15,6 @@ static void jv_mem_invalidate(void* mem, size_t n) {
}
void* jv_mem_alloc(size_t);
void* jv_mem_alloc_unguarded(size_t);
void jv_mem_free(void*);
__attribute__((warn_unused_result)) void* jv_mem_realloc(void*, size_t);
jv_aux.c
+29 -81
View File
@@ -1,21 +1,17 @@
#include "jv_aux.h"
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include "jv_alloc.h"
static int parse_slice(jv j, jv slice, int* pstart, int* pend) {
static int parse_slice(jv array, jv slice, int* pstart, int* pend) {
// Array slices
int len = jv_array_length(jv_copy(array));
jv start_jv = jv_object_get(jv_copy(slice), jv_string("start"));
jv end_jv = jv_object_get(slice, jv_string("end"));
if (jv_get_kind(start_jv) == JV_KIND_NULL) {
jv_free(start_jv);
start_jv = jv_number(0);
}
int len;
if (jv_get_kind(j) == JV_KIND_ARRAY)
len = jv_array_length(jv_copy(j));
else
len = jv_string_length_codepoints(jv_copy(j));
if (jv_get_kind(end_jv) == JV_KIND_NULL) {
jv_free(end_jv);
end_jv = jv_number(len);
@@ -26,18 +22,13 @@ static int parse_slice(jv j, jv slice, int* pstart, int* pend) {
jv_free(end_jv);
return 0;
} else {
double dstart = jv_number_value(start_jv);
double dend = jv_number_value(end_jv);
if (dstart < 0) dstart += len;
if (dend < 0) dend += len;
if (dstart < 0) dstart = 0;
if (dstart > len) dstart = len;
int start = (int)dstart;
int end = (dend > len) ? len : (int)dend;
// Ends are exclusive but e.g. 1 < 1.5 so :1.5 should be :2 not :1
if(end < dend) end += 1;
int start = (int)jv_number_value(start_jv);
int end = (int)jv_number_value(end_jv);
if (start < 0) start = len + start;
if (end < 0) end = len + end;
if (start < 0) start = 0;
if (start > len) start = len;
if (end > len) end = len;
if (end < start) end = start;
assert(0 <= start && start <= end && end <= len);
@@ -56,18 +47,10 @@ jv jv_get(jv t, jv k) {
v = jv_null();
}
} else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_NUMBER) {
if(jv_is_integer(k)){
int idx = (int)jv_number_value(k);
if (idx < 0)
idx += jv_array_length(jv_copy(t));
v = jv_array_get(t, idx);
if (!jv_is_valid(v)) {
jv_free(v);
v = jv_null();
}
} else {
jv_free(t);
jv_free(k);
// FIXME: don't do lookup for noninteger index
v = jv_array_get(t, (int)jv_number_value(k));
if (!jv_is_valid(v)) {
jv_free(v);
v = jv_null();
}
} else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_OBJECT) {
@@ -78,16 +61,6 @@ jv jv_get(jv t, jv k) {
v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers"));
jv_free(t);
}
} else if (jv_get_kind(t) == JV_KIND_STRING && jv_get_kind(k) == JV_KIND_OBJECT) {
int start, end;
if (parse_slice(t, k, &start, &end)) {
v = jv_string_slice(t, start, end);
} else {
v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an string slice must be numbers"));
jv_free(t);
}
} else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_ARRAY) {
v = jv_array_indexes(t, k);
} else if (jv_get_kind(t) == JV_KIND_NULL &&
(jv_get_kind(k) == JV_KIND_STRING ||
jv_get_kind(k) == JV_KIND_NUMBER ||
@@ -96,20 +69,9 @@ jv jv_get(jv t, jv k) {
jv_free(k);
v = jv_null();
} else {
/*
* If k is a short string it's probably from a jq .foo expression or
* similar, in which case putting it in the invalid msg may help the
* user. The length 30 is arbitrary.
*/
if (jv_get_kind(k) == JV_KIND_STRING && jv_string_length_bytes(jv_copy(k)) < 30) {
v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with string \"%s\"",
jv_kind_name(jv_get_kind(t)),
jv_string_value(k)));
} else {
v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with %s",
jv_kind_name(jv_get_kind(t)),
jv_kind_name(jv_get_kind(k))));
}
v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with %s",
jv_kind_name(jv_get_kind(t)),
jv_kind_name(jv_get_kind(k))));
jv_free(t);
jv_free(k);
}
@@ -426,18 +388,6 @@ static int string_cmp(const void* pa, const void* pb){
return r;
}
jv jv_keys_unsorted(jv x) {
if (jv_get_kind(x) != JV_KIND_OBJECT)
return jv_keys(x);
jv answer = jv_array_sized(jv_object_length(jv_copy(x)));
jv_object_foreach(x, key, value) {
answer = jv_array_append(answer, key);
jv_free(value);
}
jv_free(x);
return answer;
}
jv jv_keys(jv x) {
if (jv_get_kind(x) == JV_KIND_OBJECT) {
int nkeys = jv_object_length(jv_copy(x));
@@ -595,23 +545,21 @@ jv jv_group(jv objects, jv keys) {
int n = jv_array_length(jv_copy(objects));
struct sort_entry* entries = sort_items(objects, keys);
jv ret = jv_array();
if (n > 0) {
jv curr_key = entries[0].key;
jv group = jv_array_append(jv_array(), entries[0].object);
for (int i = 1; i < n; i++) {
if (jv_equal(jv_copy(curr_key), jv_copy(entries[i].key))) {
jv_free(entries[i].key);
} else {
jv_free(curr_key);
curr_key = entries[i].key;
ret = jv_array_append(ret, group);
group = jv_array();
}
group = jv_array_append(group, entries[i].object);
jv curr_key = entries[0].key;
jv group = jv_array_append(jv_array(), entries[0].object);
for (int i = 1; i < n; i++) {
if (jv_equal(jv_copy(curr_key), jv_copy(entries[i].key))) {
jv_free(entries[i].key);
} else {
jv_free(curr_key);
curr_key = entries[i].key;
ret = jv_array_append(ret, group);
group = jv_array();
}
jv_free(curr_key);
ret = jv_array_append(ret, group);
group = jv_array_append(group, entries[i].object);
}
jv_free(curr_key);
ret = jv_array_append(ret, group);
jv_mem_free(entries);
return ret;
}
jv_aux.h
+20
View File
@@ -0,0 +1,20 @@
#ifndef JV_AUX_H
#define JV_AUX_H
#include "jv.h"
jv jv_get(jv t, jv k);
jv jv_set(jv t, jv k, jv v);
jv jv_has(jv t, jv k);
jv jv_setpath(jv root, jv path, jv value);
jv jv_getpath(jv root, jv path);
jv jv_delpaths(jv root, jv paths);
jv jv_keys(jv /*object or array*/);
int jv_cmp(jv, jv);
jv jv_group(jv objects, jv keys);
jv jv_sort(jv objects, jv keys);
#endif
jv_dtoa.c
+2 -1
View File
@@ -187,6 +187,7 @@
* used for input more than STRTOD_DIGLIM digits long (default 40).
*/
#define IEEE_8087
#define NO_ERRNO
#define NO_HEX_FP
#define No_Hex_NaN
@@ -4224,7 +4225,7 @@ jvp_dtoa
while((*b++ = *s++));
goto done0;
}
if (decpt <= -4 || decpt > se - s + 15) {
if (decpt <= -4 || decpt > se - s + 5) {
*b++ = *s++;
if (*s) {
*b++ = '.';
jv_dtoa.h
+1 -1
View File
@@ -17,6 +17,6 @@ double jvp_strtod(struct dtoa_context* C, const char* s, char** se);
char* jvp_dtoa(struct dtoa_context* C, double dd, int mode, int ndigits, int *decpt, int *sign, char **rve);
void jvp_freedtoa(struct dtoa_context* C, char *s);
#define JVP_DTOA_FMT_MAX_LEN 64
#define JVP_DTOA_FMT_MAX_LEN 32
char* jvp_dtoa_fmt(struct dtoa_context* C, register char *b, double x);
#endif
jv_file.c
-46
View File
@@ -1,46 +0,0 @@
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "jv.h"
jv jv_load_file(const char* filename, int raw) {
FILE* file = fopen(filename, "r");
struct jv_parser* parser;
jv data;
if (!file) {
return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
filename,
strerror(errno)));
}
if (raw) {
data = jv_string("");
} else {
data = jv_array();
parser = jv_parser_new(0);
}
while (!feof(file) && !ferror(file)) {
char buf[4096];
size_t n = fread(buf, 1, sizeof(buf), file);
if (raw) {
data = jv_string_concat(data, jv_string_sized(buf, (int)n));
} else {
jv_parser_set_buf(parser, buf, n, !feof(file));
jv value;
while (jv_is_valid((value = jv_parser_next(parser))))
data = jv_array_append(data, value);
jv_free(value);
}
}
if (!raw)
jv_parser_free(parser);
int badread = ferror(file);
fclose(file);
if (badread) {
jv_free(data);
return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
filename));
}
return data;
}
jv_parse.c
+21 -395
View File
@@ -1,12 +1,11 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "jv.h"
#include "jv_dtoa.h"
#include "jv_parse.h"
#include "jv_unicode.h"
#include "jv_alloc.h"
#include "jv_dtoa.h"
typedef const char* presult;
@@ -17,62 +16,9 @@ typedef const char* presult;
#define pfunc presult
#endif
enum last_seen {
JV_LAST_NONE = 0,
JV_LAST_OPEN_ARRAY = '[',
JV_LAST_OPEN_OBJECT = '{',
JV_LAST_COLON = ':',
JV_LAST_COMMA = ',',
JV_LAST_VALUE = 'V',
};
struct jv_parser {
const char* curr_buf;
int curr_buf_length;
int curr_buf_pos;
int curr_buf_is_partial;
unsigned bom_strip_position;
int flags;
jv* stack; // parser
int stackpos; // parser
int stacklen; // both (optimization; it's really pathlen for streaming)
jv path; // streamer
enum last_seen last_seen; // streamer
jv output; // streamer
jv next; // both
char* tokenbuf;
int tokenpos;
int tokenlen;
int line, column;
struct dtoa_context dtoa;
enum {
JV_PARSER_NORMAL,
JV_PARSER_STRING,
JV_PARSER_STRING_ESCAPE,
JV_PARSER_WAITING_FOR_RS // parse error, waiting for RS
} st;
unsigned int last_ch_was_ws:1;
};
static void parser_init(struct jv_parser* p, int flags) {
p->flags = flags;
if ((p->flags & JV_PARSE_STREAMING)) {
p->path = jv_array();
} else {
p->path = jv_invalid();
p->flags &= ~(JV_PARSE_STREAM_ERRORS);
}
void jv_parser_init(struct jv_parser* p) {
p->stack = 0;
p->stacklen = p->stackpos = 0;
p->last_seen = JV_LAST_NONE;
p->output = jv_invalid();
p->next = jv_invalid();
p->tokenbuf = 0;
p->tokenlen = p->tokenpos = 0;
@@ -80,50 +26,22 @@ static void parser_init(struct jv_parser* p, int flags) {
p->curr_buf = 0;
p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0;
p->bom_strip_position = 0;
p->last_ch_was_ws = 0;
p->line = 1;
p->column = 0;
jvp_dtoa_context_init(&p->dtoa);
}
static void parser_reset(struct jv_parser* p) {
if ((p->flags & JV_PARSE_STREAMING)) {
jv_free(p->path);
p->path = jv_array();
p->stacklen = 0;
}
p->last_seen = JV_LAST_NONE;
jv_free(p->output);
p->output = jv_invalid();
void jv_parser_free(struct jv_parser* p) {
jv_free(p->next);
p->next = jv_invalid();
for (int i=0; i<p->stackpos; i++)
jv_free(p->stack[i]);
p->stackpos = 0;
p->tokenpos = 0;
p->st = JV_PARSER_NORMAL;
}
static void parser_free(struct jv_parser* p) {
parser_reset(p);
jv_free(p->path);
jv_free(p->output);
jv_mem_free(p->stack);
jv_mem_free(p->tokenbuf);
jvp_dtoa_context_free(&p->dtoa);
}
static pfunc value(struct jv_parser* p, jv val) {
if ((p->flags & JV_PARSE_STREAMING)) {
if (jv_is_valid(p->next) || p->last_seen == JV_LAST_VALUE)
return "Expected separator between values";
if (p->stacklen > 0)
p->last_seen = JV_LAST_VALUE;
else
p->last_seen = JV_LAST_NONE;
} else {
if (jv_is_valid(p->next)) return "Expected separator between values";
}
if (jv_is_valid(p->next)) return "Expected separator between values";
jv_free(p->next);
p->next = val;
return 0;
@@ -139,7 +57,7 @@ static void push(struct jv_parser* p, jv v) {
p->stack[p->stackpos++] = v;
}
static pfunc parse_token(struct jv_parser* p, char ch) {
static pfunc token(struct jv_parser* p, char ch) {
switch (ch) {
case '[':
if (jv_is_valid(p->next)) return "Expected separator between values";
@@ -222,159 +140,6 @@ static pfunc parse_token(struct jv_parser* p, char ch) {
return 0;
}
static pfunc stream_token(struct jv_parser* p, char ch) {
jv_kind k;
jv last;
switch (ch) {
case '[':
if (jv_is_valid(p->next))
return "Expected a separator between values";
p->path = jv_array_append(p->path, jv_number(0)); // push
p->last_seen = JV_LAST_OPEN_ARRAY;
p->stacklen++;
break;
case '{':
if (p->last_seen == JV_LAST_VALUE)
return "Expected a separator between values";
// Push object key: null, since we don't know it yet
p->path = jv_array_append(p->path, jv_null()); // push
p->last_seen = JV_LAST_OPEN_OBJECT;
p->stacklen++;
break;
case ':':
if (p->stacklen == 0 || jv_get_kind(jv_array_get(jv_copy(p->path), p->stacklen - 1)) == JV_KIND_NUMBER)
return "':' not as part of an object";
if (!jv_is_valid(p->next) || p->last_seen == JV_LAST_NONE)
return "Expected string key before ':'";
if (jv_get_kind(p->next) != JV_KIND_STRING)
return "Object keys must be strings";
if (p->last_seen != JV_LAST_VALUE)
return "':' should follow a key";
p->last_seen = JV_LAST_COLON;
p->path = jv_array_set(p->path, p->stacklen - 1, p->next);
p->next = jv_invalid();
break;
case ',':
if (p->last_seen != JV_LAST_VALUE)
return "Expected value before ','";
if (p->stacklen == 0)
return "',' not as part of an object or array";
last = jv_array_get(jv_copy(p->path), p->stacklen - 1);
k = jv_get_kind(last);
if (k == JV_KIND_NUMBER) {
int idx = jv_number_value(last);
if (jv_is_valid(p->next)) {
p->output = JV_ARRAY(jv_copy(p->path), p->next);
p->next = jv_invalid();
}
p->path = jv_array_set(p->path, p->stacklen - 1, jv_number(idx + 1));
p->last_seen = JV_LAST_COMMA;
} else if (k == JV_KIND_STRING) {
if (jv_is_valid(p->next)) {
p->output = JV_ARRAY(jv_copy(p->path), p->next);
p->next = jv_invalid();
}
p->path = jv_array_set(p->path, p->stacklen - 1, jv_true()); // ready for another name:value pair
p->last_seen = JV_LAST_COMMA;
} else {
assert(k == JV_KIND_NULL);
// this case hits on input like {,}
// make sure to handle input like {"a", "b"} and {"a":, ...}
jv_free(last);
return "Objects must consist of key:value pairs";
}
jv_free(last);
break;
case ']':
if (p->stacklen == 0)
return "Unmatched ']' at the top-level";
if (p->last_seen == JV_LAST_COMMA)
return "Expected another array element";
if (p->last_seen == JV_LAST_OPEN_ARRAY)
assert(!jv_is_valid(p->next));
last = jv_array_get(jv_copy(p->path), p->stacklen - 1);
k = jv_get_kind(last);
jv_free(last);
if (k != JV_KIND_NUMBER)
return "Unmatched ']' in the middle of an object";
if (jv_is_valid(p->next)) {
p->output = JV_ARRAY(jv_copy(p->path), p->next, jv_true());
p->next = jv_invalid();
} else if (p->last_seen != JV_LAST_OPEN_ARRAY) {
p->output = JV_ARRAY(jv_copy(p->path));
}
p->path = jv_array_slice(p->path, 0, --(p->stacklen)); // pop
//assert(!jv_is_valid(p->next));
jv_free(p->next);
p->next = jv_invalid();
if (p->last_seen == JV_LAST_OPEN_ARRAY)
p->output = JV_ARRAY(jv_copy(p->path), jv_array()); // Empty arrays are leaves
if (p->stacklen == 0)
p->last_seen = JV_LAST_NONE;
else
p->last_seen = JV_LAST_VALUE;
break;
case '}':
if (p->stacklen == 0)
return "Unmatched '}' at the top-level";
if (p->last_seen == JV_LAST_COMMA)
return "Expected another key:value pair";
if (p->last_seen == JV_LAST_OPEN_OBJECT)
assert(!jv_is_valid(p->next));
last = jv_array_get(jv_copy(p->path), p->stacklen - 1);
k = jv_get_kind(last);
jv_free(last);
if (k == JV_KIND_NUMBER)
return "Unmatched '}' in the middle of an array";
if (jv_is_valid(p->next)) {
if (k != JV_KIND_STRING)
return "Objects must consist of key:value pairs";
p->output = JV_ARRAY(jv_copy(p->path), p->next, jv_true());
p->next = jv_invalid();
} else {
// Perhaps {"a":[]}
if (p->last_seen == JV_LAST_COLON)
// Looks like {"a":}
return "Missing value in key:value pair";
if (p->last_seen == JV_LAST_COMMA)
// Looks like {"a":0,}
return "Expected another key-value pair";
if (p->last_seen == JV_LAST_OPEN_ARRAY)
return "Unmatched '}' in the middle of an array";
if (p->last_seen != JV_LAST_VALUE && p->last_seen != JV_LAST_OPEN_OBJECT)
return "Unmatched '}'";
if (p->last_seen != JV_LAST_OPEN_OBJECT)
p->output = JV_ARRAY(jv_copy(p->path));
}
p->path = jv_array_slice(p->path, 0, --(p->stacklen)); // pop
jv_free(p->next);
p->next = jv_invalid();
if (p->last_seen == JV_LAST_OPEN_OBJECT)
p->output = JV_ARRAY(jv_copy(p->path), jv_object()); // Empty arrays are leaves
if (p->stacklen == 0)
p->last_seen = JV_LAST_NONE;
else
p->last_seen = JV_LAST_VALUE;
break;
}
return 0;
}
static void tokenadd(struct jv_parser* p, char c) {
assert(p->tokenpos <= p->tokenlen);
@@ -450,8 +215,6 @@ static pfunc found_string(struct jv_parser* p) {
return "Invalid escape";
}
} else {
if (c > 0 && c < 0x001f)
return "Invalid string: control characters from U+0000 through U+001F must be escaped";
*out++ = c;
}
}
@@ -522,7 +285,7 @@ static chclass classify(char c) {
static const presult OK = "output produced";
static int parse_check_done(struct jv_parser* p, jv* out) {
static int check_done(struct jv_parser* p, jv* out) {
if (p->stackpos == 0 && jv_is_valid(p->next)) {
*out = p->next;
p->next = jv_invalid();
@@ -532,77 +295,15 @@ static int parse_check_done(struct jv_parser* p, jv* out) {
}
}
static int stream_check_done(struct jv_parser* p, jv* out) {
if (p->stacklen == 0 && jv_is_valid(p->next)) {
*out = JV_ARRAY(jv_copy(p->path),p->next);
p->next = jv_invalid();
return 1;
} else if (jv_is_valid(p->output)) {
if (jv_array_length(jv_copy(p->output)) > 2) {
// At end of an array or object, necessitating one more output by
// which to indicate this
*out = jv_array_slice(jv_copy(p->output), 0, 2);
p->output = jv_array_slice(p->output, 0, 1); // arrange one more output
} else {
// No further processing needed
*out = p->output;
p->output = jv_invalid();
}
return 1;
} else {
return 0;
}
}
static int parse_check_truncation(struct jv_parser* p, jv out) {
return ((p->flags & JV_PARSE_SEQ) && !p->last_ch_was_ws && jv_get_kind(out) == JV_KIND_NUMBER);
}
static int stream_check_truncation(struct jv_parser* p, jv out) {
if (!jv_is_valid(out))
return 0;
jv v = jv_array_get(jv_copy(out), 1);
jv_kind k = jv_get_kind(v);
jv_free(v);
return (k == JV_KIND_NUMBER || k == JV_KIND_TRUE || k == JV_KIND_FALSE || k == JV_KIND_NULL);
}
#define check_done(p, out) \
(((p)->flags & JV_PARSE_STREAMING) ? stream_check_done((p), (out)) : parse_check_done((p), (out)))
#define token(p, ch) \
(((p)->flags & JV_PARSE_STREAMING) ? stream_token((p), (ch)) : parse_token((p), (ch)))
#define check_truncation(p, o) \
(((p)->flags & JV_PARSE_STREAMING) ? stream_check_truncation((p), (o)) : parse_check_truncation((p), (o)))
static pfunc scan(struct jv_parser* p, char ch, jv* out) {
p->column++;
if (ch == '\n') {
p->line++;
p->column = 0;
}
if (ch == '\036' /* ASCII RS; see draft-ietf-json-sequence-07 */) {
TRY(check_literal(p));
if (p->st == JV_PARSER_NORMAL && check_done(p, out)) {
if (check_truncation(p, *out)) {
jv_free(*out);
*out = jv_invalid();
return "Potentially truncated top-level numeric value";
}
return OK;
}
parser_reset(p);
jv_free(*out);
*out = jv_invalid();
return "Truncated value";
}
presult answer = 0;
p->last_ch_was_ws = 0;
if (p->st == JV_PARSER_NORMAL) {
chclass cls = classify(ch);
if (cls == WHITESPACE)
p->last_ch_was_ws = 1;
if (cls != LITERAL) {
TRY(check_literal(p));
if (check_done(p, out)) answer = OK;
@@ -640,30 +341,12 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
return answer;
}
struct jv_parser* jv_parser_new(int flags) {
struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser));
parser_init(p, flags);
p->flags = flags;
return p;
}
void jv_parser_free(struct jv_parser* p) {
parser_free(p);
jv_mem_free(p);
}
static const unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};
int jv_parser_remaining(struct jv_parser* p) {
if (p->curr_buf == 0)
return 0;
return (p->curr_buf_length - p->curr_buf_pos);
}
void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) {
assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length)
&& "previous buffer not exhausted");
while (length > 0 && p->bom_strip_position < sizeof(UTF8_BOM)) {
while (p->bom_strip_position < sizeof(UTF8_BOM)) {
if ((unsigned char)*buf == UTF8_BOM[p->bom_strip_position]) {
// matched a BOM character
buf++;
@@ -685,100 +368,43 @@ void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_
p->curr_buf_is_partial = is_partial;
}
static jv make_error(struct jv_parser*, const char *, ...) JV_PRINTF_LIKE(2, 3);
static jv make_error(struct jv_parser* p, const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
jv e = jv_string_vfmt(fmt, ap);
va_end(ap);
if ((p->flags & JV_PARSE_STREAM_ERRORS))
return JV_ARRAY(e, jv_copy(p->path));
return jv_invalid_with_msg(e);
}
jv jv_parser_next(struct jv_parser* p) {
if (!p->curr_buf)
return jv_invalid(); // Need a buffer
assert(p->curr_buf && "a buffer must be provided");
if (p->bom_strip_position == 0xff) return jv_invalid_with_msg(jv_string("Malformed BOM"));
jv value = jv_invalid();
if ((p->flags & JV_PARSE_STREAMING) && stream_check_done(p, &value))
return value;
char ch;
jv value;
presult msg = 0;
while (!msg && p->curr_buf_pos < p->curr_buf_length) {
ch = p->curr_buf[p->curr_buf_pos++];
if (ch != '\036' && p->st == JV_PARSER_WAITING_FOR_RS)
continue; // need to resync, wait for RS
char ch = p->curr_buf[p->curr_buf_pos++];
msg = scan(p, ch, &value);
}
if (msg == OK) {
return value;
} else if (msg) {
jv_free(value);
if (ch != '\036' && (p->flags & JV_PARSE_SEQ)) {
// Skip to the next RS
p->st = JV_PARSER_WAITING_FOR_RS;
value = make_error(p, "%s at line %d, column %d (need RS to resync)", msg, p->line, p->column);
parser_reset(p);
return value;
}
value = make_error(p, "%s at line %d, column %d", msg, p->line, p->column);
parser_reset(p);
if (!(p->flags & JV_PARSE_SEQ)) {
// We're not parsing a JSON text sequence; throw this buffer away.
p->curr_buf = 0;
p->curr_buf_pos = 0;
} // Else ch must be RS; don't clear buf so we can start parsing again after this ch
return value;
return jv_invalid_with_msg(jv_string_fmt("%s at line %d, column %d", msg, p->line, p->column));
} else if (p->curr_buf_is_partial) {
assert(p->curr_buf_pos == p->curr_buf_length);
// need another buffer
return jv_invalid();
} else {
// at EOF
assert(p->curr_buf_pos == p->curr_buf_length);
jv_free(value);
if (p->st != JV_PARSER_WAITING_FOR_RS) {
if (p->st != JV_PARSER_NORMAL) {
value = make_error(p, "Unfinished string at EOF at line %d, column %d", p->line, p->column);
parser_reset(p);
p->st = JV_PARSER_WAITING_FOR_RS;
return value;
}
if ((msg = check_literal(p))) {
value = make_error(p, "%s at EOF at line %d, column %d", msg, p->line, p->column);
parser_reset(p);
p->st = JV_PARSER_WAITING_FOR_RS;
return value;
}
if (((p->flags & JV_PARSE_STREAMING) && p->stacklen != 0) ||
(!(p->flags & JV_PARSE_STREAMING) && p->stackpos != 0)) {
value = make_error(p, "Unfinished JSON term at EOF at line %d, column %d", p->line, p->column);
parser_reset(p);
p->st = JV_PARSER_WAITING_FOR_RS;
return value;
}
}
// at EOF
if (p->st != JV_PARSER_NORMAL)
return jv_invalid_with_msg(jv_string("Unfinished string"));
if ((msg = check_literal(p)))
return jv_invalid_with_msg(jv_string(msg));
if (p->stackpos != 0)
return jv_invalid_with_msg(jv_string("Unfinished JSON term"));
// p->next is either invalid (nothing here but no syntax error)
// or valid (this is the value). either way it's the thing to return
if ((p->flags & JV_PARSE_STREAMING) && jv_is_valid(p->next)) {
value = JV_ARRAY(jv_copy(p->path), p->next); // except in streaming mode we've got to make it [path,value]
} else {
value = p->next;
}
value = p->next;
p->next = jv_invalid();
if ((p->flags & JV_PARSE_SEQ) && !p->last_ch_was_ws && jv_get_kind(value) == JV_KIND_NUMBER) {
jv_free(value);
return make_error(p, "Potentially truncated top-level numeric value at EOF at line %d, column %d", p->line, p->column);
}
return value;
}
}
jv jv_parse_sized(const char* string, int length) {
struct jv_parser parser;
parser_init(&parser, 0);
jv_parser_init(&parser);
jv_parser_set_buf(&parser, string, length, 0);
jv value = jv_parser_next(&parser);
if (jv_is_valid(value)) {
@@ -803,7 +429,7 @@ jv jv_parse_sized(const char* string, int length) {
jv_free(value);
value = jv_invalid_with_msg(jv_string("Expected JSON value"));
}
parser_free(&parser);
jv_parser_free(&parser);
if (!jv_is_valid(value) && jv_invalid_has_msg(jv_copy(value))) {
jv msg = jv_invalid_get_msg(value);
jv_parse.h
+37
View File
@@ -0,0 +1,37 @@
#ifndef JV_PARSE_H
#define JV_PARSE_H
#include "jv_dtoa.h"
struct jv_parser {
const char* curr_buf;
int curr_buf_length;
int curr_buf_pos;
int curr_buf_is_partial;
unsigned bom_strip_position;
jv* stack;
int stackpos;
int stacklen;
jv next;
char* tokenbuf;
int tokenpos;
int tokenlen;
int line, column;
struct dtoa_context dtoa;
enum {
JV_PARSER_NORMAL,
JV_PARSER_STRING,
JV_PARSER_STRING_ESCAPE
} st;
};
void jv_parser_init(struct jv_parser* p);
void jv_parser_free(struct jv_parser* p);
void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial);
jv jv_parser_next(struct jv_parser* p);
#endif
jv_print.c
+6 -72
View File
@@ -2,7 +2,6 @@
#include <stdio.h>
#include <float.h>
#include <string.h>
#include <assert.h>
#include "jv_dtoa.h"
#include "jv_unicode.h"
@@ -17,8 +16,8 @@ static const jv_kind colour_kinds[] =
{JV_KIND_NULL, JV_KIND_FALSE, JV_KIND_TRUE, JV_KIND_NUMBER,
JV_KIND_STRING, JV_KIND_ARRAY, JV_KIND_OBJECT};
static const char* const colours[] =
{COL("1;30"), COL("0;39"), COL("0;39"), COL("0;39"),
COL("0;32"), COL("1;39"), COL("1;39")};
{COL("30;1"), COL("0"), COL("0"), COL("0"),
COL("32"), COL("37"), COL("37")};
#define FIELD_COLOUR COL("34;1")
static void put_buf(const char* s, int len, FILE* fout, jv* strout) {
@@ -112,18 +111,9 @@ static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) {
enum { INDENT = 2 };
static void put_refcnt(struct dtoa_context* C, int refcnt, FILE *F, jv* S){
char buf[JVP_DTOA_FMT_MAX_LEN];
put_char(' ', F, S);
put_char('(', F, S);
put_str(jvp_dtoa_fmt(C, buf, refcnt), F, S);
put_char(')', F, S);
}
static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FILE* F, jv* S) {
char buf[JVP_DTOA_FMT_MAX_LEN];
const char* colour = 0;
double refcnt = (flags & JV_PRINT_REFCOUNT) ? jv_get_refcnt(x) - 1 : -1;
if (flags & JV_PRINT_COLOUR) {
for (unsigned i=0; i<sizeof(colour_kinds)/sizeof(colour_kinds[0]); i++) {
if (jv_get_kind(x) == colour_kinds[i]) {
@@ -136,18 +126,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
switch (jv_get_kind(x)) {
default:
case JV_KIND_INVALID:
if (flags & JV_PRINT_INVALID) {
jv msg = jv_invalid_get_msg(jv_copy(x));
if (jv_get_kind(msg) == JV_KIND_STRING) {
put_str("<invalid:", F, S);
jvp_dump_string(msg, flags | JV_PRINT_ASCII, F, S);
put_str(">", F, S);
} else {
put_str("<invalid>", F, S);
}
} else {
assert(0 && "Invalid value");
}
assert(0 && "Invalid value");
break;
case JV_KIND_NULL:
put_str("null", F, S);
@@ -173,8 +152,6 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
}
case JV_KIND_STRING:
jvp_dump_string(x, flags & JV_PRINT_ASCII, F, S);
if (flags & JV_PRINT_REFCOUNT)
put_refcnt(C, refcnt, F, S);
break;
case JV_KIND_ARRAY: {
if (jv_array_length(jv_copy(x)) == 0) {
@@ -204,8 +181,6 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
}
if (colour) put_str(colour, F, S);
put_char(']', F, S);
if (flags & JV_PRINT_REFCOUNT)
put_refcnt(C, refcnt, F, S);
break;
}
case JV_KIND_OBJECT: {
@@ -219,34 +194,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
put_space(indent+INDENT, F, S);
}
int first = 1;
int i = 0;
jv keyset = jv_null();
while (1) {
jv key, value;
if (flags & JV_PRINT_SORTED) {
if (first) {
keyset = jv_keys(jv_copy(x));
i = 0;
} else {
i++;
}
if (i >= jv_array_length(jv_copy(keyset))) {
jv_free(keyset);
break;
}
key = jv_array_get(jv_copy(keyset), i);
value = jv_object_get(jv_copy(x), jv_copy(key));
} else {
if (first) {
i = jv_object_iter(x);
} else {
i = jv_object_iter_next(x, i);
}
if (!jv_object_iter_valid(x, i)) break;
key = jv_object_iter_key(x, i);
value = jv_object_iter_value(x, i);
}
jv_object_foreach(x, key, value) {
if (!first) {
if (flags & JV_PRINT_PRETTY){
put_str(",\n", F, S);
@@ -276,8 +224,6 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
}
if (colour) put_str(colour, F, S);
put_char('}', F, S);
if (flags & JV_PRINT_REFCOUNT)
put_refcnt(C, refcnt, F, S);
}
}
jv_free(x);
@@ -286,25 +232,13 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
}
}
void jv_dumpf(jv x, FILE *f, int flags) {
void jv_dump(jv x, int flags) {
struct dtoa_context C;
jvp_dtoa_context_init(&C);
jv_dump_term(&C, x, flags, 0, f, 0);
jv_dump_term(&C, x, flags, 0, stdout, 0);
jvp_dtoa_context_free(&C);
}
void jv_dump(jv x, int flags) {
jv_dumpf(x, stdout, flags);
}
/* This one is nice for use in debuggers */
void jv_show(jv x, int flags) {
if (flags == -1)
flags = JV_PRINT_PRETTY | JV_PRINT_COLOUR;
jv_dumpf(jv_copy(x), stderr, flags | JV_PRINT_INVALID);
fflush(stderr);
}
jv jv_dump_string(jv x, int flags) {
struct dtoa_context C;
jvp_dtoa_context_init(&C);
jv_unicode.c
+17 -49
View File
@@ -3,67 +3,35 @@
#include "jv_unicode.h"
#include "jv_utf8_tables.h"
const char* jvp_utf8_next(const char* in, const char* end, int* codepoint_ret) {
assert(in <= end);
const char* jvp_utf8_next(const char* in, const char* end, int* codepoint) {
if (in == end) {
codepoint = 0;
return 0;
}
int codepoint = -1;
unsigned char first = (unsigned char)in[0];
int length = utf8_coding_length[first];
if ((first & 0x80) == 0) {
/* Fast-path for ASCII */
codepoint = first;
length = 1;
} else if (length == 0 || length == UTF8_CONTINUATION_BYTE) {
/* Bad single byte - either an invalid byte or an out-of-place continuation byte */
length = 1;
} else if (in + length > end) {
/* String ends before UTF8 sequence ends */
length = end - in;
} else {
codepoint = ((unsigned)in[0]) & utf8_coding_bits[first];
for (int i=1; i<length; i++) {
unsigned ch = (unsigned char)in[i];
if (utf8_coding_length[ch] != UTF8_CONTINUATION_BYTE){
/* Invalid UTF8 sequence - not followed by the right number of continuation bytes */
codepoint = -1;
length = i;
break;
}
codepoint = (codepoint << 6) | (ch & 0x3f);
}
if (codepoint < utf8_first_codepoint[length]) {
/* Overlong UTF8 sequence */
codepoint = -1;
}
if (0xD800 <= codepoint && codepoint <= 0xDFFF) {
/* Surrogate codepoints can't be encoded in UTF8 */
codepoint = -1;
}
if (codepoint > 0x10FFFF) {
/* Outside Unicode range */
codepoint = -1;
}
if (length == 0 || length == UTF8_CONTINUATION_BYTE || in + length > end) {
*codepoint = -1;
return 0;
}
*codepoint = ((unsigned)in[0]) & utf8_coding_bits[first];
for (int i=1; i<length; i++) {
int ch = (unsigned char)in[i];
if (utf8_coding_length[(unsigned char)in[i]] != UTF8_CONTINUATION_BYTE){
*codepoint = -1;
return 0;
}
*codepoint = (*codepoint << 6) | (ch & 0x3f);
}
assert(length > 0);
*codepoint_ret = codepoint;
return in + length;
}
int jvp_utf8_is_valid(const char* in, const char* end) {
int codepoint;
int jvp_utf8_verify(const char* in, const char* end) {
int codepoint = 0;
while ((in = jvp_utf8_next(in, end, &codepoint))) {
if (codepoint == -1) return 0;
}
return 1;
}
int jvp_utf8_decode_length(char startchar) {
if ((startchar & 0x80) == 0) return 1;
else if ((startchar & 0xC0) == 0xC0) return 2;
else if ((startchar & 0xE0) == 0xE0) return 3;
else return 4;
return codepoint != -1;
}
int jvp_utf8_encode_length(int codepoint) {
jv_unicode.h
+1 -1
View File
@@ -2,7 +2,7 @@
#define JV_UNICODE_H
const char* jvp_utf8_next(const char* in, const char* end, int* codepoint);
int jvp_utf8_is_valid(const char* in, const char* end);
int jvp_utf8_decode_length(char startchar);
jv_utf8_tables.h
-2
View File
@@ -33,5 +33,3 @@ static const unsigned char utf8_coding_bits[] =
0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f,
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
0x07, 0x07, 0x07, 0x07, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
static const int utf8_first_codepoint[] =
{0x00, 0x00, 0x80, 0x800, 0x10000};
lexer.l
+3 -16
View File
@@ -1,5 +1,4 @@
%{
#include <assert.h>
#include "jv_alloc.h"
#include "compile.h"
@@ -42,8 +41,6 @@ struct lexer_param;
"!=" { return NEQ; }
"==" { return EQ; }
"as" { return AS; }
"import" { return IMPORT; }
"module" { return MODULE; }
"def" { return DEF; }
"if" { return IF; }
"then" { return THEN; }
@@ -53,25 +50,16 @@ struct lexer_param;
"or" { return OR; }
"end" { return END; }
"reduce" { return REDUCE; }
"foreach" { return FOREACH; }
"//" { return DEFINEDOR; }
"try" { return TRY; }
"catch" { return CATCH; }
"label" { return LABEL; }
"break" { return BREAK; }
"break2" { return BREAK2; }
"break3" { return BREAK3; }
"|=" { return SETPIPE; }
"+=" { return SETPLUS; }
"-=" { return SETMINUS; }
"*=" { return SETMULT; }
"/=" { return SETDIV; }
"%=" { return SETMOD; }
"//=" { return SETDEFINEDOR; }
"<=" { return LESSEQ; }
">=" { return GREATEREQ; }
".." { return REC; }
"."|"?"|"="|";"|","|":"|"|"|"+"|"-"|"*"|"/"|"%"|"\$"|"<"|">" { return yytext[0];}
"."|"="|";"|","|":"|"|"|"+"|"-"|"*"|"/"|"\$"|"<"|">" { return yytext[0];}
"["|"{"|"(" {
return enter(yytext[0], YY_START, yyscanner);
@@ -104,7 +92,7 @@ struct lexer_param;
}
(\\[^u(]|\\u[a-zA-Z0-9]{0,4})+ {
/* pass escapes to the json parser */
jv escapes = jv_string_fmt("\"%.*s\"", (int)yyleng, yytext);
jv escapes = jv_string_fmt("\"%.*s\"", yyleng, yytext);
yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length_bytes(jv_copy(escapes)));
jv_free(escapes);
return QQSTRING_TEXT;
@@ -119,8 +107,7 @@ struct lexer_param;
}
([a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]* { yylval->literal = jv_string(yytext); return IDENT;}
\.[a-zA-Z_][a-zA-Z_0-9]* { yylval->literal = jv_string(yytext+1); return FIELD;}
[a-zA-Z_][a-zA-Z_0-9]* { yylval->literal = jv_string(yytext); return IDENT;}
[ \n\t]+ {}
libm.h
-34
View File
@@ -1,34 +0,0 @@
LIBM_DD(acos)
LIBM_DD(acosh)
LIBM_DD(asin)
LIBM_DD(asinh)
LIBM_DD(atan)
LIBM_DD(atanh)
LIBM_DD(cbrt)
LIBM_DD(cos)
LIBM_DD(cosh)
LIBM_DD(exp2)
LIBM_DD(exp)
LIBM_DD(floor)
LIBM_DD(j0)
LIBM_DD(j1)
LIBM_DD(log10)
LIBM_DD(log2)
LIBM_DD(log)
LIBM_DD(sin)
LIBM_DD(sinh)
LIBM_DD(sqrt)
LIBM_DD(tan)
LIBM_DD(tanh)
LIBM_DD(tgamma)
LIBM_DD(y0)
LIBM_DD(y1)
/* LIBM_DID(jn) */
/* LIBM_DID(yn) */
/* LIBM_DDD(pow) */
/* LIBM_DDD(atan2) */
/* LIBM_DDD(hypot) */
/* LIBM_DDD(remainder) */
/* LIBM_DDI(scalbn) */
/* LIBM_DDIP(lgamma_r) */
linker.c
-344
View File
@@ -1,344 +0,0 @@
#include <assert.h>
#include <errno.h>
#include <limits.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <sys/stat.h>
#include <libgen.h>
#include "jq_parser.h"
#include "locfile.h"
#include "jv.h"
#include "jq.h"
#include "parser.h"
#include "util.h"
#include "compile.h"
struct lib_loading_state {
char **names;
block *defs;
uint64_t ct;
};
static int load_library(jq_state *jq, jv lib_path, int is_data, int raw,
const char *as, block *out_block,
struct lib_loading_state *lib_state);
// Given a lib_path to search first, creates a chain of search paths
// in the following order:
// 1. lib_path
// 2. -L paths passed in on the command line (from jq_state*) or builtin list
static jv build_lib_search_chain(jq_state *jq, jv search_path, jv jq_origin, jv lib_origin) {
assert(jv_get_kind(search_path) == JV_KIND_ARRAY);
jv expanded = jv_array();
jv expanded_elt;
jv_array_foreach(search_path, i, path) {
if (strcmp(".",jv_string_value(path)) == 0) {
expanded_elt = jv_copy(path);
} else if (jv_get_kind(lib_origin) == JV_KIND_STRING &&
strncmp("./", jv_string_value(path),sizeof("./")-1) == 0) {
expanded_elt = jv_string_fmt("%s/%s",
jv_string_value(lib_origin),
jv_string_value(path) + sizeof ("./") - 1);
} else if (strncmp("$ORIGIN/",jv_string_value(path),sizeof("$ORIGIN/")-1) == 0) {
expanded_elt = jv_string_fmt("%s/%s",
jv_string_value(jq_origin),
jv_string_value(path) + sizeof ("$ORIGIN/") - 1);
} else {
expanded_elt = expand_path(path);
if (!jv_is_valid(expanded_elt)) {
jv_free(search_path);
jv_free(expanded);
jv_free(path);
return expanded_elt;
}
path = jv_invalid();
}
expanded = jv_array_append(expanded, expanded_elt);
jv_free(path);
}
jv_free(jq_origin);
jv_free(lib_origin);
jv_free(search_path);
return expanded;
}
// Doesn't actually check that name not be an absolute path; we could
// just always prepend "./"!
static jv validate_relpath(jv name) {
const char *s = jv_string_value(name);
if (strchr(s, '\\')) {
jv res = jv_invalid_with_msg(jv_string_fmt("Modules must be named by relative paths using '/', not '\\' (%s)", s));
jv_free(name);
return res;
}
jv components = jv_string_split(jv_copy(name), jv_string("/"));
jv rp = jv_array_get(jv_copy(components), 0);
components = jv_array_slice(components, 1, jv_array_length(jv_copy(components)));
jv_array_foreach(components, i, x) {
if (!strcmp(jv_string_value(x), "..")) {
jv_free(x);
jv_free(rp);
jv_free(components);
jv res = jv_invalid_with_msg(jv_string_fmt("Relative paths to modules may not traverse to parent directories (%s)", s));
jv_free(name);
return res;
}
if (i > 0 && jv_equal(jv_copy(x), jv_array_get(jv_copy(components), i - 1))) {
jv_free(x);
jv_free(rp);
jv_free(components);
jv res = jv_invalid_with_msg(jv_string_fmt("module names must not have equal consecutive components: %s",
jv_string_value(name)));
jv_free(name);
return res;
}
rp = jv_string_concat(rp, jv_string_concat(jv_string("/"), x));
}
jv_free(components);
jv_free(name);
return rp;
}
// Assumes name has been validated
static jv jv_basename(jv name) {
const char *s = jv_string_value(name);
const char *p = strrchr(s, '/');
if (!p)
return name;
jv res = jv_string_fmt("%s", p);
jv_free(name);
return res;
}
// Asummes validated relative path to module
static jv find_lib(jq_state *jq, jv rel_path, jv search, const char *suffix, jv jq_origin, jv lib_origin) {
if (jv_get_kind(search) != JV_KIND_ARRAY)
return jv_invalid_with_msg(jv_string_fmt("Module search path must be an array"));
if (jv_get_kind(rel_path) != JV_KIND_STRING)
return jv_invalid_with_msg(jv_string_fmt("Module path must be a string"));
struct stat st;
int ret;
// Ideally we should cache this somewhere
search = build_lib_search_chain(jq, search, jq_origin, lib_origin);
jv bname = jv_basename(jv_copy(rel_path));
jv_array_foreach(search, i, spath) {
if (jv_get_kind(spath) == JV_KIND_NULL) {
jv_free(spath);
break;
}
if (jv_get_kind(spath) != JV_KIND_STRING ||
strcmp(jv_string_value(spath), "") == 0) {
jv_free(spath);
continue; /* XXX report non-strings in search path?? */
}
// Try .../module/last/component.jq
jv testpath = jq_realpath(jv_string_fmt("%s/%s%s",
jv_string_value(spath),
jv_string_value(rel_path),
suffix));
ret = stat(jv_string_value(testpath),&st);
if (ret == -1 && errno == ENOENT) {
jv_free(testpath);
// Try .../module/last/component/component.jq
testpath = jq_realpath(jv_string_fmt("%s/%s/%s%s",
jv_string_value(spath),
jv_string_value(rel_path),
jv_string_value(bname),
suffix));
ret = stat(jv_string_value(testpath),&st);
}
if (ret == 0) {
jv_free(rel_path);
jv_free(search);
jv_free(bname);
jv_free(spath);
return testpath;
}
jv_free(testpath);
jv_free(spath);
}
jv output = jv_invalid_with_msg(jv_string_fmt("module not found: %s", jv_string_value(rel_path)));
jv_free(rel_path);
jv_free(search);
jv_free(bname);
return output;
}
static jv default_search(jq_state *jq, jv value) {
if (!jv_is_valid(value)) {
// dependent didn't say; prepend . to system search path listj
jv_free(value);
return jv_array_concat(JV_ARRAY(jv_string(".")), jq_get_lib_dirs(jq));
}
if (jv_get_kind(value) != JV_KIND_ARRAY)
return JV_ARRAY(value);
return value;
}
// XXX Split this into a util that takes a callback, and then...
static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block *src_block, struct lib_loading_state *lib_state) {
jv deps = block_take_imports(src_block);
block bk = *src_block;
int nerrors = 0;
jv_array_foreach(deps, i, dep) {
int is_data = jv_get_kind(jv_object_get(jv_copy(dep), jv_string("is_data"))) == JV_KIND_TRUE;
int raw = 0;
jv v = jv_object_get(jv_copy(dep), jv_string("raw"));
if (jv_get_kind(v) == JV_KIND_TRUE)
raw = 1;
jv_free(v);
jv relpath = validate_relpath(jv_object_get(jv_copy(dep), jv_string("relpath")));
jv as = jv_object_get(jv_copy(dep), jv_string("as"));
assert(jv_is_valid(as) && jv_get_kind(as) == JV_KIND_STRING);
jv search = default_search(jq, jv_object_get(dep, jv_string("search")));
// dep is now freed; do not reuse
// find_lib does a lot of work that could be cached...
jv resolved = find_lib(jq, relpath, search, is_data ? ".json" : ".jq", jv_copy(jq_origin), jv_copy(lib_origin));
// XXX ...move the rest of this into a callback.
if (!jv_is_valid(resolved)) {
jv emsg = jv_invalid_get_msg(resolved);
jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg)));
jv_free(emsg);
jv_free(as);
jv_free(deps);
jv_free(jq_origin);
jv_free(lib_origin);
return 1;
}
uint64_t state_idx = 0;
for (; state_idx < lib_state->ct; ++state_idx) {
if (strcmp(lib_state->names[state_idx],jv_string_value(resolved)) == 0)
break;
}
if (state_idx < lib_state->ct) { // Found
jv_free(resolved);
// Bind the library to the program
bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, jv_string_value(as));
} else { // Not found. Add it to the table before binding.
block dep_def_block = gen_noop();
nerrors += load_library(jq, resolved, is_data, raw, jv_string_value(as), &dep_def_block, lib_state);
// resolved has been freed
if (nerrors == 0) {
// Bind the library to the program
bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, jv_string_value(as));
}
}
jv_free(as);
}
jv_free(lib_origin);
jv_free(jq_origin);
jv_free(deps);
return nerrors;
}
// Loads the library at lib_path into lib_state, putting the library's defs
// into *out_block
static int load_library(jq_state *jq, jv lib_path, int is_data, int raw, const char *as, block *out_block, struct lib_loading_state *lib_state) {
int nerrors = 0;
struct locfile* src = NULL;
block program;
jv data;
if (is_data && !raw)
data = jv_load_file(jv_string_value(lib_path), 0);
else
data = jv_load_file(jv_string_value(lib_path), 1);
int state_idx;
if (!jv_is_valid(data)) {
if (jv_invalid_has_msg(jv_copy(data)))
data = jv_invalid_get_msg(data);
else
data = jv_string("unknown error");
jq_report_error(jq, jv_string_fmt("jq: error loading data file %s: %s\n", jv_string_value(lib_path), jv_string_value(data)));
nerrors++;
goto out;
} else if (is_data) {
// import "foo" as $bar;
program = gen_const_global(jv_copy(data), as);
} else {
// import "foo" as bar;
src = locfile_init(jq, jv_string_value(data), jv_string_length_bytes(jv_copy(data)));
nerrors += jq_parse_library(src, &program);
if (nerrors == 0) {
char *lib_origin = strdup(jv_string_value(lib_path));
nerrors += process_dependencies(jq, jq_get_jq_origin(jq),
jv_string(dirname(lib_origin)),
&program, lib_state);
free(lib_origin);
}
}
state_idx = lib_state->ct++;
lib_state->names = realloc(lib_state->names, lib_state->ct * sizeof(const char *));
lib_state->defs = realloc(lib_state->defs, lib_state->ct * sizeof(block));
lib_state->names[state_idx] = strdup(jv_string_value(lib_path));
lib_state->defs[state_idx] = program;
*out_block = program;
if (src)
locfile_free(src);
out:
jv_free(lib_path);
jv_free(data);
return nerrors;
}
// FIXME It'd be nice to have an option to search the same search path
// as we do in process_dependencies.
jv load_module_meta(jq_state *jq, jv mod_relpath) {
// We can't know the caller's origin; we could though, if it was passed in
jv lib_path = find_lib(jq, validate_relpath(mod_relpath), jq_get_lib_dirs(jq), ".jq", jq_get_jq_origin(jq), jv_null());
if (!jv_is_valid(lib_path))
return lib_path;
jv meta = jv_null();
jv data = jv_load_file(jv_string_value(lib_path), 1);
if (jv_is_valid(data)) {
block program;
struct locfile* src = locfile_init(jq, jv_string_value(data), jv_string_length_bytes(jv_copy(data)));
int nerrors = jq_parse_library(src, &program);
if (nerrors == 0) {
meta = block_module_meta(program);
if (jv_get_kind(meta) == JV_KIND_NULL)
meta = jv_object();
meta = jv_object_set(meta, jv_string("deps"), block_take_imports(&program));
}
locfile_free(src);
block_free(program);
}
jv_free(lib_path);
jv_free(data);
return meta;
}
int load_program(jq_state *jq, struct locfile* src, block *out_block) {
int nerrors = 0;
block program;
struct lib_loading_state lib_state = {0,0,0};
nerrors = jq_parse(src, &program);
if (nerrors)
return nerrors;
nerrors = process_dependencies(jq, jq_get_jq_origin(jq), jq_get_prog_origin(jq), &program, &lib_state);
block libs = gen_noop();
for (uint64_t i = 0; i < lib_state.ct; ++i) {
free(lib_state.names[i]);
if (nerrors == 0 && !block_is_const(lib_state.defs[i]))
libs = block_join(libs, lib_state.defs[i]);
else
block_free(lib_state.defs[i]);
}
free(lib_state.names);
free(lib_state.defs);
if (nerrors)
block_free(program);
else
*out_block = block_drop_unreferenced(block_join(libs, program));
return nerrors;
}
linker.h
-7
View File
@@ -1,7 +0,0 @@
#ifndef LINKER_H
#define LINKER_H
int load_program(jq_state *jq, struct locfile* src, block *out_block);
jv load_module_meta(jq_state *jq, jv modname);
#endif
locfile.c
-88
View File
@@ -1,88 +0,0 @@
#include <assert.h>
#include <errno.h>
#include <stdarg.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "jq.h"
#include "jv_alloc.h"
#include "locfile.h"
struct locfile* locfile_init(jq_state *jq, const char* data, int length) {
struct locfile* l = jv_mem_alloc(sizeof(struct locfile));
l->jq = jq;
l->data = jv_mem_alloc(length);
memcpy((char*)l->data,data,length);
l->length = length;
l->nlines = 1;
l->refct = 1;
for (int i=0; i<length; i++) {
if (data[i] == '\n') l->nlines++;
}
l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
l->linemap[0] = 0;
int line = 1;
for (int i=0; i<length; i++) {
if (data[i] == '\n') {
l->linemap[line] = i+1; // at start of line, not of \n
line++;
}
}
l->linemap[l->nlines] = length+1; // virtual last \n
return l;
}
struct locfile* locfile_retain(struct locfile* l) {
l->refct++;
return l;
}
void locfile_free(struct locfile* l) {
if (--(l->refct) == 0) {
jv_mem_free(l->linemap);
jv_mem_free((char*)l->data);
jv_mem_free(l);
}
}
static int locfile_get_line(struct locfile* l, int pos) {
assert(pos < l->length);
int line = 1;
while (l->linemap[line] <= pos) line++; // == if pos at start (before, never ==, because pos never on \n)
assert(line-1 < l->nlines);
return line-1;
}
static int locfile_line_length(struct locfile* l, int line) {
assert(line < l->nlines);
return l->linemap[line+1] - l->linemap[line] -1; // -1 to omit \n
}
void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
va_list fmtargs;
va_start(fmtargs, fmt);
int startline;
int offset;
if (loc.start != -1) {
startline = locfile_get_line(l, loc.start);
offset = l->linemap[startline];
}
jv m1 = jv_string_vfmt(fmt, fmtargs);
if (!jv_is_valid(m1)) {
jq_report_error(l->jq, m1);
return;
}
if (loc.start == -1) {
jq_report_error(l->jq, jv_string_fmt("jq: error: %s\n<unknown location>", jv_string_value(m1)));
jv_free(m1);
return;
}
jv m2 = jv_string_fmt("%s\n%.*s%*s", jv_string_value(m1),
locfile_line_length(l, startline), l->data + offset,
loc.start - offset, "");
jv_free(m1);
jq_report_error(l->jq, m2);
return;
}
locfile.h
+60 -10
View File
@@ -1,8 +1,10 @@
#ifndef _LOCFILE_H
#define _LOCFILE_H
#include "jq.h"
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <stdarg.h>
#include "jv_alloc.h"
typedef struct {
int start, end;
} location;
@@ -14,16 +16,64 @@ struct locfile {
int length;
int* linemap;
int nlines;
char *error;
jq_state *jq;
int refct;
};
struct locfile* locfile_init(jq_state *jq, const char* data, int length);
struct locfile* locfile_retain(struct locfile* l);
static void locfile_init(struct locfile* l, const char* data, int length) {
l->data = data;
l->length = length;
l->nlines = 1;
for (int i=0; i<length; i++) {
if (data[i] == '\n') l->nlines++;
}
l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
l->linemap[0] = 0;
int line = 1;
for (int i=0; i<length; i++) {
if (data[i] == '\n') {
l->linemap[line] = i;
line++;
}
}
l->linemap[l->nlines] = length;
}
void locfile_free(struct locfile* l);
static void locfile_free(struct locfile* l) {
jv_mem_free(l->linemap);
}
void locfile_locate(struct locfile* l, location loc, const char* fmt, ...);
static int locfile_get_line(struct locfile* l, int pos) {
assert(pos < l->length);
int line = 0;
while (l->linemap[line+1] < pos) line++;
assert(line < l->nlines);
return line;
}
static int locfile_line_length(struct locfile* l, int line) {
assert(line < l->nlines);
return l->linemap[line+1] - l->linemap[line];
}
static void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) {
va_list fmtargs;
va_start(fmtargs, fmt);
vfprintf(stderr, fmt, fmtargs);
va_end(fmtargs);
fprintf(stderr, "\n");
if (loc.start == -1) {
fprintf(stderr, "<unknown location>\n");
return;
}
int startline = locfile_get_line(l, loc.start);
int offset = l->linemap[startline];
fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset);
fprintf(stderr, "%*s", loc.start - offset, "");
for (int i = loc.start;
i < loc.end && i < offset + locfile_line_length(l, startline);
i++){
fprintf(stderr, "^");
}
fprintf(stderr, "\n");
}
#endif
m4/ax_compare_version.m4
-177
View File
@@ -1,177 +0,0 @@
# ===========================================================================
# http://www.gnu.org/software/autoconf-archive/ax_compare_version.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_COMPARE_VERSION(VERSION_A, OP, VERSION_B, [ACTION-IF-TRUE], [ACTION-IF-FALSE])
#
# DESCRIPTION
#
# This macro compares two version strings. Due to the various number of
# minor-version numbers that can exist, and the fact that string
# comparisons are not compatible with numeric comparisons, this is not
# necessarily trivial to do in a autoconf script. This macro makes doing
# these comparisons easy.
#
# The six basic comparisons are available, as well as checking equality
# limited to a certain number of minor-version levels.
#
# The operator OP determines what type of comparison to do, and can be one
# of:
#
# eq - equal (test A == B)
# ne - not equal (test A != B)
# le - less than or equal (test A <= B)
# ge - greater than or equal (test A >= B)
# lt - less than (test A < B)
# gt - greater than (test A > B)
#
# Additionally, the eq and ne operator can have a number after it to limit
# the test to that number of minor versions.
#
# eq0 - equal up to the length of the shorter version
# ne0 - not equal up to the length of the shorter version
# eqN - equal up to N sub-version levels
# neN - not equal up to N sub-version levels
#
# When the condition is true, shell commands ACTION-IF-TRUE are run,
# otherwise shell commands ACTION-IF-FALSE are run. The environment
# variable 'ax_compare_version' is always set to either 'true' or 'false'
# as well.
#
# Examples:
#
# AX_COMPARE_VERSION([3.15.7],[lt],[3.15.8])
# AX_COMPARE_VERSION([3.15],[lt],[3.15.8])
#
# would both be true.
#
# AX_COMPARE_VERSION([3.15.7],[eq],[3.15.8])
# AX_COMPARE_VERSION([3.15],[gt],[3.15.8])
#
# would both be false.
#
# AX_COMPARE_VERSION([3.15.7],[eq2],[3.15.8])
#
# would be true because it is only comparing two minor versions.
#
# AX_COMPARE_VERSION([3.15.7],[eq0],[3.15])
#
# would be true because it is only comparing the lesser number of minor
# versions of the two values.
#
# Note: The characters that separate the version numbers do not matter. An
# empty string is the same as version 0. OP is evaluated by autoconf, not
# configure, so must be a string, not a variable.
#
# The author would like to acknowledge Guido Draheim whose advice about
# the m4_case and m4_ifvaln functions make this macro only include the
# portions necessary to perform the specific comparison specified by the
# OP argument in the final configure script.
#
# LICENSE
#
# Copyright (c) 2008 Tim Toolan <toolan@ele.uri.edu>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved. This file is offered as-is, without any
# warranty.
#serial 11
dnl #########################################################################
AC_DEFUN([AX_COMPARE_VERSION], [
AC_REQUIRE([AC_PROG_AWK])
# Used to indicate true or false condition
ax_compare_version=false
# Convert the two version strings to be compared into a format that
# allows a simple string comparison. The end result is that a version
# string of the form 1.12.5-r617 will be converted to the form
# 0001001200050617. In other words, each number is zero padded to four
# digits, and non digits are removed.
AS_VAR_PUSHDEF([A],[ax_compare_version_A])
A=`echo "$1" | sed -e 's/\([[0-9]]*\)/Z\1Z/g' \
-e 's/Z\([[0-9]]\)Z/Z0\1Z/g' \
-e 's/Z\([[0-9]][[0-9]]\)Z/Z0\1Z/g' \
-e 's/Z\([[0-9]][[0-9]][[0-9]]\)Z/Z0\1Z/g' \
-e 's/[[^0-9]]//g'`
AS_VAR_PUSHDEF([B],[ax_compare_version_B])
B=`echo "$3" | sed -e 's/\([[0-9]]*\)/Z\1Z/g' \
-e 's/Z\([[0-9]]\)Z/Z0\1Z/g' \
-e 's/Z\([[0-9]][[0-9]]\)Z/Z0\1Z/g' \
-e 's/Z\([[0-9]][[0-9]][[0-9]]\)Z/Z0\1Z/g' \
-e 's/[[^0-9]]//g'`
dnl # In the case of le, ge, lt, and gt, the strings are sorted as necessary
dnl # then the first line is used to determine if the condition is true.
dnl # The sed right after the echo is to remove any indented white space.
m4_case(m4_tolower($2),
[lt],[
ax_compare_version=`echo "x$A
x$B" | sed 's/^ *//' | sort -r | sed "s/x${A}/false/;s/x${B}/true/;1q"`
],
[gt],[
ax_compare_version=`echo "x$A
x$B" | sed 's/^ *//' | sort | sed "s/x${A}/false/;s/x${B}/true/;1q"`
],
[le],[
ax_compare_version=`echo "x$A
x$B" | sed 's/^ *//' | sort | sed "s/x${A}/true/;s/x${B}/false/;1q"`
],
[ge],[
ax_compare_version=`echo "x$A
x$B" | sed 's/^ *//' | sort -r | sed "s/x${A}/true/;s/x${B}/false/;1q"`
],[
dnl Split the operator from the subversion count if present.
m4_bmatch(m4_substr($2,2),
[0],[
# A count of zero means use the length of the shorter version.
# Determine the number of characters in A and B.
ax_compare_version_len_A=`echo "$A" | $AWK '{print(length)}'`
ax_compare_version_len_B=`echo "$B" | $AWK '{print(length)}'`
# Set A to no more than B's length and B to no more than A's length.
A=`echo "$A" | sed "s/\(.\{$ax_compare_version_len_B\}\).*/\1/"`
B=`echo "$B" | sed "s/\(.\{$ax_compare_version_len_A\}\).*/\1/"`
],
[[0-9]+],[
# A count greater than zero means use only that many subversions
A=`echo "$A" | sed "s/\(\([[0-9]]\{4\}\)\{m4_substr($2,2)\}\).*/\1/"`
B=`echo "$B" | sed "s/\(\([[0-9]]\{4\}\)\{m4_substr($2,2)\}\).*/\1/"`
],
[.+],[
AC_WARNING(
[illegal OP numeric parameter: $2])
],[])
# Pad zeros at end of numbers to make same length.
ax_compare_version_tmp_A="$A`echo $B | sed 's/./0/g'`"
B="$B`echo $A | sed 's/./0/g'`"
A="$ax_compare_version_tmp_A"
# Check for equality or inequality as necessary.
m4_case(m4_tolower(m4_substr($2,0,2)),
[eq],[
test "x$A" = "x$B" && ax_compare_version=true
],
[ne],[
test "x$A" != "x$B" && ax_compare_version=true
],[
AC_WARNING([illegal OP parameter: $2])
])
])
AS_VAR_POPDEF([A])dnl
AS_VAR_POPDEF([B])dnl
dnl # Execute ACTION-IF-TRUE / ACTION-IF-FALSE.
if test "$ax_compare_version" = "true" ; then
m4_ifvaln([$4],[$4],[:])dnl
m4_ifvaln([$5],[else $5])dnl
fi
]) dnl AX_COMPARE_VERSION
m4/ax_prog_bison_version.m4
-68
View File
@@ -1,68 +0,0 @@
# ===========================================================================
# Modified from
# http://www.gnu.org/software/autoconf-archive/ax_prog_perl_version.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_PROG_BISON_VERSION([VERSION],[ACTION-IF-TRUE],[ACTION-IF-FALSE])
#
# DESCRIPTION
#
# Makes sure that bison supports the version indicated. If true the shell
# commands in ACTION-IF-TRUE are executed. If not the shell commands in
# ACTION-IF-FALSE are run. Note if $PERL is not set the macro will fail.
#
# Example:
#
# AC_PROG_YACC
# AX_PROG_BISON_VERSION([3.0.0],[ ... ],[ ... ])
#
# This will check to make sure that the bison you have supports at least
# version 3.0.0.
#
# NOTE: This macro uses the $YACC variable to perform the check.
# AX_WITH_YACC can be used to set that variable prior to running this
# macro. The $BISON_VERSION variable will be set with the detected
# version.
#
# LICENSE
#
# Copyright (c) 2009 Francesco Salvestrini <salvestrini@users.sourceforge.net>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved. This file is offered as-is, without any
# warranty.
#serial 12
AC_DEFUN([AX_PROG_BISON_VERSION],[
AC_REQUIRE([AC_PROG_SED])
AC_REQUIRE([AC_PROG_GREP])
AS_IF([test -n "$YACC"],[
ax_bison_version="$1"
AC_MSG_CHECKING([for bison version])
changequote(<<,>>)
bison_version=`$YACC --version 2>&1 \
| $GREP bison \
| $SED -e 's/^[^0-9]*//' -e 's/[^0-9]*$//'`
changequote([,])
AC_MSG_RESULT($bison_version)
AC_SUBST([BISON_VERSION],[$bison_version])
AX_COMPARE_VERSION([$ax_bison_version],[le],[$bison_version],[
:
$2
],[
:
$3
])
],[
AC_MSG_WARN([could not find bison])
$3
])
])
main.c
+182 -499
View File
@@ -1,55 +1,33 @@
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <libgen.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include "compile.h"
#include "jv.h"
#include "jq.h"
#include "jv_parse.h"
#include "execute.h"
#include "config.h" /* Autoconf generated header file */
#include "jv_alloc.h"
#include "util.h"
#include "version.h"
int jq_testsuite(jv lib_dirs, int argc, char* argv[]);
int jq_testsuite(int argc, char* argv[]);
static const char* progname;
/*
* For a longer help message we could use a better option parsing
* strategy, one that lets stack options.
*/
static void usage(int code) {
fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", JQ_VERSION);
static void usage() {
fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", PACKAGE_VERSION);
fprintf(stderr, "Usage: %s [options] <jq filter> [file...]\n\n", progname);
fprintf(stderr, "\tjq is a tool for processing JSON inputs, applying the\n");
fprintf(stderr, "\tgiven filter to its JSON text inputs and producing the\n");
fprintf(stderr, "\tfilter's results as JSON on standard output.\n");
fprintf(stderr, "\tThe simplest filter is ., which is the identity filter,\n");
fprintf(stderr, "\tcopying jq's input to its output.\n");
fprintf(stderr, "\tFor more advanced filters see the jq(1) manpage (\"man jq\")\n");
fprintf(stderr, "\tand/or http://stedolan.github.com/jq\n\n");
fprintf(stderr, "\tSome of the options include:\n");
fprintf(stderr, "\t -h\t\tthis message;\n");
fprintf(stderr, "\t -c\t\tcompact instead of pretty-printed output;\n");
fprintf(stderr, "\t -n\t\tuse `null` as the single input value;\n");
fprintf(stderr, "\t -i\t\tedit the [first] file in-place;\n");
fprintf(stderr, "\t -s\t\tread (slurp) all inputs into an array; apply filter to it;\n");
fprintf(stderr, "\t -r\t\toutput raw strings, not JSON texts;\n");
fprintf(stderr, "\t -R\t\tread raw strings, not JSON texts;\n");
fprintf(stderr, "\t --arg a v\tset variable $a to value <v>;\n");
fprintf(stderr, "\t --argjson a v\tset variable $a to JSON value <v>;\n");
fprintf(stderr, "\t --argfile a f\tset variable $a to JSON texts read from <f>;\n");
fprintf(stderr, "\tSee the manpage for more options.\n");
exit(code);
fprintf(stderr, "For a description of the command line options and\n");
fprintf(stderr, "how to write jq filters (and why you might want to)\n");
fprintf(stderr, "see the jq manpage, or the online documentation at\n");
fprintf(stderr, "http://stedolan.github.io/jq\n\n");
exit(1);
}
static void die() {
fprintf(stderr, "Use %s --help for help with command-line options,\n", progname);
fprintf(stderr, "or see the jq manpage, or online docs at http://stedolan.github.com/jq\n");
exit(2);
fprintf(stderr, "or see the jq documentation at http://stedolan.github.io/jq\n");
exit(1);
}
@@ -59,189 +37,118 @@ static int isoptish(const char* text) {
return text[0] == '-' && (text[1] == '-' || isalpha(text[1]));
}
static int isoption(const char* text, char shortopt, const char* longopt, size_t *short_opts) {
if (text[0] != '-' || text[1] == '-')
*short_opts = 0;
static int isoption(const char* text, char shortopt, const char* longopt) {
if (text[0] != '-') return 0;
// check long option
if (strlen(text) == 2 && text[1] == shortopt) return 1;
if (text[1] == '-' && !strcmp(text+2, longopt)) return 1;
else if (text[1] == '-') return 0;
// must be short option; check it and...
if (!shortopt) return 0;
if (strchr(text, shortopt) != NULL) {
(*short_opts)++; // ...count it (for option stacking)
return 1;
}
return 0;
}
enum {
SLURP = 1,
RAW_INPUT = 2,
PROVIDE_NULL = 4,
RAW_OUTPUT = 8,
COMPACT_OUTPUT = 16,
ASCII_OUTPUT = 32,
COLOUR_OUTPUT = 64,
NO_COLOUR_OUTPUT = 128,
SORTED_OUTPUT = 256,
FROM_FILE = 512,
RAW_NO_LF = 1024,
UNBUFFERED_OUTPUT = 2048,
EXIT_STATUS = 4096,
IN_PLACE = 8192,
SEQ = 16384,
RUN_TESTS = 32768,
SLURP = 1,
RAW_INPUT = 2,
PROVIDE_NULL = 4,
RAW_OUTPUT = 8,
COMPACT_OUTPUT = 16,
ASCII_OUTPUT = 32,
COLOUR_OUTPUT = 64,
NO_COLOUR_OUTPUT = 128,
FROM_FILE = 256,
/* debugging only */
DUMP_DISASM = 65536,
DUMP_DISASM = 2048,
};
static int options = 0;
static struct bytecode* bc;
static int process(jq_state *jq, jv value, int flags, int dumpopts) {
int ret = 14; // No valid results && -e -> exit(4)
jq_start(jq, value, flags);
static void process(jv value, int flags) {
jq_state *jq = NULL;
jq_init(bc, value, &jq, flags);
jv result;
while (jv_is_valid(result = jq_next(jq))) {
if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout);
ret = 0;
jv_free(result);
} else {
if (jv_get_kind(result) == JV_KIND_FALSE || jv_get_kind(result) == JV_KIND_NULL)
ret = 11;
else
ret = 0;
if (options & SEQ)
fwrite("\036", 1, 1, stdout);
int dumpopts;
/* Disable colour by default on Windows builds as Windows
terminals tend not to display it correctly */
#ifdef WIN32
dumpopts = 0;
#else
dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0;
#endif
if (!(options & COMPACT_OUTPUT)) dumpopts |= JV_PRINT_PRETTY;
if (options & ASCII_OUTPUT) dumpopts |= JV_PRINT_ASCII;
if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR;
if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR;
jv_dump(result, dumpopts);
}
if (!(options & RAW_NO_LF))
printf("\n");
if (options & UNBUFFERED_OUTPUT)
fflush(stdout);
}
if (jv_invalid_has_msg(jv_copy(result))) {
// Uncaught jq exception
jv msg = jv_invalid_get_msg(jv_copy(result));
if (jv_get_kind(msg) == JV_KIND_STRING) {
fprintf(stderr, "jq: error: %s\n", jv_string_value(msg));
} else {
msg = jv_dump_string(msg, 0);
fprintf(stderr, "jq: error (not a string): %s\n", jv_string_value(msg));
}
jv_free(msg);
printf("\n");
}
jv_free(result);
return ret;
jq_teardown(&jq);
}
static jv slurp_file(const char* filename) {
FILE* file = fopen(filename, "r");
if (!file) {
return jv_invalid_with_msg(jv_string_fmt("Could not open %s: %s",
filename,
strerror(errno)));
}
jv data = jv_string("");
while (!feof(file) && !ferror(file)) {
char buf[4096];
size_t n = fread(buf, 1, sizeof(buf), file);
data = jv_string_concat(data, jv_string_sized(buf, (int)n));
}
int badread = ferror(file);
fclose(file);
if (badread) {
jv_free(data);
return jv_invalid_with_msg(jv_string_fmt("Error reading from %s",
filename));
}
return data;
}
// XXX Move all this into the next_input_state struct
FILE* current_input;
const char** input_filenames = NULL;
const char** input_filenames;
int ninput_files;
int next_input_idx;
static int read_more(char* buf, size_t size) {
if (!current_input || feof(current_input)) {
while (!current_input || feof(current_input)) {
if (current_input) {
if (current_input == stdin) {
clearerr(stdin); // perhaps we can read again; anyways, we don't fclose(stdin)
} else {
fclose(current_input);
}
fclose(current_input);
current_input = 0;
}
if (next_input_idx < ninput_files) {
if (!strcmp(input_filenames[next_input_idx], "-")) {
current_input = stdin;
} else {
current_input = fopen(input_filenames[next_input_idx], "r");
if (!current_input)
fprintf(stderr, "%s: %s: %s\n", progname, input_filenames[next_input_idx], strerror(errno));
}
next_input_idx++;
if (next_input_idx == ninput_files) {
return 0;
}
}
buf[0] = 0;
if (current_input) {
if (!fgets(buf, size, current_input))
buf[0] = 0;
}
return next_input_idx == ninput_files && (!current_input || feof(current_input));
}
struct next_input_state {
struct jv_parser *parser;
jv slurped;
char buf[4096];
};
// Blocks to read one more input from stdin and/or given files
// When slurping, it returns just one value
static jv next_input(jq_state *jq, void *data) {
struct next_input_state *state = data;
int is_last = 0;
jv value = jv_invalid(); // need more input
do {
if (options & RAW_INPUT) {
is_last = read_more(state->buf, sizeof(state->buf));
if (state->buf[0] == '\0')
continue;
int len = strlen(state->buf); // Raw input doesn't support NULs
if (len > 0) {
if (options & SLURP) {
state->slurped = jv_string_concat(state->slurped, jv_string(state->buf));
} else if (jv_is_valid(value)) {
if (state->buf[len-1] == '\n') {
// whole line
state->buf[len-1] = 0;
return jv_string_concat(value, jv_string(state->buf));
}
value = jv_string_concat(value, jv_string(state->buf));
}
}
if (!strcmp(input_filenames[next_input_idx], "-")) {
current_input = stdin;
} else {
if (jv_parser_remaining(state->parser) == 0) {
is_last = read_more(state->buf, sizeof(state->buf));
jv_parser_set_buf(state->parser, state->buf, strlen(state->buf), !is_last); // NULs also not supported here
}
value = jv_parser_next(state->parser);
if (options & SLURP) {
if (jv_is_valid(value)) {
state->slurped = jv_array_append(state->slurped, value);
value = jv_invalid();
} else if (jv_invalid_has_msg(jv_copy(value)))
return value;
} else if (jv_is_valid(value) || jv_invalid_has_msg(jv_copy(value))) {
return value;
}
current_input = fopen(input_filenames[next_input_idx], "r");
}
} while (!is_last);
return value;
}
if (!current_input) {
fprintf(stderr, "%s: %s: %s\n", progname, input_filenames[next_input_idx], strerror(errno));
}
next_input_idx++;
}
static void debug_cb(jq_state *jq, void *data, jv input) {
int dumpopts = *(int *)data;
jv_dumpf(JV_ARRAY(jv_string("DEBUG:"), input), stderr, dumpopts & ~(JV_PRINT_PRETTY));
fprintf(stderr, "\n");
if (!fgets(buf, size, current_input)) buf[0] = 0;
return 1;
}
int main(int argc, char* argv[]) {
jq_state *jq = NULL;
int ret = 0;
int compiled = 0;
int parser_flags = 0;
char *t = NULL;
if (argc) progname = argv[0];
jq = jq_init();
if (jq == NULL) {
perror("malloc");
ret = 2;
goto out;
if (argc > 1 && !strcmp(argv[1], "--run-tests")) {
return jq_testsuite(argc - 1, argv + 1);
}
const char* program = 0;
@@ -249,14 +156,12 @@ int main(int argc, char* argv[]) {
ninput_files = 0;
int further_args_are_files = 0;
int jq_flags = 0;
size_t short_opts = 0;
jv program_arguments = jv_array();
jv lib_search_paths = jv_null();
for (int i=1; i<argc; i++, short_opts = 0) {
for (int i=1; i<argc; i++) {
if (further_args_are_files) {
input_filenames[ninput_files++] = argv[i];
} else if (!strcmp(argv[i], "--")) {
if (!program) usage(2);
if (!program) usage();
further_args_are_files = 1;
} else if (!isoptish(argv[i])) {
if (program) {
@@ -264,237 +169,49 @@ int main(int argc, char* argv[]) {
} else {
program = argv[i];
}
} else {
if (argv[i][1] == 'L') {
if (jv_get_kind(lib_search_paths) == JV_KIND_NULL)
lib_search_paths = jv_array();
if (argv[i][2] != 0) { // -Lname (faster check than strlen)
lib_search_paths = jv_array_append(lib_search_paths, jv_string(argv[i]+2));
} else if (i >= argc - 1) {
fprintf(stderr, "-L takes a parameter: (e.g. -L /search/path or -L/search/path)\n");
die();
} else {
lib_search_paths = jv_array_append(lib_search_paths, jv_string(argv[i+1]));
i++;
}
continue;
}
if (isoption(argv[i], 's', "slurp", &short_opts)) {
options |= SLURP;
if (!short_opts) continue;
}
if (isoption(argv[i], 'r', "raw-output", &short_opts)) {
options |= RAW_OUTPUT;
if (!short_opts) continue;
}
if (isoption(argv[i], 'c', "compact-output", &short_opts)) {
options |= COMPACT_OUTPUT;
if (!short_opts) continue;
}
if (isoption(argv[i], 'C', "color-output", &short_opts)) {
options |= COLOUR_OUTPUT;
if (!short_opts) continue;
}
if (isoption(argv[i], 'M', "monochrome-output", &short_opts)) {
options |= NO_COLOUR_OUTPUT;
if (!short_opts) continue;
}
if (isoption(argv[i], 'a', "ascii-output", &short_opts)) {
options |= ASCII_OUTPUT;
if (!short_opts) continue;
}
if (isoption(argv[i], 0, "unbuffered", &short_opts)) {
options |= UNBUFFERED_OUTPUT;
if (!short_opts) continue;
}
if (isoption(argv[i], 'S', "sort-keys", &short_opts)) {
options |= SORTED_OUTPUT;
if (!short_opts) continue;
}
if (isoption(argv[i], 'R', "raw-input", &short_opts)) {
options |= RAW_INPUT;
if (!short_opts) continue;
}
if (isoption(argv[i], 'n', "null-input", &short_opts)) {
options |= PROVIDE_NULL;
if (!short_opts) continue;
}
if (isoption(argv[i], 'f', "from-file", &short_opts)) {
options |= FROM_FILE;
if (!short_opts) continue;
}
if (isoption(argv[i], 'j', "join-output", &short_opts)) {
options |= RAW_OUTPUT | RAW_NO_LF;
if (!short_opts) continue;
}
if (isoption(argv[i], 'i', "in-place", &short_opts)) {
options |= IN_PLACE;
if (!short_opts) continue;
}
if (isoption(argv[i], 0, "seq", &short_opts)) {
options |= SEQ;
if (!short_opts) continue;
}
if (isoption(argv[i], 0, "stream", &short_opts)) {
parser_flags |= JV_PARSE_STREAMING;
if (!short_opts) continue;
}
if (isoption(argv[i], 0, "stream-errors", &short_opts)) {
parser_flags |= JV_PARSE_STREAM_ERRORS;
if (!short_opts) continue;
}
if (isoption(argv[i], 'e', "exit-status", &short_opts)) {
options |= EXIT_STATUS;
if (!short_opts) continue;
}
// FIXME: For --arg* we should check that the varname is acceptable
if (isoption(argv[i], 0, "arg", &short_opts)) {
if (i >= argc - 2) {
fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname);
die();
}
jv arg = jv_object();
arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
arg = jv_object_set(arg, jv_string("value"), jv_string(argv[i+2]));
program_arguments = jv_array_append(program_arguments, arg);
i += 2; // skip the next two arguments
if (!short_opts) continue;
}
if (isoption(argv[i], 0, "argjson", &short_opts)) {
if (i >= argc - 2) {
fprintf(stderr, "%s: --argjson takes two parameters (e.g. -a varname text)\n", progname);
die();
}
jv v = jv_parse(argv[i+2]);
if (!jv_is_valid(v)) {
fprintf(stderr, "%s: invalid JSON text passed to --argjson\n", progname);
die();
}
jv arg = jv_object();
arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
arg = jv_object_set(arg, jv_string("value"), v);
program_arguments = jv_array_append(program_arguments, arg);
i += 2; // skip the next two arguments
if (!short_opts) continue;
}
if (isoption(argv[i], 0, "argfile", &short_opts)) {
if (i >= argc - 2) {
fprintf(stderr, "%s: --argfile takes two parameters (e.g. -a varname filename)\n", progname);
die();
}
jv arg = jv_object();
arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
jv data = jv_load_file(argv[i+2], 0);
if (!jv_is_valid(data)) {
data = jv_invalid_get_msg(data);
fprintf(stderr, "%s: Bad JSON in --argfile %s %s: %s\n", progname,
argv[i+1], argv[i+2], jv_string_value(data));
jv_free(data);
ret = 2;
goto out;
}
if (jv_get_kind(data) == JV_KIND_ARRAY && jv_array_length(jv_copy(data)) == 1)
data = jv_array_get(data, 0);
arg = jv_object_set(arg, jv_string("value"), data);
program_arguments = jv_array_append(program_arguments, arg);
i += 2; // skip the next two arguments
if (!short_opts) continue;
}
if (isoption(argv[i], 0, "debug-dump-disasm", &short_opts)) {
options |= DUMP_DISASM;
if (!short_opts) continue;
}
if (isoption(argv[i], 0, "debug-trace", &short_opts)) {
jq_flags |= JQ_DEBUG_TRACE;
if (!short_opts) continue;
}
if (isoption(argv[i], 'h', "help", &short_opts)) {
usage(0);
if (!short_opts) continue;
}
if (isoption(argv[i], 'V', "version", &short_opts)) {
printf("jq-%s\n", JQ_VERSION);
ret = 0;
goto out;
}
if (isoption(argv[i], 0, "run-tests", &short_opts)) {
jv_free(program_arguments);
ret = jq_testsuite(lib_search_paths, argc - i, argv + i + 1);
goto out;
}
// check for unknown options... if this argument was a short option
if (strlen(argv[i]) != short_opts + 1) {
fprintf(stderr, "%s: Unknown option %s\n", progname, argv[i]);
} else if (isoption(argv[i], 's', "slurp")) {
options |= SLURP;
} else if (isoption(argv[i], 'r', "raw-output")) {
options |= RAW_OUTPUT;
} else if (isoption(argv[i], 'c', "compact-output")) {
options |= COMPACT_OUTPUT;
} else if (isoption(argv[i], 'C', "color-output")) {
options |= COLOUR_OUTPUT;
} else if (isoption(argv[i], 'M', "monochrome-output")) {
options |= NO_COLOUR_OUTPUT;
} else if (isoption(argv[i], 'a', "ascii-output")) {
options |= ASCII_OUTPUT;
} else if (isoption(argv[i], 'R', "raw-input")) {
options |= RAW_INPUT;
} else if (isoption(argv[i], 'n', "null-input")) {
options |= PROVIDE_NULL;
} else if (isoption(argv[i], 'f', "from-file")) {
options |= FROM_FILE;
} else if (isoption(argv[i], 0, "arg")) {
if (i >= argc - 2) {
fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname);
die();
}
jv arg = jv_object();
arg = jv_object_set(arg, jv_string("name"), jv_string(argv[i+1]));
arg = jv_object_set(arg, jv_string("value"), jv_string(argv[i+2]));
program_arguments = jv_array_append(program_arguments, arg);
i += 2; // skip the next two arguments
} else if (isoption(argv[i], 0, "debug-dump-disasm")) {
options |= DUMP_DISASM;
} else if (isoption(argv[i], 0, "debug-trace")) {
jq_flags |= JQ_DEBUG_TRACE;
} else if (isoption(argv[i], 'h', "help")) {
usage();
} else if (isoption(argv[i], 'V', "version")) {
fprintf(stderr, "jq version %s\n", PACKAGE_VERSION);
return 0;
} else {
fprintf(stderr, "%s: Unknown option %s\n", progname, argv[i]);
die();
}
}
int dumpopts;
/* Disable colour by default on Windows builds as Windows
terminals tend not to display it correctly */
#ifdef WIN32
dumpopts = 0;
#else
dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0;
#endif
if (options & SORTED_OUTPUT) dumpopts |= JV_PRINT_SORTED;
if (!(options & COMPACT_OUTPUT)) dumpopts |= JV_PRINT_PRETTY;
if (options & ASCII_OUTPUT) dumpopts |= JV_PRINT_ASCII;
if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR;
if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR;
if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) {
// Default search path list
lib_search_paths = JV_ARRAY(jv_string("~/.jq"),
jv_string("$ORIGIN/../lib/jq"),
jv_string("$ORIGIN/lib"));
}
jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_search_paths);
char *origin = strdup(argv[0]);
if (origin == NULL) {
fprintf(stderr, "Error: out of memory\n");
exit(1);
}
jq_set_attr(jq, jv_string("JQ_ORIGIN"), jv_string(dirname(origin)));
free(origin);
if (strchr(JQ_VERSION, '-') == NULL)
jq_set_attr(jq, jv_string("VERSION_DIR"), jv_string(JQ_VERSION));
else
jq_set_attr(jq, jv_string("VERSION_DIR"), jv_string_fmt("%.*s-master", (int)(strchr(JQ_VERSION, '-') - JQ_VERSION), JQ_VERSION));
#if (!defined(WIN32) && defined(HAVE_ISATTY)) || defined(HAVE__ISATTY)
#if defined(HAVE__ISATTY) && defined(isatty)
#undef isatty
#define isatty _isatty
#endif
if (!program && isatty(STDOUT_FILENO) && !isatty(STDIN_FILENO))
program = ".";
#endif
if (!program) usage(2);
if ((options & IN_PLACE)) {
if (ninput_files == 0) usage(2);
if (strcmp(input_filenames[0], "-") == 0) usage(2);
size_t tlen = strlen(input_filenames[0]) + 7;
t = jv_mem_alloc(tlen);
int n = snprintf(t, tlen,"%sXXXXXX", input_filenames[0]);
assert(n > 0 && (size_t)n < tlen);
if (mkstemp(t) == -1) {
fprintf(stderr, "Error: %s creating temporary file", strerror(errno));
exit(3);
}
if (freopen(t, "w", stdout) == NULL) {
fprintf(stderr, "Error: %s redirecting stdout to temporary file", strerror(errno));
exit(3);
}
}
if (!program) usage();
if (ninput_files == 0) current_input = stdin;
if ((options & PROVIDE_NULL) && (options & (RAW_INPUT | SLURP))) {
@@ -503,114 +220,80 @@ int main(int argc, char* argv[]) {
}
if (options & FROM_FILE) {
char *program_origin = strdup(program);
if (program_origin == NULL) {
perror("malloc");
exit(2);
}
jv data = jv_load_file(program, 1);
jv data = slurp_file(program);
if (!jv_is_valid(data)) {
data = jv_invalid_get_msg(data);
fprintf(stderr, "%s: %s\n", progname, jv_string_value(data));
jv_free(data);
ret = 2;
goto out;
return 1;
}
jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string(dirname(program_origin))));
compiled = jq_compile_args(jq, jv_string_value(data), program_arguments);
free(program_origin);
bc = jq_compile_args(jv_string_value(data), program_arguments);
jv_free(data);
} else {
jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string("."))); // XXX is this good?
compiled = jq_compile_args(jq, program, program_arguments);
}
if (!compiled){
ret = 3;
goto out;
bc = jq_compile_args(program, program_arguments);
}
if (!bc) return 1;
if (options & DUMP_DISASM) {
jq_dump_disassembly(jq, 0);
dump_disassembly(0, bc);
printf("\n");
}
// XXX Refactor this and input_filenames[] and related setup into a
// function to setup struct next_input_state.
if ((options & SEQ))
parser_flags |= JV_PARSE_SEQ;
struct next_input_state input_state;
input_state.parser = jv_parser_new(parser_flags);
if ((options & RAW_INPUT) && (options & SLURP))
input_state.slurped = jv_string("");
else if ((options & SLURP))
input_state.slurped = jv_array();
else
input_state.slurped = jv_invalid();
// Let jq program read from inputs
jq_set_input_cb(jq, next_input, &input_state);
jq_set_debug_cb(jq, debug_cb, &dumpopts);
if (options & PROVIDE_NULL) {
ret = process(jq, jv_null(), jq_flags, dumpopts);
process(jv_null(), jq_flags);
} else {
jv value;
while (jv_is_valid((value = next_input(jq, &input_state))) || jv_invalid_has_msg(jv_copy(value))) {
if (jv_is_valid(value)) {
ret = process(jq, value, jq_flags, dumpopts);
continue;
}
// Parse error
jv msg = jv_invalid_get_msg(value);
if (!(options & SEQ)) {
// --seq -> errors are not fatal
ret = 4;
fprintf(stderr, "parse error: %s\n", jv_string_value(msg));
jv_free(msg);
break;
}
fprintf(stderr, "ignoring parse error: %s\n", jv_string_value(msg));
jv_free(msg);
}
jv slurped;
if (options & SLURP) {
ret = process(jq, input_state.slurped, jq_flags, dumpopts);
input_state.slurped = jv_invalid();
if (options & RAW_INPUT) {
slurped = jv_string("");
} else {
slurped = jv_array();
}
}
}
jv_free(input_state.slurped);
jv_parser_free(input_state.parser);
if (ret != 0)
goto out;
if ((options & IN_PLACE)) {
FILE *devnull;
#ifdef WIN32
devnull = freopen("NUL", "w+", stdout);
#else
devnull = freopen("/dev/null", "w+", stdout);
#endif
if (devnull == NULL) {
fprintf(stderr, "Error: %s opening /dev/null\n", strerror(errno));
exit(3);
struct jv_parser parser;
jv_parser_init(&parser);
char buf[4096];
while (read_more(buf, sizeof(buf))) {
if (options & RAW_INPUT) {
int len = strlen(buf);
if (len > 0) {
if (options & SLURP) {
slurped = jv_string_concat(slurped, jv_string(buf));
} else {
if (buf[len-1] == '\n') buf[len-1] = 0;
process(jv_string(buf), jq_flags);
}
}
} else {
jv_parser_set_buf(&parser, buf, strlen(buf), !feof(stdin));
jv value;
while (jv_is_valid((value = jv_parser_next(&parser)))) {
if (options & SLURP) {
slurped = jv_array_append(slurped, value);
} else {
process(value, jq_flags);
}
}
if (jv_invalid_has_msg(jv_copy(value))) {
jv msg = jv_invalid_get_msg(value);
fprintf(stderr, "parse error: %s\n", jv_string_value(msg));
jv_free(msg);
ret = 1;
break;
} else {
jv_free(value);
}
}
}
if (rename(t, input_filenames[0]) == -1) {
fprintf(stderr, "Error: %s renaming temporary file\n", strerror(errno));
exit(3);
jv_parser_free(&parser);
if (ret != 0)
goto out;
if (options & SLURP) {
process(slurped, jq_flags);
}
jv_mem_free(t);
}
out:
jv_mem_free(input_filenames);
jq_teardown(&jq);
if (ret >= 10 && (options & EXIT_STATUS))
return ret - 10;
if (ret >= 10)
return 0;
bytecode_free(bc);
return ret;
}
opcode.c
+31
View File
@@ -0,0 +1,31 @@
#include "opcode.h"
// flags, length
#define NONE 0, 1
#define CONSTANT OP_HAS_CONSTANT, 2
#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
#define BRANCH OP_HAS_BRANCH, 2
#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
#define OP(name, imm, in, out) \
{name, #name, imm, in, out},
static const struct opcode_description opcode_descriptions[] = {
#include "opcode_list.h"
};
static const struct opcode_description invalid_opcode_description = {
-1, "#INVALID", 0, 0, 0, 0
};
const struct opcode_description* opcode_describe(opcode op) {
if ((int)op >= 0 && (int)op < NUM_OPCODES) {
return &opcode_descriptions[op];
} else {
return &invalid_opcode_description;
}
}
opcode.h
+41
View File
@@ -0,0 +1,41 @@
#ifndef OPCODE_H
#define OPCODE_H
#include <assert.h>
typedef enum {
#define OP(name, imm, in, out) name,
#include "opcode_list.h"
#undef OP
} opcode;
enum {
NUM_OPCODES =
#define OP(name, imm, in, out) +1
#include "opcode_list.h"
#undef OP
};
enum {
OP_HAS_CONSTANT = 2,
OP_HAS_VARIABLE = 4,
OP_HAS_BRANCH = 8,
OP_HAS_CFUNC = 32,
OP_HAS_UFUNC = 64,
OP_IS_CALL_PSEUDO = 128,
OP_HAS_BINDING = 1024,
};
struct opcode_description {
opcode op;
const char* name;
int flags;
// length in 16-bit units
int length;
int stack_in, stack_out;
};
const struct opcode_description* opcode_describe(opcode op);
#endif
opcode_list.h
-12
View File
@@ -1,18 +1,13 @@
OP(LOADK, CONSTANT, 1, 1)
OP(DUP, NONE, 1, 2)
OP(DUPN, NONE, 1, 2)
OP(DUP2, NONE, 2, 3)
OP(POP, NONE, 1, 0)
OP(LOADV, VARIABLE, 1, 1)
OP(LOADVN, VARIABLE, 1, 1)
OP(STOREV, VARIABLE, 1, 0)
OP(STORE_GLOBAL, GLOBAL, 0, 0)
OP(INDEX, NONE, 2, 1)
OP(INDEX_OPT, NONE, 2, 1)
OP(EACH, NONE, 1, 1)
OP(EACH_OPT, NONE, 1, 1)
OP(FORK, BRANCH, 0, 0)
OP(FORK_OPT, BRANCH, 0, 0)
OP(JUMP, BRANCH, 0, 0)
OP(JUMP_F,BRANCH, 1, 0)
OP(BACKTRACK, NONE, 0, 0)
@@ -30,15 +25,8 @@ OP(CALL_BUILTIN, CFUNC, -1, 1)
OP(CALL_JQ, UFUNC, 1, 1)
OP(RET, NONE, 1, 1)
OP(TAIL_CALL_JQ, UFUNC, 1, 1)
OP(CLOSURE_PARAM, DEFINITION, 0, 0)
OP(CLOSURE_REF, CLOSURE_REF_IMM, 0, 0)
OP(CLOSURE_CREATE, DEFINITION, 0, 0)
OP(CLOSURE_CREATE_C, DEFINITION, 0, 0)
OP(TOP, NONE, 0, 0)
OP(CLOSURE_PARAM_REGULAR, DEFINITION, 0, 0)
OP(DEPS, CONSTANT, 0, 0)
OP(MODULEMETA, CONSTANT, 0, 0)
OP(GENLABEL, NONE, 0, 1)
parser.y
+79 -359
View File
@@ -1,16 +1,16 @@
%{
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include "compile.h"
#include "jv_alloc.h"
#define YYMALLOC jv_mem_alloc
#define YYFREE jv_mem_free
struct lexer_param;
%}
%code requires {
#include "locfile.h"
struct lexer_param;
#define YYLTYPE location
#define YYLLOC_DEFAULT(Loc, Rhs, N) \
do { \
@@ -47,33 +47,21 @@ struct lexer_param;
%token INVALID_CHARACTER
%token <literal> IDENT
%token <literal> FIELD
%token <literal> LITERAL
%token <literal> FORMAT
%token REC ".."
%token SETMOD "%="
%token EQ "=="
%token NEQ "!="
%token DEFINEDOR "//"
%token AS "as"
%token DEF "def"
%token MODULE "module"
%token IMPORT "import"
%token IF "if"
%token THEN "then"
%token ELSE "else"
%token ELSE_IF "elif"
%token REDUCE "reduce"
%token FOREACH "foreach"
%token END "end"
%token AND "and"
%token OR "or"
%token TRY "try"
%token CATCH "catch"
%token LABEL "label"
%token BREAK "break"
%token BREAK2 "break2"
%token BREAK3 "break3"
%token SETPIPE "|="
%token SETPLUS "+="
%token SETMINUS "-="
@@ -89,26 +77,20 @@ struct lexer_param;
%token QQSTRING_INTERP_END
%token QQSTRING_END
/* see Exp '?' rule */
%expect 34
/* revolting hack */
%left ';'
%right '|'
%left ','
%right "//"
%nonassoc '=' SETPIPE SETPLUS SETMINUS SETMULT SETDIV SETMOD SETDEFINEDOR
%nonassoc '=' SETPIPE SETPLUS SETMINUS SETMULT SETDIV SETDEFINEDOR
%left OR
%left AND
%nonassoc NEQ EQ '<' '>' LESSEQ GREATEREQ
%left '+' '-'
%left '*' '/' '%'
%precedence '?'
%precedence "try"
%precedence "catch"
%left '*' '/'
%type <blk> Exp Term MkDict MkDictPair ExpD ElseBody QQString
%type <blk> FuncDef FuncDefs String Import Imports Param Params
%type <blk> Arg Args Module
%type <blk> Exp Term MkDict MkDictPair ExpD ElseBody QQString FuncDef FuncDefs String
%{
#include "lexer.h"
struct lexer_param {
@@ -124,37 +106,33 @@ struct lexer_param {
void yyerror(YYLTYPE* loc, block* answer, int* errors,
struct locfile* locations, struct lexer_param* lexer_param_ptr, const char *s){
(*errors)++;
if (strstr(s, "unexpected")) {
#ifdef WIN32
locfile_locate(locations, *loc, "jq: error: %s (Windows cmd shell quoting issues?)", s);
#else
locfile_locate(locations, *loc, "jq: error: %s (Unix shell quoting issues?)", s);
#endif
} else {
locfile_locate(locations, *loc, "jq: error: %s", s);
}
locfile_locate(locations, *loc, "error: %s", s);
}
int yylex(YYSTYPE* yylval, YYLTYPE* yylloc, block* answer, int* errors,
struct locfile* locations, struct lexer_param* lexer_param_ptr) {
yyscan_t lexer = lexer_param_ptr->lexer;
int tok = jq_yylex(yylval, yylloc, lexer);
if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
if (jv_get_kind(msg) == JV_KIND_STRING) {
FAIL(*yylloc, jv_string_value(msg));
while (1) {
int tok = jq_yylex(yylval, yylloc, lexer);
if (tok == INVALID_CHARACTER) {
FAIL(*yylloc, "Invalid character");
} else {
FAIL(*yylloc, "Invalid literal");
if ((tok == LITERAL || tok == QQSTRING_TEXT) && !jv_is_valid(yylval->literal)) {
jv msg = jv_invalid_get_msg(jv_copy(yylval->literal));
if (jv_get_kind(msg) == JV_KIND_STRING) {
FAIL(*yylloc, jv_string_value(msg));
} else {
FAIL(*yylloc, "Invalid literal");
}
jv_free(msg);
jv_free(yylval->literal);
yylval->literal = jv_null();
}
return tok;
}
jv_free(msg);
jv_free(yylval->literal);
yylval->literal = jv_null();
}
return tok;
}
static unsigned int next_label = 0;
static block gen_dictpair(block k, block v) {
return BLOCK(gen_subexp(k), gen_subexp(v), gen_op_simple(INSERT));
}
@@ -163,11 +141,7 @@ static block gen_index(block obj, block key) {
return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX));
}
static block gen_index_opt(block obj, block key) {
return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX_OPT));
}
static block gen_slice_index(block obj, block start, block end, opcode idx_op) {
static block gen_slice_index(block obj, block start, block end) {
block key = BLOCK(gen_subexp(gen_const(jv_object())),
gen_subexp(gen_const(jv_string("start"))),
gen_subexp(start),
@@ -175,60 +149,16 @@ static block gen_slice_index(block obj, block start, block end, opcode idx_op) {
gen_subexp(gen_const(jv_string("end"))),
gen_subexp(end),
gen_op_simple(INSERT));
return BLOCK(key, obj, gen_op_simple(idx_op));
}
static block constant_fold(block a, block b, int op) {
if (!block_is_single(a) || !block_is_const(a) ||
!block_is_single(b) || !block_is_const(b))
return gen_noop();
if (block_const_kind(a) != block_const_kind(b))
return gen_noop();
jv res = jv_invalid();
if (block_const_kind(a) == JV_KIND_NUMBER) {
double na = jv_number_value(block_const(a));
double nb = jv_number_value(block_const(b));
switch (op) {
case '+': res = jv_number(na + nb); break;
case '-': res = jv_number(na - nb); break;
case '*': res = jv_number(na * nb); break;
case '/': res = jv_number(na / nb); break;
case EQ: res = (na == nb ? jv_true() : jv_false()); break;
case NEQ: res = (na != nb ? jv_true() : jv_false()); break;
case '<': res = (na < nb ? jv_true() : jv_false()); break;
case '>': res = (na > nb ? jv_true() : jv_false()); break;
case LESSEQ: res = (na <= nb ? jv_true() : jv_false()); break;
case GREATEREQ: res = (na >= nb ? jv_true() : jv_false()); break;
default: break;
}
} else if (op == '+' && block_const_kind(a) == JV_KIND_STRING) {
res = jv_string_concat(block_const(a), block_const(b));
} else {
return gen_noop();
}
if (jv_get_kind(res) == JV_KIND_INVALID)
return gen_noop();
block_free(a);
block_free(b);
return gen_const(res);
return BLOCK(key, obj, gen_op_simple(INDEX));
}
static block gen_binop(block a, block b, int op) {
block folded = constant_fold(a, b, op);
if (!block_is_noop(folded))
return folded;
const char* funcname = 0;
switch (op) {
case '+': funcname = "_plus"; break;
case '-': funcname = "_minus"; break;
case '*': funcname = "_multiply"; break;
case '/': funcname = "_divide"; break;
case '%': funcname = "_mod"; break;
case EQ: funcname = "_equal"; break;
case NEQ: funcname = "_notequal"; break;
case '<': funcname = "_less"; break;
@@ -246,22 +176,24 @@ static block gen_format(block a, jv fmt) {
}
static block gen_definedor_assign(block object, block val) {
block tmp = gen_op_var_fresh(STOREV, "tmp");
block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
gen_noop(), OP_HAS_VARIABLE);
return BLOCK(gen_op_simple(DUP),
val, tmp,
gen_call("_modify", BLOCK(gen_lambda(object),
gen_lambda(gen_definedor(gen_noop(),
gen_op_bound(LOADV, tmp))))));
gen_op_var_bound(LOADV, tmp))))));
}
static block gen_update(block object, block val, int optype) {
block tmp = gen_op_var_fresh(STOREV, "tmp");
block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
gen_noop(), OP_HAS_VARIABLE);
return BLOCK(gen_op_simple(DUP),
val,
tmp,
gen_call("_modify", BLOCK(gen_lambda(object),
gen_lambda(gen_binop(gen_noop(),
gen_op_bound(LOADV, tmp),
gen_op_var_bound(LOADV, tmp),
optype)))));
}
@@ -269,45 +201,24 @@ static block gen_update(block object, block val, int optype) {
%%
TopLevel:
Module Imports Exp {
*answer = BLOCK($1, $2, gen_op_simple(TOP), $3);
Exp {
*answer = $1;
} |
Module Imports FuncDefs {
*answer = BLOCK($1, $2, $3);
FuncDefs {
*answer = $1;
}
Module:
%empty {
$$ = gen_noop();
} |
"module" Exp ';' {
if (!block_is_const($2)) {
FAIL(@$, "Module metadata must be constant.");
$$ = gen_noop();
} else {
$$ = gen_module($2);
}
}
Imports:
%empty {
$$ = gen_noop();
} |
Import Imports {
$$ = BLOCK($1, $2);
}
FuncDefs:
%empty {
/* empty */ {
$$ = gen_noop();
} |
FuncDef FuncDefs {
$$ = block_bind($1, $2, OP_IS_CALL_PSEUDO);
$$ = block_join($1, $2);
}
Exp:
FuncDef Exp %prec ';' {
$$ = block_bind_referenced($1, $2, OP_IS_CALL_PSEUDO);
$$ = block_bind($1, $2, OP_IS_CALL_PSEUDO);
} |
Term "as" '$' IDENT '|' Exp {
@@ -320,59 +231,14 @@ Term "as" '$' IDENT '|' Exp {
jv_free($5);
} |
"foreach" Term "as" '$' IDENT '(' Exp ';' Exp ';' Exp ')' {
$$ = gen_foreach(jv_string_value($5), $2, $7, $9, $11);
jv_free($5);
} |
"foreach" Term "as" '$' IDENT '(' Exp ';' Exp ')' {
$$ = gen_foreach(jv_string_value($5), $2, $7, $9, gen_noop());
jv_free($5);
} |
"if" Exp "then" Exp ElseBody {
$$ = gen_cond($2, $4, $5);
} |
"if" Exp "then" error {
FAIL(@$, "Possibly unterminated 'if' statement");
"if" Exp error {
FAIL(@$, "Possibly unterminated 'if' statment");
$$ = $2;
} |
"try" Exp "catch" Exp {
//$$ = BLOCK(gen_op_target(FORK_OPT, $2), $2, $4);
$$ = gen_try($2, gen_try_handler($4));
} |
"try" Exp {
//$$ = BLOCK(gen_op_target(FORK_OPT, $2), $2, gen_op_simple(BACKTRACK));
$$ = gen_try($2, gen_op_simple(BACKTRACK));
} |
"try" Exp "catch" error {
FAIL(@$, "Possibly unterminated 'try' statement");
$$ = $2;
} |
"label" '|' Exp {
jv v = jv_string_fmt("*anonlabel%u", next_label++);
$$ = gen_location(@$, locations, gen_label(jv_string_value(v), $3));
jv_free(v);
} |
"label" '$' IDENT '|' Exp {
jv v = jv_string_fmt("*label-%s", jv_string_value($3));
$$ = gen_location(@$, locations, gen_label(jv_string_value(v), $5));
jv_free($3);
jv_free(v);
} |
// This rule conflicts with all the other rules using the '?' operator.
// It doesn't matter which bison prefers: all of them result in the same
// syntax and semantics, but the more specific rules optimize better
// because they use opcodes specifically made for the purpose. We
// expect 9 such conflicts.
Exp '?' {
$$ = gen_try($1, gen_op_simple(BACKTRACK));
} |
Exp '=' Exp {
$$ = gen_call("_assign", BLOCK(gen_lambda($1), gen_lambda($3)));
} |
@@ -437,18 +303,10 @@ Exp '/' Exp {
$$ = gen_binop($1, $3, '/');
} |
Exp '%' Exp {
$$ = gen_binop($1, $3, '%');
} |
Exp "/=" Exp {
$$ = gen_update($1, $3, '/');
} |
Exp SETMOD Exp {
$$ = gen_update($1, $3, '%');
} |
Exp "==" Exp {
$$ = gen_binop($1, $3, EQ);
} |
@@ -477,79 +335,32 @@ Term {
$$ = $1;
}
Import:
"import" String "as" '$' IDENT ';' {
jv v = block_const($2);
// XXX Make gen_import take only blocks and the int is_data so we
// don't have to free so much stuff here
$$ = gen_import(jv_string_value(v), gen_noop(), jv_string_value($5), 1);
block_free($2);
jv_free($5);
jv_free(v);
} |
"import" String "as" IDENT ';' {
jv v = block_const($2);
$$ = gen_import(jv_string_value(v), gen_noop(), jv_string_value($4), 0);
block_free($2);
jv_free($4);
jv_free(v);
} |
"import" String "as" IDENT Exp ';' {
if (!block_is_const($5)) {
FAIL(@$, "Module metadata must be constant.");
$$ = gen_noop();
} else {
jv v = block_const($2);
$$ = gen_import(jv_string_value(v), $5, jv_string_value($4), 0);
jv_free(v);
}
block_free($2);
jv_free($4);
} |
"import" String "as" '$' IDENT Exp ';' {
if (!block_is_const($6)) {
FAIL(@$, "Module metadata must be constant.");
$$ = gen_noop();
} else {
jv v = block_const($2);
$$ = gen_import(jv_string_value(v), $6, jv_string_value($5), 1);
jv_free(v);
}
block_free($2);
jv_free($5);
}
FuncDef:
"def" IDENT ':' Exp ';' {
$$ = gen_function(jv_string_value($2), gen_noop(), $4);
jv_free($2);
} |
"def" IDENT '(' Params ')' ':' Exp ';' {
$$ = gen_function(jv_string_value($2), $4, $7);
jv_free($2);
}
Params:
Param {
$$ = $1;
} |
Params ';' Param {
$$ = BLOCK($1, $3);
}
Param:
'$' IDENT {
$$ = gen_param_regular(jv_string_value($2));
"def" IDENT '(' IDENT ')' ':' Exp ';' {
$$ = gen_function(jv_string_value($2),
gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)),
$7);
jv_free($2);
jv_free($4);
} |
IDENT {
$$ = gen_param(jv_string_value($1));
jv_free($1);
"def" IDENT '(' IDENT ';' IDENT ')' ':' Exp ';' {
$$ = gen_function(jv_string_value($2),
BLOCK(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)),
gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($6))),
$9);
jv_free($2);
jv_free($4);
jv_free($6);
}
String:
QQSTRING_START { $<literal>$ = jv_string("text"); } QQString QQSTRING_END {
$$ = $3;
@@ -562,7 +373,7 @@ FORMAT QQSTRING_START { $<literal>$ = $1; } QQString QQSTRING_END {
QQString:
%empty {
/* empty */ {
$$ = gen_const(jv_string(""));
} |
QQString QQSTRING_TEXT {
@@ -585,9 +396,7 @@ ExpD:
ExpD '|' ExpD {
$$ = block_join($1, $3);
} |
'-' ExpD {
$$ = BLOCK($2, gen_call("_negate", gen_noop()));
} |
Term {
$$ = $1;
}
@@ -597,95 +406,27 @@ Term:
'.' {
$$ = gen_noop();
} |
REC {
$$ = gen_call("recurse", gen_noop());
Term '.' IDENT {
$$ = gen_index($1, gen_const($3));
} |
BREAK {
$$ = gen_location(@$, locations,
BLOCK(gen_op_unbound(LOADV, "*1"), // impossible symbol
gen_call("error", gen_noop())));
'.' IDENT {
$$ = gen_index(gen_noop(), gen_const($2));
} |
BREAK2 {
$$ = gen_location(@$, locations,
BLOCK(gen_op_unbound(LOADV, "*2"), // impossible symbol
gen_call("error", gen_noop())));
} |
BREAK3 {
$$ = gen_location(@$, locations,
BLOCK(gen_op_unbound(LOADV, "*3"), // impossible symbol
gen_call("error", gen_noop())));
} |
BREAK '$' IDENT {
jv v = jv_string_fmt("*label-%s", jv_string_value($3)); // impossible symbol
$$ = gen_location(@$, locations,
BLOCK(gen_op_unbound(LOADV, jv_string_value(v)),
gen_call("error", gen_noop())));
jv_free(v);
jv_free($3);
} |
Term FIELD '?' {
$$ = gen_index_opt($1, gen_const($2));
} |
FIELD '?' {
$$ = gen_index_opt(gen_noop(), gen_const($1));
} |
Term '.' String '?' {
$$ = gen_index_opt($1, $3);
} |
'.' String '?' {
$$ = gen_index_opt(gen_noop(), $2);
} |
Term FIELD {
$$ = gen_index($1, gen_const($2));
} |
FIELD {
$$ = gen_index(gen_noop(), gen_const($1));
} |
Term '.' String {
$$ = gen_index($1, $3);
} |
'.' String {
$$ = gen_index(gen_noop(), $2);
} |
'.' error {
FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
$$ = gen_noop();
} |
'.' IDENT error {
jv_free($2);
FAIL(@$, "try .[\"field\"] instead of .field for unusually named fields");
$$ = gen_noop();
} |
/* FIXME: string literals */
Term '[' Exp ']' '?' {
$$ = gen_index_opt($1, $3);
} |
Term '[' Exp ']' {
$$ = gen_index($1, $3);
} |
Term '[' ']' '?' {
$$ = block_join($1, gen_op_simple(EACH_OPT));
} |
Term '[' ']' {
$$ = block_join($1, gen_op_simple(EACH));
} |
Term '[' Exp ':' Exp ']' '?' {
$$ = gen_slice_index($1, $3, $5, INDEX_OPT);
} |
Term '[' Exp ':' ']' '?' {
$$ = gen_slice_index($1, $3, gen_const(jv_null()), INDEX_OPT);
} |
Term '[' ':' Exp ']' '?' {
$$ = gen_slice_index($1, gen_const(jv_null()), $4, INDEX_OPT);
} |
Term '[' Exp ':' Exp ']' {
$$ = gen_slice_index($1, $3, $5, INDEX);
$$ = gen_slice_index($1, $3, $5);
} |
Term '[' Exp ':' ']' {
$$ = gen_slice_index($1, $3, gen_const(jv_null()), INDEX);
$$ = gen_slice_index($1, $3, gen_const(jv_null()));
} |
Term '[' ':' Exp ']' {
$$ = gen_slice_index($1, gen_const(jv_null()), $4, INDEX);
$$ = gen_slice_index($1, gen_const(jv_null()), $4);
} |
LITERAL {
$$ = gen_const($1);
@@ -706,31 +447,24 @@ FORMAT {
$$ = gen_const(jv_array());
} |
'{' MkDict '}' {
block o = gen_const_object($2);
if (o.first != NULL)
$$ = o;
else
$$ = BLOCK(gen_subexp(gen_const(jv_object())), $2, gen_op_simple(POP));
$$ = BLOCK(gen_subexp(gen_const(jv_object())), $2, gen_op_simple(POP));
} |
'$' IDENT {
$$ = gen_location(@$, locations, gen_op_unbound(LOADV, jv_string_value($2)));
$$ = gen_location(@$, gen_op_var_unbound(LOADV, jv_string_value($2)));
jv_free($2);
} |
IDENT {
const char *s = jv_string_value($1);
if (strcmp(s, "false") == 0)
$$ = gen_const(jv_false());
else if (strcmp(s, "true") == 0)
$$ = gen_const(jv_true());
else if (strcmp(s, "null") == 0)
$$ = gen_const(jv_null());
else
$$ = gen_location(@$, locations, gen_call(s, gen_noop()));
$$ = gen_location(@$, gen_call(jv_string_value($1), gen_noop()));
jv_free($1);
} |
IDENT '(' Args ')' {
$$ = gen_call(jv_string_value($1), $3);
$$ = gen_location(@1, locations, $$);
IDENT '(' Exp ')' {
$$ = gen_call(jv_string_value($1), gen_lambda($3));
$$ = gen_location(@1, $$);
jv_free($1);
} |
IDENT '(' Exp ';' Exp ')' {
$$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5)));
$$ = gen_location(@1, $$);
jv_free($1);
} |
'(' error ')' { $$ = gen_noop(); } |
@@ -738,21 +472,8 @@ IDENT '(' Args ')' {
Term '[' error ']' { $$ = $1; } |
'{' error '}' { $$ = gen_noop(); }
Args:
Arg {
$$ = $1;
} |
Args ';' Arg {
$$ = BLOCK($1, $3);
}
Arg:
Exp {
$$ = gen_lambda($1);
}
MkDict:
%empty {
{
$$=gen_noop();
} |
MkDictPair { $$ = $1; }
@@ -800,10 +521,9 @@ int jq_parse(struct locfile* locations, block* answer) {
int jq_parse_library(struct locfile* locations, block* answer) {
int errs = jq_parse(locations, answer);
if (errs) return errs;
if (block_has_main(*answer)) {
locfile_locate(locations, UNKNOWN_LOCATION, "jq: error: library should only have function definitions, not a main expression");
if (!block_has_only_binders(*answer, OP_IS_CALL_PSEUDO)) {
locfile_locate(locations, UNKNOWN_LOCATION, "error: library should only have function definitions, not a main expression");
return 1;
}
assert(block_has_only_binders_and_imports(*answer, OP_IS_CALL_PSEUDO));
return 0;
}
scripts/crosscompile
+8 -19
View File
@@ -13,30 +13,19 @@
set -e
cd `dirname "$0"`/../build
jobs=-j4
case "X$1" in
X-j*) jobs=$1; shift;;
esac
plat="$1"
[ -z "$plat" ] && exit 1
shift
case "$plat" in
*/*) echo "platform name must not be a path"; exit 1;;
*..*) echo "platform name must not be a path"; exit 1;;
*) plat=$PWD/$plat;;
esac
[ -d "$plat" ] || mkdir "$plat"
rm -rf "$plat/tmp"
mkdir "$plat/tmp"
cd "$plat/tmp"
../../../configure "$@"
make "$jobs" DESTDIR=$plat install
set -x
for jq in `find . -type f \( -name jq -o -name jq.exe \) -print`; do
cp "$jq" ..
done
rm -rf tmp
mkdir install_other
../../../configure \
--prefix="`pwd`/install_other" \
--bindir="`pwd`/.." \
"$@"
make install-binaries
cd ..
rm -rf tmp
scripts/version
-5
View File
@@ -1,5 +0,0 @@
#!/bin/sh
set -e
cd `dirname "$0"`
git rev-parse --verify -q jq-1.0 > /dev/null 2>&1
git describe --tags --match 'jq-*' --dirty | sed 's/^jq-//'
setup.sh
+2 -5
View File
@@ -9,14 +9,12 @@
if [ "clean" == "$1" ]; then
make distclean || echo "continuing..."
find config/m4 -type f \( -name libtool\* -o -name lt\* \) | xargs rm -f
find config -name m4 -prune -o -type f -print | xargs rm -f
rm -rf tmp autom4te.cache
rm -rf tmp autom4te.cache config
rm -f INSTALL Makefile.in aclocal.m4 configure config.h.in ChangeLog
rm -f jv_utf8_tables.gen.h lexer.c lexer.h parser.c parser.h
elif [ "superclean" == "$1" ]; then
# if autoconf errors during distcheck, it leaves files that need chmod'ing
ver=$(scripts/version|tr -d '\n')
ver=$(tr -d '\n' <VERSION)
if [ "x${ver}" != "x" ]; then
if [ -d jq-${ver} ]; then
chmod -R u+w jq-${ver}
@@ -28,7 +26,6 @@ else
autoreconf --install
./configure --prefix=/opt/junk
make check
[ -d tmp ] && mv tmp tmp-
mkdir tmp
make DESTDIR=./tmp install
make distcheck
tests/all.test
+11 -560
View File
@@ -36,10 +36,6 @@ null
null
[]
{x: -1}
null
{"x": -1}
# The input line starts with a 0xFEFF (byte order mark) codepoint
# No, there is no reason to have a byte order mark in UTF8 text.
# But apparently people do, so jq shouldn't break on it.
@@ -61,34 +57,24 @@ null
null
"interpolation"
@text,@json,([1,.] | (@csv, @tsv)),@html,@uri,@sh,@base64
"<>&'\"\t"
"<>&'\"\t"
"\"<>&'\\\"\\t\""
"1,\"<>&'\"\"\t\""
"1\t<>&'\"\\t"
"&lt;&gt;&amp;&apos;&quot;\t"
"%3C%3E%26'%22%09"
"'<>&'\\''\"\t'"
"PD4mJyIJ"
# regression test for #436
@base64
"foóbar\n"
"Zm/Ds2Jhcgo="
@text,@json,([1,.] | @csv),@html,@uri,@sh,@base64
"<>&'\""
"<>&'\""
"\"<>&'\\\"\""
"1,\"<>&'\"\"\""
"&lt;&gt;&amp;&apos;&quot;"
"%3c%3e%26'%22"
"'<>&'\\''\"'"
"PD4mJyI="
@uri
"\u03bc"
"%CE%BC"
"%ce%bc"
@html "<b>\(.)</b>"
"<script>hax</script>"
"<b>&lt;script&gt;hax&lt;/script&gt;</b>"
[.[]|tojson|fromjson]
["foo", 1, ["a", 1, "b", 2, {"foo":"bar"}]]
["foo",1,["a",1,"b",2,{"foo":"bar"}]]
#
# Dictionary construction syntax
#
@@ -129,49 +115,6 @@ null
{"foo": {"bar": 42}, "bar": "badvalue"}
42
."foo"."bar"
{"foo": {"bar": 20}}
20
[.[]|.foo?]
[1,[2],{"foo":3,"bar":4},{},{"foo":5}]
[3,null,5]
[.[]|.foo?.bar?]
[1,[2],[],{"foo":3},{"foo":{"bar":4}},{}]
[4,null]
[..]
[1,[[2]],{ "a":[1]}]
[[1,[[2]],{"a":[1]}],1,[[2]],[2],2,{"a":[1]},[1],1]
[.[]|.[]?]
[1,null,[],[1,[2,[[3]]]],[{}],[{"a":[1,[2]]}]]
[1,[2,[[3]]],{},{"a":[1,[2]]}]
[.[]|.[1:3]?]
[1,null,true,false,"abcdef",{},{"a":1,"b":2},[],[1,2,3,4,5],[1,2]]
[null,"bc",[],[2,3],[2]]
#
# Negative array indices
#
try (.foo[-1] = 0) catch .
null
"Out of bounds negative array index"
try (.foo[-2] = 0) catch .
null
"Out of bounds negative array index"
.[-1] = 5
[0,1,2]
[0,1,5]
.[-2] = 5
[0,1,2]
[0,5,2]
#
# Multiple outputs, iteration
@@ -223,119 +166,6 @@ null
2
3
.[-2]
[1,2,3]
2
[range(0;10)]
null
[0,1,2,3,4,5,6,7,8,9]
[range(0;10;3)]
null
[0,3,6,9]
[range(0;10;-1)]
null
[]
[range(0;-5;-1)]
null
[0,-1,-2,-3,-4]
[while(.<100; .*2)]
1
[1,2,4,8,16,32,64]
[label | while(.<100; .*2|if . > 10 then break else . end)]
1
[1,2,4,8]
[(label $here | .[] | if .>1 then break $here else . end), "hi!"]
[0,1,2]
[0,1,"hi!"]
[(label $here | .[] | if .>1 then break $here else . end), "hi!"]
[0,2,1]
[0,"hi!"]
(label | (label | 2 | break2)), 1
null
1
%%FAIL
break
jq: error: break used outside labeled control structure
%%FAIL
. as $foo | break $foo
jq: error: *label-foo/0 is not defined
[.[]|[.,1]|until(.[0] < 1; [.[0] - 1, .[1] * .[0]])|.[1]]
[1,2,3,4,5]
[1,2,6,24,120]
[label | foreach .[] as $item ([3, null]; if .[0] < 1 then break else [.[0] -1, $item] end; .[1])]
[11,22,33,44,55,66,77,88,99]
[11,22,33]
[foreach range(5) as $item (0; $item)]
null
[0,1,2,3,4]
[limit(3; .[])]
[11,22,33,44,55,66,77,88,99]
[11,22,33]
[first(range(.)), last(range(.)), nth(0; range(.)), nth(5; range(.)), try nth(-1; range(.)) catch .]
10
[0,9,0,5,"nth doesn't support negative indices"]
#
# Check that various builtins evalute all arguments where appropriate,
# doing cartesian products where appropriate.
#
# Check that limit does work for each value produced by n!
[limit(5,7; range(9))]
null
[0,1,2,3,4,0,1,2,3,4,5,6]
# Same check for nth
[nth(5,7; range(9;0;-1))]
null
[4,2]
# Same check for range/3
[range(0,1,2;4,3,2;2,3)]
null
[0,2,0,3,0,2,0,0,0,1,3,1,1,1,1,1,2,2,2,2]
# Same check for range/1
[range(3,5)]
null
[0,1,2,0,1,2,3,4]
# Same check for index/1, rindex/1, indices/1
[(index(",","|"), rindex(",","|")), indices(",","|")]
"a,b|c,d,e||f,g,h,|,|,i,j"
[1,3,22,19,[1,5,7,12,14,16,18,20,22],[3,9,10,17,19]]
# Same check for join/1
join(",","/")
["a","b","c","d"]
"a,b,c,d"
"a/b/c/d"
# Same check for flatten/1
flatten(-1,3,2,1)
[0, [1], [[2]], [[[3]]]]
[0,[1],[[2]],[[[3]]]]
[0,1,2,3]
[0,1,2,[3]]
[0,1,[2],[[3]]]
#
# Slices
#
@@ -344,10 +174,6 @@ flatten(-1,3,2,1)
[0,1,2,3,4,5,6]
[[], [2,3], [0,1,2,3,4], [5,6], [], []]
[.[3:2], .[-5:4], .[:-2], .[-2:], .[3:3][1:], .[10:]]
"abcdefghi"
["","","abcdefg","hi","",""]
del(.[2:4],.[0],.[-2:])
[0,1,2,3,4,5,6,7]
[1,4,5]
@@ -459,14 +285,6 @@ null
null
[2, 8, 10, 14]
25 % 7
null
4
49732 % 472
null
172
1 + tonumber + ("10" | tonumber)
4
15
@@ -514,15 +332,6 @@ def f: (1000,2000); f
1000
2000
def f(a;b;c;d;e;f): [a+1,b,c,d,e,f]; f(.[0];.[1];.[0];.[0];.[0];.[0])
[1,2]
[2,2,1,1,1,1]
# Many arguments
def f(a;b;c;d;e;f;g;h;i;j): [j,i,h,g,f,e,d,c,b,a]; f(.[0];.[1];.[2];.[3];.[4];.[5];.[6];.[7];.[8];.[9])
[0,1,2,3,4,5,6,7,8,9]
[9,8,7,6,5,4,3,2,1,0]
([1,2] + [4,5])
[1,2,3]
[1,2,4,5]
@@ -541,36 +350,6 @@ null
[5,6]
[1,2,3]
[.[]|floor]
[-1.1,1.1,1.9]
[-2, 1, 1]
[.[]|sqrt]
[4,9]
[2,3]
(add / length) as $m | map((. - $m) as $d | $d * $d) | add / length | sqrt
[2,4,4,4,5,5,7,9]
2
# Should write a test that calls the -lm function from C (or bc(1)) to
# check that they match the corresponding jq functions. However,
# there's so little template code standing between that it suffices to
# test a handful of these. The results were checked by eye against
# bc(1).
atan * 4 * 1000000|floor / 1000000
1
3.141592
[(3.141592 / 2) * (range(0;20) / 20)|cos * 1000000|floor / 1000000]
null
[1,0.996917,0.987688,0.972369,0.951056,0.923879,0.891006,0.85264,0.809017,0.760406,0.707106,0.649448,0.587785,0.522498,0.45399,0.382683,0.309017,0.233445,0.156434,0.078459]
[(3.141592 / 2) * (range(0;20) / 20)|sin * 1000000|floor / 1000000]
null
[0,0.078459,0.156434,0.233445,0.309016,0.382683,0.45399,0.522498,0.587785,0.649447,0.707106,0.760405,0.809016,0.85264,0.891006,0.923879,0.951056,0.972369,0.987688,0.996917]
def f(x): x | x; f([.], . + [42])
[1,2,3]
[[[1,2,3]]]
@@ -578,21 +357,11 @@ def f(x): x | x; f([.], . + [42])
[[1,2,3,42]]
[1,2,3,42,42]
# test multiple function arities and redefinition
def f: .+1; def g: f; def f: .+100; def f(a):a+.+11; [(g|f(20)), f]
1
[33,101]
# test closures and lexical scoping
def id(x):x; 2000 as $x | def f(x):1 as $x | id([$x, x, x]); def g(x): 100 as $x | f($x,$x+x); g($x)
"more testing"
[1,100,2100.0,100,2100.0]
# test def f($a) syntax
def x(a;b): a as $a | b as $b | $a + $b; def y($a;$b): $a + $b; def check(a;b): [x(a;b)] == [y(a;b)]; check(.[];.[]*2)
[1,2,3]
true
# test backtracking through function calls and returns
# this test is *evil*
[[20,10][1,0] as $x | def f: (100,200) as $y | def g: [$x + $y, .]; . + $x | g; f[0] | [f][0][1] | f]
@@ -614,22 +383,6 @@ reduce .[] as $x (0; . + $x)
[1,2,4]
7
. as $dot|any($dot[];not)
[1,2,3,4,true,false,1,2,3,4,5]
true
. as $dot|any($dot[];not)
[1,2,3,4,true]
false
. as $dot|all($dot[];.)
[1,2,3,4,true,false,1,2,3,4,5]
false
. as $dot|all($dot[];.)
[1,2,3,4,true]
true
#
# Paths
#
@@ -647,14 +400,6 @@ path(.)
42
[]
[paths]
[1,[[],{"a":2}]]
[[0],[1],[1,0],[1,1],[1,1,"a"]]
[leaf_paths]
[1,[[],{"a":2}]]
[[0],[1,1,"a"]]
["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
{"bar": 42, "foo": ["a", "b", "c", "d"]}
"b"
@@ -703,17 +448,12 @@ null
{"foo": 42}
{"foo": 43}
.[] += 2, .[] *= 2, .[] -= 2, .[] /= 2, .[] %=2
.[] += 2, .[] *= 2, .[] -= 2, .[] /= 2
[1,3,5]
[3,5,7]
[2,6,10]
[-1,1,3]
[0.5, 1.5, 2.5]
[1,1,1]
[.[] % 7]
[-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7]
[0,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,0]
.foo += .foo
{"foo":2}
@@ -805,163 +545,6 @@ def inc(x): x |= .+1; inc(.[].a)
{}
[true, true, false]
# Try/catch and general `?` operator
[.[]|try if . == 0 then error("foo") elif . == 1 then .a elif . == 2 then empty else . end catch .]
[0,1,2,3]
["foo","Cannot index number with string \"a\"",3]
[.[]|(.a, .a)?]
[null,true,{"a":1}]
[null,null,1,1]
[[.[]|[.a,.a]]?]
[null,true,{"a":1}]
[]
# string operations
[.[]|startswith("foo")]
["fo", "foo", "barfoo", "foobar", "barfoob"]
[false, true, false, true, false]
[.[]|endswith("foo")]
["fo", "foo", "barfoo", "foobar", "barfoob"]
[false, true, true, false, false]
# match builtin
[match("( )*"; "g")]
"abc"
[{"offset":0, "length":0, "string":"", "captures":[]},{"offset":1, "length":0, "string":"", "captures":[]},{"offset":2, "length":0, "string":"", "captures":[]}]
[match("( )*"; "gn")]
"abc"
[]
[match("a"; "gi")]
"āáàä"
[]
[match(["(bar)"])]
"foo bar"
[{"offset": 4, "length": 3, "string": "bar", "captures":[{"offset": 4, "length": 3, "string": "bar", "name": null}]}]
# offsets account for combining codepoints and multi-byte UTF-8
[match("bar")]
"ā bar with a combining codepoint U+0304"
[{"offset": 3, "length": 3, "string": "bar", "captures":[]}]
# matches with combining codepoints still count them in their length
[match("bār")]
"a bār"
[{"offset": 2, "length": 4, "string": "bār", "captures":[]}]
[match(".+?\\b")]
"ā two-codepoint grapheme"
[{"offset": 0, "length": 2, "string": "ā", "captures":[]}]
[match(["foo (?<bar123>bar)? foo", "ig"])]
"foo bar foo foo foo"
[{"offset": 0, "length": 11, "string": "foo bar foo", "captures":[{"offset": 4, "length": 3, "string": "bar", "name": "bar123"}]},{"offset":12, "length": 8, "string": "foo foo", "captures":[{"offset": -1, "length": 0, "string": null, "name": "bar123"}]}]
#test builtin
[test("( )*"; "gn")]
"abc"
[false]
[test("ā")]
"ā"
[true]
capture("(?<a>[a-z]+)-(?<n>[0-9]+)")
"xyzzy-14"
{"a":"xyzzy","n":"14"}
# jq-coded utilities built on match:
#
# The second element in these tests' inputs tests the case where the
# fromstring matches both the head and tail of the string
[.[] | sub(", "; ":")]
["a,b, c, d, e,f", ", a,b, c, d, e,f, "]
["a,b:c, d, e,f",":a,b, c, d, e,f, "]
sub("^(?<head>.)"; "Head=\(.head) Tail=")
"abcdef"
"Head=a Tail=bcdef"
[.[] | gsub(", "; ":")]
["a,b, c, d, e,f",", a,b, c, d, e,f, "]
["a,b:c:d:e,f",":a,b:c:d:e,f:"]
[.[] | scan(", ")]
["a,b, c, d, e,f",", a,b, c, d, e,f, "]
[", ",", ",", ",", ",", ",", ",", ",", "]
[.[] | split(", ")]
["a,b, c, d, e,f",", a,b, c, d, e,f, "]
[["a,b","c","d","e,f"],["","a,b","c","d","e,f"]]
########################
[.[]|[[sub(", *";":")], [gsub(", *";":")], [scan(", *")]]]
["a,b, c, d, e,f",", a,b, c, d, e,f, "]
[[["a:b, c, d, e,f"],["a:b:c:d:e:f"],[",",", ",", ",", ",","]],[[":a,b, c, d, e,f, "],[":a:b:c:d:e:f:"],[", ",",",", ",", ",", ",",",", "]]]
[.[]|[[sub(", +";":")], [gsub(", +";":")], [scan(", +")]]]
["a,b, c, d, e,f",", a,b, c, d, e,f, "]
[[["a,b:c, d, e,f"],["a,b:c:d:e,f"],[", ",", ",", "]],[[":a,b, c, d, e,f, "],[":a,b:c:d:e,f:"],[", ",", ",", ",", ",", "]]]
# reference to named captures
gsub("(?<x>.)[^a]*"; "+\(.x)-")
"Abcabc"
"+A-+a-"
[.[]|ltrimstr("foo")]
["fo", "foo", "barfoo", "foobar", "afoo"]
["fo","","barfoo","bar","afoo"]
[.[]|rtrimstr("foo")]
["fo", "foo", "barfoo", "foobar", "foob"]
["fo","","bar","foobar","foob"]
[(index(","), rindex(",")), indices(",")]
"a,bc,def,ghij,klmno"
[1,13,[1,4,8,13]]
indices(1)
[0,1,1,2,3,4,1,5]
[1,2,6]
indices([1,2])
[0,1,2,3,1,4,2,5,1,2,6,7]
[1,8]
indices([1,2])
[1]
[]
indices(", ")
"a,b, cd,e, fgh, ijkl"
[3,9,14]
[.[]|split(",")]
["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
[["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]]
[.[]|split(", ")]
["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
[["a","bc","def","ghij","jklmn","a,b","c,d","e,f"],["a,b,c,d","e,f,g,h"]]
[.[] * 3]
["a", "ab", "abc"]
["aaa", "ababab", "abcabcabc"]
[.[] / ","]
["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
[["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]]
[.[] / ", "]
["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"]
[["a","bc","def","ghij","jklmn","a,b","c,d","e,f"],["a,b,c,d","e,f,g,h"]]
map(.[1] as $needle | .[0] | contains($needle))
[[[],[]], [[1,2,3], [1,2]], [[1,2,3], [3,1]], [[1,2,3], [4]], [[1,2,3], [1,4]]]
[true, true, true, false, false]
@@ -998,10 +581,6 @@ unique
[1,2,5,3,5,3,1,3]
[1,2,3,5]
unique
[]
[]
[min, max, min_by(.[1]), max_by(.[1]), min_by(.[2]), max_by(.[2])]
[[4,2,"a"],[3,1,"a"],[2,4,"a"],[1,3,"a"]]
[[1,3,"a"],[4,2,"a"],[3,1,"a"],[2,4,"a"],[4,2,"a"],[1,3,"a"]]
@@ -1021,131 +600,3 @@ unique
[{a:1}] | .[] | .a=999
null
{"a": 999}
to_entries
{"a": 1, "b": 2}
[{"key":"a", "value":1}, {"key":"b", "value":2}]
from_entries
[{"key":"a", "value":1}, {"key":"b", "value":2}]
{"a": 1, "b": 2}
with_entries(.key |= "KEY_" + .)
{"a": 1, "b": 2}
{"KEY_a": 1, "KEY_b": 2}
map(has("foo"))
[{"foo": 42}, {}]
[true, false]
map(has(2))
[[0,1], ["a","b","c"]]
[false, true]
keys
[42,3,35]
[0,1,2]
[][.]
1000000000000000000
null
map([1,2][0:.])
[-1, 1, 2, 3, 1000000000000000000]
[[1], [1], [1,2], [1,2], [1,2]]
# Test recursive object merge
{"k": {"a": 1, "b": 2}} * .
{"k": {"a": 0,"c": 3}}
{"k": {"a": 0, "b": 2, "c": 3}}
{"k": {"a": 1, "b": 2}, "hello": {"x": 1}} * .
{"k": {"a": 0,"c": 3}, "hello": 1}
{"k": {"a": 0, "b": 2, "c": 3}, "hello": 1}
{"k": {"a": 1, "b": 2}, "hello": 1} * .
{"k": {"a": 0,"c": 3}, "hello": {"x": 1}}
{"k": {"a": 0, "b": 2, "c": 3}, "hello": {"x": 1}}
{"a": {"b": 1}, "c": {"d": 2}, "e": 5} * .
{"a": {"b": 2}, "c": {"d": 3, "f": 9}}
{"a": {"b": 2}, "c": {"d": 3, "f": 9}, "e": 5}
[.[]|arrays]
[1,2,"foo",[],[3,[]],{},true,false,null]
[[],[3,[]]]
[.[]|objects]
[1,2,"foo",[],[3,[]],{},true,false,null]
[{}]
[.[]|iterables]
[1,2,"foo",[],[3,[]],{},true,false,null]
[[],[3,[]],{}]
[.[]|scalars]
[1,2,"foo",[],[3,[]],{},true,false,null]
[1,2,"foo",true,false,null]
[.[]|values]
[1,2,"foo",[],[3,[]],{},true,false,null]
[1,2,"foo",[],[3,[]],{},true,false]
[.[]|booleans]
[1,2,"foo",[],[3,[]],{},true,false,null]
[true,false]
[.[]|nulls]
[1,2,"foo",[],[3,[]],{},true,false,null]
[null]
flatten
[0, [1], [[2]], [[[3]]]]
[0, 1, 2, 3]
flatten(2)
[0, [1], [[2]], [[[3]]]]
[0, 1, 2, [3]]
flatten(2)
[0, [1, [2]], [1, [[3], 2]]]
[0, 1, 2, 1, [3], 2]
transpose
[[1], [2,3]]
[[1,2],[null,3]]
ascii_upcase
"useful but not for é"
"USEFUL BUT NOT FOR é"
bsearch(4)
[1,2,3]
-4
# module system
import "a" as foo; import "b" as bar; def fooa: foo::a; [fooa, bar::a, bar::b, foo::a]
null
["a","b","c","a"]
import "c" as foo; [foo::a, foo::c]
null
[0,"acmehbah"]
modulemeta
"c"
{"whatever":null,"deps":[{"as":"foo","is_data":false,"relpath":"a"},{"search":"./","as":"d","is_data":false,"relpath":"d"},{"search":"./","as":"d2","is_data":false,"relpath":"d"},{"search":"./../lib/jq","as":"e","is_data":false,"relpath":"e"},{"search":"./../lib/jq","as":"f","is_data":false,"relpath":"f"},{"as":"d","is_data":true,"relpath":"data"}]}
%%FAIL
import "syntaxerror" as e; .
jq: error: syntax error, unexpected ';', expecting $end (Unix shell quoting issues?)
%%FAIL
%::wat
jq: error: syntax error, unexpected '%', expecting $end (Unix shell quoting issues?)
import "test_bind_order" as check; check::check
null
true
tests/modules/.jq
-5
View File
@@ -1,5 +0,0 @@
def foo: "baz";
def f: "wat";
def f: "foo";
def g: "bar";
def fg: f+g;
tests/modules/a.jq
-2
View File
@@ -1,2 +0,0 @@
module {version:1.7};
def a: "a";
tests/modules/b/b.jq
-2
View File
@@ -1,2 +0,0 @@
def a: "b";
def b: "c";
tests/modules/c/c.jq
-16
View File
@@ -1,16 +0,0 @@
module {whatever:null};
import "a" as foo;
import "d" as d {search:"./"};
import "d" as d2{search:"./"};
import "e" as e {search:"./../lib/jq"};
import "f" as f {search:"./../lib/jq"};
import "data" as $d;
def a: 0;
def c:
if $d::d[0] != {this:"is a test",that:"is too"} then error("data import is busted")
elif d2::meh != d::meh then error("import twice doesn't work")
elif foo::a != "a" then error("foo::a didn't work as expected")
elif d::meh != "meh" then error("d::meh didn't work as expected")
elif e::bah != "bah" then error("e::bah didn't work as expected")
elif f::f != "f is here" then error("f::f didn't work as expected")
else foo::a + "c" + d::meh + e::bah end;
tests/modules/c/d.jq
-1
View File
@@ -1 +0,0 @@
def meh: "meh";
tests/modules/data.json
-4
View File
@@ -1,4 +0,0 @@
{
"this": "is a test",
"that": "is too"
}
tests/modules/lib/jq/e/e.jq
-1
View File
@@ -1 +0,0 @@
def bah: "bah";
tests/modules/lib/jq/f.jq
-1
View File
@@ -1 +0,0 @@
def f: "f is here";
tests/modules/streaming.jq
-41
View File
@@ -1,41 +0,0 @@
# Filter and adjust streamed values so that only values from the .th
# level are output.
def trunc(inputs):
. as $n | inputs | . as $input | if (.[0]|length) > 1 then setpath([0];$input[0][1:]) else empty end;
# Reduce streamed values back to normal
def tovalues(i):
foreach i as $item (
# [<current value being built>,
# <is current value valid?>,
# <previous, complete value>,
# <is previous value valid?>]
[null,false,null,false];
# Updator
#
# If the new $item is a top-level value,
# then clear out the current value
if ($item[0]|length) == 0 then [null,false,.[2],.[3]]
# else if the new $item terminates the current value,
# then rotate the current value into the previous value slot.
elif ($item|length) == 1 and ($item[0]|length) < 2 then [null,false,.[0],.[1]]
else . end |
. as $state |
# If the new $item has a leaf, upate the current value
if ($item|length) > 1 and ($item[0]|length) > 0 then
[.[0]|setpath(($item|.[0]); ($item|.[1])), # update current value
true, # current value is now valid (if, perhaps, incomplete)
$state[2], # previous value is unchanged
$state[3]] # previous value is unchanged
else .
end;
# Extractor
#
# If previous value is valid, output it
if ($item[0]|length) == 1 and ($item|length == 1) and .[3] then .[2] else empty end,
# and/or if the new $item is a top-level scalar, output it
if ($item[0]|length) == 0 then $item[1] else empty end
);
tests/modules/syntaxerror/syntaxerror.jq
-1
View File
@@ -1 +0,0 @@
wat;
tests/modules/test_bind_order.jq
-4
View File
@@ -1,4 +0,0 @@
import "test_bind_order0" as t;
import "test_bind_order1" as t;
import "test_bind_order2" as t;
def check: if [t::sym0,t::sym1,t::sym2] == [0,1,2] then true else false end;
tests/modules/test_bind_order0.jq
-1
View File
@@ -1 +0,0 @@
def sym0: 0;
tests/modules/test_bind_order1.jq
-2
View File
@@ -1,2 +0,0 @@
def sym0: 1;
def sym1: 1;
tests/modules/test_bind_order2.jq
-2
View File
@@ -1,2 +0,0 @@
def sym1: 2;
def sym2: 2;
tests/onig.supp
-21
View File
@@ -1,21 +0,0 @@
{
onig node recycling
Memcheck:Leak
...
fun:onig_parse_make_tree
...
}
{
onig unicode case insensitivity 1
Memcheck:Leak
...
fun:setup_tree
...
}
{
onig unicode case insensitivity 2
Memcheck:Leak
...
fun:onig*unicode*
...
}
tests/run
+1 -203
View File
@@ -1,205 +1,3 @@
#!/bin/sh
set -e
if which valgrind > /dev/null; then
VALGRIND='valgrind --error-exitcode=1 --leak-check=full --suppressions=tests/onig.supp'
Q=-q
else
VALGRIND=
Q=
fi
mods=$PWD/tests/modules
cat $@ | $VALGRIND $Q ./jq -L "$mods" --run-tests
set -x
clean=true
d=
clean () {
if ! $clean; then
echo "See temp files in $d!"
elif [ -n "$d" ]; then
rm -rf "$d"
fi
}
trap clean EXIT
d=`mktemp -d -t || true`
if [ -z "$d" ]; then
echo "Your OS does not support mktemp(1) -d" 1>&2
exit 0
fi
## Test constant folding
# String constant folding (addition only)
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '"foo"' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
# Numeric constant folding (not all ops yet)
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '1+1' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '1-1' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '2*3' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '9/3' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '9==3' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '9!=3' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '9<=3' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
n=`$VALGRIND $Q ./jq -n --debug-dump-disasm '9>=3' | wc -l`
if [ $n -ne 5 ]; then
echo "Constant expression folding for strings didn't work"
exit 1
fi
v=`scripts/version`
case "$v" in
*-*) v=`echo $v|sed -e 's/-.*$/-master/'`;;
*) true;;
esac
## Test JSON sequence support
cat > $d/expected <<EOF
ignoring parse error: Potentially truncated top-level numeric value at line 1, column 2
ignoring parse error: Truncated value at line 2, column 5
ignoring parse error: Truncated value at line 2, column 25
ignoring parse error: Truncated value at line 2, column 41
EOF
printf '1\0362 3\n[0,1\036[4,5]true"ab"{"c":4\036{}{"d":5,"e":6"\036false\n'|$VALGRIND $Q ./jq -ces --seq '. == [2,3,[4,5],true,"ab",{},false]' > /dev/null 2> $d/out
cmp $d/out $d/expected
cat > $d/expected <<EOF
ignoring parse error: Potentially truncated top-level numeric value at line 1, column 2
ignoring parse error: Truncated value at line 2, column 5
ignoring parse error: Truncated value at line 2, column 25
ignoring parse error: Invalid literal at line 3, column 1
EOF
printf '1\0362 3\n[0,1\036[4,5]true"ab"{"c":4\036{}{"d":5,"e":6"false\n\036null'|$VALGRIND $Q ./jq -ces --seq '. == [2,3,[4,5],true,"ab",{},null]' > /dev/null 2> $d/out
cmp $d/out $d/expected
# Note that here jq sees no inputs at all but it still succeeds because
# --seq ignores parse errors
cat > $d/expected <<EOF
ignoring parse error: Unfinished string at EOF at line 1, column 4
EOF
printf '"foo'|./jq -ce --seq . > $d/out 2>&1
cmp $d/out $d/expected
# Numeric values truncated by EOF are ignored
cat > $d/expected <<EOF
ignoring parse error: Potentially truncated top-level numeric value at EOF at line 1, column 1
EOF
printf '1'|./jq -ce --seq . > $d/out 2>&1
cmp $d/out $d/expected
cat > $d/expected <<EOF
EOF
printf '1\n'|./jq -ces --seq '. == [1]' >/dev/null 2> $d/out
cmp $d/out $d/expected
## Test streaming parser
$VALGRIND $Q ./jq -c '. as $d|path(..) as $p|$d|getpath($p)|scalars_or_empty|[$p,.]' < "$PWD/tests/torture/input0.json" > $d/out0
$VALGRIND $Q ./jq --stream -c '.|select(length==2)' < "$PWD/tests/torture/input0.json" > $d/out1
diff $d/out0 $d/out1
clean=false
if which seq > /dev/null 2>&1; then
# XXX We should try every prefix of input0.json, but that makes this
# test very, very slow when run with valgrind, and the whole point
# is to run it with valgrind.
#
#len=`wc -c < "$PWD/tests/torture/input0.json"`
if [ -z "$VALGRIND" ]; then
start=1
end=`wc -c < "$PWD/tests/torture/input0.json"`
else
start=120
end=151
fi
for i in `seq $start $end`; do
dd "if=tests/torture/input0.json" bs=$i count=1 2>/dev/null |
$VALGRIND ./jq -c . > $d/out0 2>$d/err || true
if [ -n "$VALGRIND" ]; then
grep '^==[0-9][0-9]*== ERROR SUMMARY: 0 errors' $d/err > /dev/null
else
tail -1 $d/err | egrep -i 'assert|abort|core' && false
fi
dd "if=tests/torture/input0.json" bs=$i count=1 2>/dev/null |
$VALGRIND ./jq -cn --stream -L "$mods" 'import "streaming" as streaming; streaming::tovalues(inputs)' > $d/out1 2>$d/err || true
if [ -n "$VALGRIND" ]; then
grep '^==[0-9][0-9]*== ERROR SUMMARY: 0 errors' $d/err > /dev/null
else
tail -1 $d/err | egrep -i 'assert|abort|core' && false
fi
diff $d/out0 $d/out1
done
else
echo "Not doing torture tests"
fi
## Fuzz parser
clean=false
if dd if=/dev/urandom bs=16 count=1024 > $d/rand 2>/dev/null; then
# Have a /dev/urandom, good
$VALGRIND $Q ./jq --seq . $d/rand >/dev/null 2>&1
$VALGRIND $Q ./jq --seq --stream . $d/rand >/dev/null 2>&1
dd if=/dev/urandom bs=16 count=1024 > $d/rand 2>/dev/null
$VALGRIND $Q ./jq --seq . $d/rand >/dev/null 2>&1
$VALGRIND $Q ./jq --seq --stream . $d/rand >/dev/null 2>&1
dd if=/dev/urandom bs=16 count=1024 > $d/rand 2>/dev/null
$VALGRIND $Q ./jq --seq . $d/rand >/dev/null 2>&1
$VALGRIND $Q ./jq --seq --stream . $d/rand >/dev/null 2>&1
fi
clean=true
## Test library/module system
# Check handling of ~/.jq; these can't move into jq_test.c yet because
# they depend on $HOME
if [ "`HOME="$mods" $VALGRIND $Q ./jq -nr fg`" != foobar ]; then
echo "Bug #479 appears to be back" 1>&2
exit 1
fi
if [ `HOME="$mods" $VALGRIND $Q ./jq --debug-dump-disasm -n fg | grep '^[a-z]' | wc -l` -gt 3 ]; then
echo "Binding too many defs into program" 1>&2
exit 1
fi
exit 0
cat $@ | valgrind --error-exitcode=1 -q --leak-check=full ./jq --run-tests
tests/torture/input0.json
-7
View File
@@ -1,7 +0,0 @@
0 12 true false null [] {} [{}] [[]]
[0,01,[12,22,[34,[45,56],7]],[]]
{"a":[1]}
{"a":[{}]}
{"a":[{},[],[[]]]}
{"a":[{"b":[]}]}
{"a":[{"b":[]},{},[2]]}
util.c
-145
View File
@@ -1,145 +0,0 @@
#ifdef HAVE_MEMMEM
#define _GNU_SOURCE
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <assert.h>
#include <fcntl.h>
#include <limits.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#ifndef WIN32
#include <pwd.h>
#endif
#include "util.h"
#include "jv.h"
#ifndef HAVE_MKSTEMP
int mkstemp(char *template) {
size_t len = strlen(template);
int tries=5;
int fd;
// mktemp() truncates template when it fails
char *s = alloca(len + 1);
assert(s != NULL);
strcpy(s, template);
do {
// Restore template
strcpy(template, s);
(void) mktemp(template);
fd = open(template, O_CREAT | O_EXCL | O_RDWR, 0600);
} while (fd == -1 && tries-- > 0);
return fd;
}
#endif
jv expand_path(jv path) {
assert(jv_get_kind(path) == JV_KIND_STRING);
const char *pstr = jv_string_value(path);
jv ret = path;
if (jv_string_length_bytes(jv_copy(path)) > 1 && pstr[0] == '~' && pstr[1] == '/') {
jv home = get_home();
if (jv_is_valid(home)) {
ret = jv_string_fmt("%s/%s",jv_string_value(home),pstr+2);
jv_free(home);
} else {
jv emsg = jv_invalid_get_msg(home);
ret = jv_invalid_with_msg(jv_string_fmt("Could not expand %s. (%s)", pstr, jv_string_value(emsg)));
jv_free(emsg);
}
jv_free(path);
}
return ret;
}
jv get_home() {
jv ret;
char *home = getenv("HOME");
if (!home) {
#ifndef WIN32
struct passwd* pwd = getpwuid(getuid());
if (pwd)
ret = jv_string(pwd->pw_dir);
else
ret = jv_invalid_with_msg(jv_string("Could not find home directory."));
#else
home = getenv("USERPROFILE");
if (!home) {
char *hd = getenv("HOMEDRIVE");
if (!hd) hd = "";
home = getenv("HOMEPATH");
if (!home) {
ret = jv_invalid_with_msg(jv_string("Could not find home directory."));
} else {
ret = jv_string_fmt("%s%s",hd,home);
}
} else {
ret = jv_string(home);
}
#endif
} else {
ret = jv_string(home);
}
return ret;
}
jv jq_realpath(jv path) {
int path_max;
char *buf = NULL;
#ifdef _PC_PATH_MAX
path_max = pathconf(jv_string_value(path),_PC_PATH_MAX);
#else
path_max = PATH_MAX;
#endif
if (path_max > 0) {
buf = malloc(sizeof(char) * path_max);
}
#ifdef WIN32
char *tmp = _fullpath(buf, jv_string_value(path), path_max);
#else
char *tmp = realpath(jv_string_value(path), buf);
#endif
if (tmp == NULL) {
free(buf);
return path;
}
jv_free(path);
path = jv_string(tmp);
free(tmp);
return path;
}
const void *_jq_memmem(const void *haystack, size_t haystacklen,
const void *needle, size_t needlelen) {
#ifdef HAVE_MEMMEM
return (const void*)memmem(haystack, haystacklen, needle, needlelen);
#else
const char *h = haystack;
const char *n = needle;
size_t hi, hi2, ni;
if (haystacklen < needlelen || haystacklen == 0)
return NULL;
for (hi = 0; hi < (haystacklen - needlelen + 1); hi++) {
for (ni = 0, hi2 = hi; ni < needlelen; ni++, hi2++) {
if (h[hi2] != n[ni])
goto not_this;
}
return &h[hi];
not_this:
continue;
}
return NULL;
#endif /* !HAVE_MEMMEM */
}
util.h
-30
View File
@@ -1,30 +0,0 @@
#ifndef UTIL_H
#define UTIL_H
#include "jv.h"
#ifndef HAVE_MKSTEMP
int mkstemp(char *template);
#endif
jv expand_path(jv);
jv get_home(void);
jv jq_realpath(jv);
const void *_jq_memmem(const void *haystack, size_t haystacklen,
const void *needle, size_t needlelen);
#ifndef MIN
#define MIN(a,b) \
({ __typeof__ (a) _a = (a); \
__typeof__ (b) _b = (b); \
_a < _b ? _a : _b; })
#endif
#ifndef MAX
#define MAX(a,b) \
({ __typeof__ (a) _a = (a); \
__typeof__ (b) _b = (b); \
_a > _b ? _a : _b; })
#endif
#endif /* UTIL_H */