From 73c657cdc477a9ec97548991e355b766e06634a2 Mon Sep 17 00:00:00 2001 From: Stephen Dolan Date: Sun, 16 Dec 2012 13:06:03 +0000 Subject: [PATCH] Lots of build system and docs improvements, including a manpage. - Build binaries for multiple platforms - Make a manpage out of the manual (see #19) - Extract more tests from the documentation - Fix a few documentation bugs uncovered by above. --- .gitignore | 2 + Makefile | 54 ++++++++++--- docs/Gemfile | 1 + docs/Gemfile.lock | 8 ++ docs/Rakefile | 42 ++++++++++ docs/content/3.manual/manual.yml | 132 +++++++++++++++++-------------- jq_test.c | 40 +++++++--- 7 files changed, 197 insertions(+), 82 deletions(-) diff --git a/.gitignore b/.gitignore index e43b388..6caf43c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,7 @@ jv_test jv_parse parsertest*~ +build + # Something delightfully recursive happens otherwise docs/content/2.download/source/* \ No newline at end of file diff --git a/Makefile b/Makefile index 0c2d317..5dfbb7f 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,13 @@ -CC=gcc -Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function +CC=gcc +CFLAGS=-Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function + prefix=/usr/local +mandir=$(prefix)/share/man + .PHONY: all clean releasedep tarball install uninstall test releasetag all: jq - lexer.gen.c: lexer.l flex -o lexer.gen.c --header-file=lexer.gen.h lexer.l lexer.gen.h: lexer.gen.c @@ -23,16 +26,45 @@ main.c: version.gen.h JQ_SRC=parser.gen.c lexer.gen.c opcode.c bytecode.c compile.c execute.c builtin.c jv.c jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c - +jq_test: CFLAGS += -DJQ_DEBUG=1 jq_test: $(JQ_SRC) jq_test.c - $(CC) -DJQ_DEBUG=1 -o $@ $^ + $(CC) $(CFLAGS) $(CFLAGS_DEBUG) -o $@ $^ +jq: CFLAGS += -O -DJQ_DEBUG=0 jq: $(JQ_SRC) main.c - $(CC) -O -DJQ_DEBUG=0 -o $@ $^ + $(CC) $(CFLAGS) $(CFLAGS_OPT) -o $@ $^ test: jq_test valgrind --error-exitcode=1 -q --leak-check=full ./jq_test >/dev/null +BINARIES=jq jq_test +PLATFORMS=linux32 linux64 osx32 osx64 win32 win64 + +build/linux32%: CC='x86_64-linux-gnu-gcc -m32' +build/linux64%: CC='x86_64-linux-gnu-gcc -m64' + +# OS X cross compilers can be gotten from +# https://launchpad.net/~flosoft/+archive/cross-apple +build/osx32%: CC='i686-apple-darwin10-gcc -m32' +build/osx64%: CC='i686-apple-darwin10-gcc -m64' + +# On Debian, you can get windows compilers in the +# gcc-mingw-w64-i686 and gcc-mingw-w64-x86-64 packages. +build/win32%: CC='i686-w64-mingw32-gcc -m32' +build/win64%: CC='x86_64-w64-mingw32-gcc -m64' + +ALL_BINARIES=$(foreach platform, $(PLATFORMS), $(foreach binary, $(BINARIES), build/$(platform)/$(binary))) + +$(ALL_BINARIES): build/%: + mkdir -p $(@D) + make -B $(BINARIES) CC=$(CC) + cp $(BINARIES) $(@D) + +binaries: $(ALL_BINARIES) + +clean: + rm -rf build + rm -f $(BINARIES) *.gen.* releasedep: lexer.gen.c parser.gen.c jv_utf8_tables.gen.h @@ -45,12 +77,16 @@ docs/content/2.download/source/jq.tgz: jq tarball: docs/content/2.download/source/jq.tgz -install: jq +jq.1: docs/content/3.manual/manual.yml + ( cd docs; rake manpage; ) > $@ + +install: jq jq.1 install -d -m 0755 $(prefix)/bin install -m 0755 jq $(prefix)/bin + install -d -m 0755 $(mandir)/man1 + install -m 0755 jq.1 $(mandir)/man1 uninstall: - test -d $(prefix)/bin && \ - cd $(prefix)/bin && \ - rm -f jq + rm -vf $(prefix)/bin/jq + rm -vf $(mandir)/man1/jq.1 diff --git a/docs/Gemfile b/docs/Gemfile index 84f2c4e..f06950f 100644 --- a/docs/Gemfile +++ b/docs/Gemfile @@ -2,3 +2,4 @@ source :rubygems gem "bonsai" gem "maruku" +gem "ronn" diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 6926b4a..ccbc84f 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -18,6 +18,7 @@ GEM tilt (>= 1.3) watch (>= 0.1.0) builder (3.1.4) + hpricot (0.8.6) i18n (0.6.1) launchy (2.1.2) addressable (~> 2.3) @@ -25,9 +26,15 @@ GEM maruku (0.6.1) syntax (>= 1.0.0) multi_json (1.5.0) + mustache (0.99.4) rack (1.4.1) rack-protection (1.3.2) rack + rdiscount (1.6.8) + ronn (0.7.3) + hpricot (>= 0.8.2) + mustache (>= 0.7.0) + rdiscount (>= 1.5.8) sass (3.2.3) sinatra (1.3.3) rack (~> 1.3, >= 1.3.6) @@ -43,3 +50,4 @@ PLATFORMS DEPENDENCIES bonsai maruku + ronn diff --git a/docs/Rakefile b/docs/Rakefile index 51e9b48..da2d4cf 100644 --- a/docs/Rakefile +++ b/docs/Rakefile @@ -2,6 +2,8 @@ require 'bonsai' require 'liquid' require 'maruku' require 'json' +require 'ronn' +require 'tempfile' module ExtraFilters def markdownify(input) @@ -59,3 +61,43 @@ task :build do Bonsai.root_dir = Dir.pwd Bonsai::Exporter.publish! end + +def load_manual + YAML::ENGINE.yamler = 'syck' + YAML::load(File.open("content/3.manual/manual.yml")) +end + +task :manpage do + Tempfile.open "manpage" do |f| + manual = load_manual + f.puts manual['manpage_intro'] + f.puts manual['body'] + manual['sections'].each do |section| + + f.puts "## #{section['title'].upcase}\n" + f.puts section['body'] + f.puts "" + (section['entries'] || []).each do |entry| + f.puts "### #{entry['title']}\n" + f.puts entry['body'] + f.puts "" + end + f.puts "" + end + f.close + puts Ronn::Document.new(f.path).convert('roff').gsub(/<\/?code>/,"") + end +end + +task :mantests do + load_manual['sections'].each do |section| + (section['entries'] || []).each do |entry| + (entry['examples'] || []).each do |example| + puts example['program'].gsub("\n", " ") + puts example['input'] + example['output'].each do |s| puts s end + puts + end + end + end +end diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 15dcb48..f0cedf4 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -26,6 +26,15 @@ body: | But that's getting ahead of ourselves. :) Let's start with something simpler: + +manpage_intro: | + jq(1) -- Command-line JSON processor + ==================================== + + ## SYNOPSIS + + `jq` [...] + sections: - title: Invoking jq @@ -40,51 +49,51 @@ sections: You can affect how jq reads and writes its input and output using some command-line options: - * `--slurp`/`-s` - - Instead of running the filter for each JSON object in the - input, read the entire input stream into a large array and run - the filter just once. - - * `--raw-input`/`-R` - - Don't parse the input as JSON. Instead, each line of text is - passed to the filter as a string. If combined with `--slurp`, - then the entire input is passed to the filter as a single long - string. - - * `--null-input`/`-n` - - Don't read any input at all! Instead, the filter is run once - using `null` as the input. This is useful when using jq as a - simple calculator or to construct JSON data from scratch. - - * `--compact-output` / `-c` - - By default, jq pretty-prints JSON output. Using this option - will result in more compact output by instead putting each - JSON object on a single line. - - * `--colour-output` / `-C` and `--monochrome-output` / `-M` - - By default, jq outputs colored JSON if writing to a - terminal. You can force it to produce color even if writing to - a pipe or a file using `-C`, and disable color with `-M`. - - * `--ascii-output` / `-a` - - jq usually outputs non-ASCII Unicode codepoints as UTF-8, even - if the input specified them as escape sequences (like - "\u03bc"). Using this option, you can force jq to produce pure - ASCII output with every non-ASCII character replaced with the - equivalent escape sequence. - - * `--raw-output` / `-r` - - With this option, if the filter's result is a string then it - will be written directly to standard output rather than being - formatted as a JSON string with quotes. This can be useful for - making jq filters talk to non-JSON-based systems. + * `--slurp`/`-s`: + + Instead of running the filter for each JSON object in the + input, read the entire input stream into a large array and run + the filter just once. + + * `--raw-input`/`-R`: + + Don't parse the input as JSON. Instead, each line of text is + passed to the filter as a string. If combined with `--slurp`, + then the entire input is passed to the filter as a single long + string. + + * `--null-input`/`-n`: + + Don't read any input at all! Instead, the filter is run once + using `null` as the input. This is useful when using jq as a + simple calculator or to construct JSON data from scratch. + + * `--compact-output` / `-c`: + + By default, jq pretty-prints JSON output. Using this option + will result in more compact output by instead putting each + JSON object on a single line. + + * `--colour-output` / `-C` and `--monochrome-output` / `-M`: + + By default, jq outputs colored JSON if writing to a + terminal. You can force it to produce color even if writing to + a pipe or a file using `-C`, and disable color with `-M`. + + * `--ascii-output` / `-a`: + + jq usually outputs non-ASCII Unicode codepoints as UTF-8, even + if the input specified them as escape sequences (like + "\u03bc"). Using this option, you can force jq to produce pure + ASCII output with every non-ASCII character replaced with the + equivalent escape sequence. + + * `--raw-output` / `-r`: + + With this option, if the filter's result is a string then it + will be written directly to standard output rather than being + formatted as a JSON string with quotes. This can be useful for + making jq filters talk to non-JSON-based systems. - title: Basic filters entries: @@ -148,7 +157,7 @@ sections: examples: - program: '.[]' - input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]' + input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' output: - '{"name":"JSON", "good":true}' - '{"name":"XML", "good":false}' @@ -178,7 +187,7 @@ sections: - program: '.[4,2]' input: '["a","b","c","d","e"]' - output: ['"d"', '"c"'] + output: ['"e"', '"c"'] - title: "`|`" body: | @@ -194,7 +203,7 @@ sections: examples: - program: '.[] | .name' - input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]' + input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' output: ['"JSON"', '"XML"'] - title: Types and Values @@ -325,7 +334,7 @@ sections: examples: - program: '.a + 1' input: '{"a": 7}' - output: '{"a": 8}' + output: ['8'] - program: '.a + .b' input: '{"a": [1,2], "b": [3,4]}' output: ['[1,2,3,4]'] @@ -409,11 +418,11 @@ sections: `foo` returns true for that input, and produces no output otherwise. - It's useful for filtering lists: `[1,2,3] | select(. >= 2)` + It's useful for filtering lists: `[1,2,3] | map(select(. >= 2))` will give you `[3]`. examples: - - program: 'select(. >= 2)' + - program: 'map(select(. >= 2))' input: '[1,5,3,0,7]' output: ['[5,3,7]'] @@ -427,7 +436,7 @@ sections: examples: - program: '1, empty, 2' input: 'null' - output: [1,2] + output: [1, 2] - program: '[1,2,empty,3]' input: 'null' output: ['[1,2,3]'] @@ -461,7 +470,7 @@ sections: examples: - program: add input: '["a","b","c"]' - output: ["abc"] + output: ['"abc"'] - program: add input: '[1, 2, 3]' output: [6] @@ -479,7 +488,7 @@ sections: examples: - program: '.[] | tonumber' input: '[1, "1"]' - output: [1,1] + output: [1, 1] - title: `tostring` body: | @@ -593,10 +602,10 @@ sections: input: '["foobar", "foobaz", "blarp"]' output: ['false'] - program: 'contains({foo: 12, bar: [{barp: 12}]})' - input: '{foo: 12, bar:[1,2,{barp:12, blip:13}]}' + input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}' output: ['true'] - program: 'contains({foo: 12, bar: [{barp: 15}]})' - input: '{foo: 12, bar:[1,2,{barp:12, blip:13}]}' + input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}' output: ['false'] - title: "String interpolation - `\(foo)`" @@ -633,7 +642,7 @@ sections: examples: - program: '.[] == 1' input: '[1, 1.0, "1", "banana"]' - output: ['[true, true, false, false]'] + output: ['true', 'true', 'false', 'false'] - title: if-then-else body: | @@ -854,12 +863,12 @@ sections: input's `.foo` field to each element of the array. examples: - - program: 'def addvalue(f): map(. + [$value]); addvalue(.[0])' + - program: 'def addvalue(f): . + [f]; map(addvalue(.[0]))' input: '[[1,2],[10,20]]' output: ['[[1,2,1], [10,20,10]]'] - - program: 'def addvalue(f): f as $x | map (. + $x); addvalue(.[0])' + - program: 'def addvalue(f): f as $x | map(. + $x); addvalue(.[0])' input: '[[1,2],[10,20]]' - output: ['[[1,2,[1,2]], [10,20,[1,2]]]'] + output: ['[[1,2,1,2], [10,20,1,2]]'] - title: Assignment @@ -971,4 +980,5 @@ sections: "stedolan" wrote, and we can comment on each of them in the same way that we did before: - (.posts[] | select(.author == "stedolan") | .comments) |= . + ["terrible."] + (.posts[] | select(.author == "stedolan") | .comments) |= + . + ["terrible."] diff --git a/jq_test.c b/jq_test.c index 2960162..c9bdaef 100644 --- a/jq_test.c +++ b/jq_test.c @@ -8,9 +8,24 @@ static void jv_test(); static void run_jq_tests(); -int main() { +FILE* testdata; + +int main(int argc, char* argv[]) { jv_test(); + if (argc == 1) { + testdata = fopen("testdata", "r"); + } else if (argc == 2) { + if (!strcmp(argv[1], "-")) { + testdata = stdin; + } else { + testdata = fopen(argv[1], "r"); + } + } else { + printf("usage: %s OR cat testdata | %s - OR %s testdata\n", argv[0], argv[0], argv[0]); + return 127; + } run_jq_tests(); + if (testdata != stdin) fclose(testdata); } @@ -24,37 +39,40 @@ static int skipline(const char* buf) { } static void run_jq_tests() { - FILE* testdata = fopen("testdata","r"); char buf[4096]; - int tests = 0, passed = 0; + int tests = 0, passed = 0, invalid = 0; while (1) { if (!fgets(buf, sizeof(buf), testdata)) break; if (skipline(buf)) continue; + if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0; printf("Testing %s\n", buf); int pass = 1; + tests++; struct bytecode* bc = jq_compile(buf); - assert(bc); + if (!bc) {invalid++; continue;} +#if JQ_DEBUG printf("Disassembly:\n"); dump_disassembly(2, bc); printf("\n"); +#endif fgets(buf, sizeof(buf), testdata); jv input = jv_parse(buf); - assert(jv_is_valid(input)); + if (!jv_is_valid(input)){ invalid++; continue; } jq_init(bc, input); while (fgets(buf, sizeof(buf), testdata)) { if (skipline(buf)) break; jv expected = jv_parse(buf); - assert(jv_is_valid(expected)); + if (!jv_is_valid(expected)){ invalid++; continue; } jv actual = jq_next(); if (!jv_is_valid(actual)) { jv_free(actual); - printf("Insufficient results\n"); + printf("*** Insufficient results\n"); pass = 0; break; } else if (!jv_equal(jv_copy(expected), jv_copy(actual))) { - printf("Expected "); + printf("*** Expected "); jv_dump(jv_copy(expected), 0); printf(", but got "); jv_dump(jv_copy(actual), 0); @@ -72,7 +90,7 @@ static void run_jq_tests() { if (pass) { jv extra = jq_next(); if (jv_is_valid(extra)) { - printf("Superfluous result: "); + printf("*** Superfluous result: "); jv_dump(extra, 0); printf("\n"); pass = 0; @@ -82,11 +100,9 @@ static void run_jq_tests() { } jq_teardown(); bytecode_free(bc); - tests++; passed+=pass; } - fclose(testdata); - printf("%d of %d tests passed\n", passed,tests); + printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid); if (passed != tests) exit(1); } -- 2.40.0