diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3277419..cfa89e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,22 @@ jobs: print('SQLite amalgamation ready') " + - name: Download libyaml amalgamation + shell: bash + run: | + mkdir -p lib/yaml + curl -fsSL "https://github.com/yaml/libyaml/archive/refs/tags/0.2.5.tar.gz" -o /tmp/yaml-0.2.5.tar.gz + tar -xzf /tmp/yaml-0.2.5.tar.gz -C /tmp + cp /tmp/libyaml-0.2.5/include/yaml.h lib/yaml/yaml.h + cp /tmp/libyaml-0.2.5/src/yaml_private.h lib/yaml/yaml_private.h + cp /tmp/libyaml-0.2.5/src/api.c lib/yaml/api.c + cp /tmp/libyaml-0.2.5/src/parser.c lib/yaml/parser.c + cp /tmp/libyaml-0.2.5/src/scanner.c lib/yaml/scanner.c + cp /tmp/libyaml-0.2.5/src/reader.c lib/yaml/reader.c + cp /tmp/libyaml-0.2.5/src/writer.c lib/yaml/writer.c + cp /tmp/libyaml-0.2.5/src/loader.c lib/yaml/loader.c + echo "libyaml 0.2.5 ready" + - name: Build run: zig build -Dbundle-sqlite=true -Doptimize=ReleaseSafe diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e81f68..34f9b6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,6 +86,21 @@ jobs: unzip -j sqlite.zip "*/sqlite3.c" "*/sqlite3.h" -d lib/ echo "SQLite $(grep -m1 SQLITE_VERSION lib/sqlite3.h | awk '{print $3}')" + - name: Download libyaml amalgamation + run: | + mkdir -p lib/yaml + curl -fsSL "https://github.com/yaml/libyaml/archive/refs/tags/0.2.5.tar.gz" -o /tmp/yaml-0.2.5.tar.gz + tar -xzf /tmp/yaml-0.2.5.tar.gz -C /tmp + cp /tmp/libyaml-0.2.5/include/yaml.h lib/yaml/yaml.h + cp /tmp/libyaml-0.2.5/src/yaml_private.h lib/yaml/yaml_private.h + cp /tmp/libyaml-0.2.5/src/api.c lib/yaml/api.c + cp /tmp/libyaml-0.2.5/src/parser.c lib/yaml/parser.c + cp /tmp/libyaml-0.2.5/src/scanner.c lib/yaml/scanner.c + cp /tmp/libyaml-0.2.5/src/reader.c lib/yaml/reader.c + cp /tmp/libyaml-0.2.5/src/writer.c lib/yaml/writer.c + cp /tmp/libyaml-0.2.5/src/loader.c lib/yaml/loader.c + echo "libyaml 0.2.5 ready" + - name: Build run: | VERSION="${GITHUB_REF_NAME#v}" diff --git a/README.md b/README.md index 8e9d282..5cc00c9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Release](https://img.shields.io/github/v/release/vmvarela/sql-pipe)](https://github.com/vmvarela/sql-pipe/releases/latest) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -`sql-pipe` reads CSV, JSON, or NDJSON from stdin or file arguments, loads it into an in-memory SQLite database, runs a SQL query, and prints the results. No server, no schema files, no setup. +`sql-pipe` reads CSV, JSON, NDJSON, YAML, or XML from stdin or file arguments, loads it into an in-memory SQLite database, runs a SQL query, and prints the results. No server, no schema files, no setup. It exists because `awk` is cryptic, spinning up a Python interpreter for a one-liner feels wrong, and `sqlite3 :memory:` takes four commands before you can query anything. If you know SQL and work with CSV in the terminal, this is the tool you've been reaching for. @@ -196,6 +196,24 @@ $ printf '[{"name":"Alice","score":95},{"name":"Bob","score":72}]' \ Alice,95 ``` +For YAML input, pass `-I yaml` (reads a top-level sequence of mappings). Both `.yaml` and `.yml` extensions are auto-detected. All values are loaded as `TEXT`: + +```sh +$ cat users.yaml +- name: Alice + country: US +- name: Bob + country: UK +- name: Carol + country: DE + +$ cat users.yaml | sql-pipe -I yaml 'SELECT name FROM t WHERE country != "US" ORDER BY name' +Bob +Carol +``` + +YAML is auto-detected from the `.yaml` and `.yml` extensions, or use `-I yaml` explicitly. Comments (`#`), flow-style mappings (`[{name: Alice}]`), and standard scalar types (strings, numbers, booleans) work transparently. Multi-document streams and anchors/aliases are rejected with a clear error. + Columns are auto-detected as `INTEGER`, `REAL`, `DATE`, `DATETIME`, or `TEXT` based on the first 100 rows. Date and datetime values are normalized to ISO 8601 on insert, so SQLite date functions (`date()`, `strftime()`, `julianday()`) work immediately. Use `--no-type-inference` to force all columns to `TEXT`: ```sh @@ -258,7 +276,7 @@ $ cat events.xml | sql-pipe -I xml --xml-root events --xml-row event \ ### File arguments -Pass files as positional arguments instead of piping through stdin. Each file becomes a table named after its basename (without extension). The input format is auto-detected from the file extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.xml`): +Pass files as positional arguments instead of piping through stdin. Each file becomes a table named after its basename (without extension). The input format is auto-detected from the file extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.yaml`, `.yml`, `.xml`): ```sh # Single file — no more cat @@ -267,6 +285,9 @@ $ sql-pipe orders.csv 'SELECT * FROM orders WHERE amount > 100' # JSON file — extension tells sql-pipe the format, no -I needed $ sql-pipe data.json 'SELECT * FROM data WHERE score > 80' +# YAML file — .yaml and .yml both auto-detect +$ sql-pipe users.yaml 'SELECT name FROM users WHERE country = "US"' + # Multi-file join — the #1 reason people reach for DuckDB $ sql-pipe orders.csv customers.csv \ 'SELECT c.name, SUM(o.amount) FROM orders o @@ -321,15 +342,15 @@ When `-f` is used, all positional arguments are treated as data files (no positi |------|-------------| | `-d`, `--delimiter ` | Input field delimiter (single character, default `,`) | | `--tsv` | Alias for `--delimiter '\t'` | -| `-I`, `--input-format ` | Input format: `csv` (default), `tsv`, `json`, `ndjson`, `xml`. Overrides file extension auto-detection. | +| `-I`, `--input-format ` | Input format: `csv` (default), `tsv`, `json`, `ndjson`, `yaml`, `xml`. Overrides file extension auto-detection. Both `.yaml` and `.yml` file extensions auto-detect to `yaml`. | | `-O`, `--output-format ` | Output format: `csv` (default), `tsv`, `json`, `ndjson`, `xml`, `markdown` (alias: `md`), `html`, `sql` | | `--sql-table ` | Target table name for `-O sql` INSERT output (default: `t`) | | `--no-type-inference` | Treat all columns as TEXT (skip auto-detection) | | `-H`, `--header` | Print column names as the first output row (CSV/TSV/HTML) | | `--json` | Alias for `--output-format json` (mutually exclusive with `-H`) | | `--max-rows ` | Stop if more than `n` data rows are read (exit 1) | -| `--validate` | Parse the entire input and print a summary (`OK: rows, columns (col TYPE, ...)`) to stdout. Exit 0 on success, exit 2 on parse error. No query required. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, xml). JSON/NDJSON/XML columns are reported as TEXT. | -| `--columns` | Read the input header, print each column name on its own line, and exit 0. Supports CSV, TSV, JSON, NDJSON, and XML input. With `-v`/`--verbose`, also shows the inferred type per column (`name INTEGER`). Respects `--delimiter` and `--tsv`. Mutually exclusive with a query argument. | +| `--validate` | Parse the entire input and print a summary (`OK: rows, columns (col TYPE, ...)`) to stdout. Exit 0 on success, exit 2 on parse error. No query required. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, yaml, xml). JSON/NDJSON/YAML/XML columns are reported as TEXT. | +| `--columns` | Read the input header, print each column name on its own line, and exit 0. Supports CSV, TSV, JSON, NDJSON, YAML, and XML input. With `-v`/`--verbose`, also shows the inferred type per column (`name INTEGER`). Respects `--delimiter` and `--tsv`. Mutually exclusive with a query argument. | | `--sample []` | Print a schema comment block to stderr and the first `` data rows to stdout as CSV (default: `n=10`). The schema block lists each column name and its inferred type, prefixed with `#`. Implies `--header`. Compatible with `--delimiter` and `--tsv`. Mutually exclusive with `--json` and a query argument. No query required. | | `--stats` | Load input and print per-column statistics (column name, type, non-null count, min, max, mean) as a formatted table. Mean is blank for non-numeric columns. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format`. Mutually exclusive with a query, `--columns`, `--validate`, `--sample`, and `--output`. | | `--profile` | Alias for `--stats` | @@ -643,10 +664,10 @@ The database never touches disk and vanishes when the process exits. No state, n ## Limitations -- **File format auto-detection** is based on file extension. Files without a recognized extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.xml`) default to CSV. Use `-I` to override. +- **File format auto-detection** is based on file extension. Files without a recognized extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.yaml`, `.yml`, `.xml`) default to CSV. Use `-I` to override. ## Related - **[q](https://harelba.github.io/q/)** — Python-based SQL on tabular data. Similar concept, but requires Python runtime. Better if you're already in a Python environment or need Python-specific integrations. -- **[trdsql](https://github.com/noborus/trdsql)** — Go alternative with broader format support (LTSV, TBLN) and more output options. Better if you need formats beyond CSV/JSON/NDJSON/XML or want more output formatting choices. +- **[trdsql](https://github.com/noborus/trdsql)** — Go alternative with broader format support (LTSV, TBLN) and more output options. Better if you need formats beyond CSV/JSON/NDJSON/YAML/XML or want more output formatting choices. - **[sqlite-utils](https://sqlite-utils.datasette.io/)** — better if you need persistent databases, schema management, or Python scripting. sql-pipe is designed for one-shot queries on ephemeral in-memory data. diff --git a/build.zig b/build.zig index 61f9aa9..78737e8 100644 --- a/build.zig +++ b/build.zig @@ -44,6 +44,16 @@ pub fn build(b: *std.Build) void { }); exe.root_module.addImport("c", translate_c.createModule()); + // Translate yaml.h to Zig declarations, exposed as @import("yaml"). + // libyaml is always bundled (no --bundle-yaml option) — it's tiny and + // not always available as a system library. + const translate_yaml = b.addTranslateC(.{ + .root_source_file = b.path("lib/yaml/yaml_translate.h"), + .target = target, + .optimize = optimize, + }); + exe.root_module.addImport("yaml", translate_yaml.createModule()); + if (bundle_sqlite) { exe.root_module.addIncludePath(b.path("lib")); exe.root_module.addCSourceFile(.{ @@ -54,6 +64,22 @@ pub fn build(b: *std.Build) void { exe.root_module.linkSystemLibrary("sqlite3", .{}); } + // Bundle libyaml — only used for loading, so compile api.c, parser.c, + // scanner.c, reader.c, writer.c, loader.c. Skip dumper.c, emitter.c + // (we don't emit YAML). + exe.root_module.addIncludePath(b.path("lib/yaml")); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"), .flags = &.{ + "-DYAML_VERSION_MAJOR=0", + "-DYAML_VERSION_MINOR=2", + "-DYAML_VERSION_PATCH=5", + "-DYAML_VERSION_STRING=\"0.2.5\"", + } }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"), .flags = &.{} }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"), .flags = &.{} }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); + b.installArtifact(exe); // Generate man page from scdoc source if scdoc (and gzip) are available (optional dependencies) @@ -2643,6 +2669,77 @@ pub fn build(b: *std.Build) void { }); fixture_test_step.dependOn(&fixture_sample_file.step); + // Fixture test 168a: YAML file argument — auto-detected from .yaml extension + // Note: YAML preserves original number formatting (e.g., 25.00 stays 25.00) + const fixture_yaml_file = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yaml 'SELECT name, price FROM users ORDER BY CAST(price AS REAL) DESC') + \\expected=$(printf 'Thingamajig,60.00\nDoohickey,50.00\nGadget,40.25\nWidget,25.00') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_file.step); + + // Fixture test 168b: YAML file argument — .yml extension alias + const fixture_yaml_yml_alias = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yml 'SELECT name FROM users WHERE category = "hardware" ORDER BY name') + \\expected=$(printf 'Doohickey\nWidget') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_yml_alias.step); + + // Fixture test 168c: YAML file via stdin with -I yaml + const fixture_yaml_stdin = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(cat tests/fixtures/users.yaml | ./zig-out/bin/sql-pipe -I yaml 'SELECT name FROM t WHERE CAST(stock AS INTEGER) > 0 ORDER BY name') + \\expected=$(printf 'Doohickey\nGadget\nWidget') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_stdin.step); + + // Fixture test 168d: YAML file — filter and aggregate + const fixture_yaml_aggregate = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yaml 'SELECT category, COUNT(*) FROM users GROUP BY category ORDER BY category') + \\expected=$(printf 'electronics,2\nhardware,2') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_aggregate.step); + + // Fixture test 168e: YAML comments are transparent + const fixture_yaml_comments = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(printf -- '# This is a comment\n# Another comment\n- name: Alice\n age: 30\n# Mid comment\n- name: Bob\n age: 25\n' \ + \\ | ./zig-out/bin/sql-pipe -I yaml 'SELECT name FROM t ORDER BY name') + \\expected=$(printf 'Alice\nBob') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_comments.step); + + // Fixture test 168f: Malformed YAML — error exit 2 with line number + const fixture_yaml_malformed = b.addSystemCommand(&.{ + "bash", "-c", + \\msg=$(printf -- '- name: Alice\n\tbad: indent\n' | ./zig-out/bin/sql-pipe -I yaml 'SELECT 1' 2>&1 >/dev/null; echo "EXIT:$?") + \\echo "$msg" | grep -q 'YAML parse error at line' && echo "$msg" | grep -q 'EXIT:2' + }); + fixture_test_step.dependOn(&fixture_yaml_malformed.step); + + // Fixture test 168g: Empty YAML input — graceful (returns 0, no crash) + const fixture_yaml_empty = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(./zig-out/bin/sql-pipe -I yaml 'SELECT 1 AS one' < /dev/null 2>/dev/null) + \\[ "$result" = "1" ] + }); + fixture_test_step.dependOn(&fixture_yaml_empty.step); + + // Fixture test 168h: Multi-document YAML stream — error exit 2 + const fixture_yaml_multidoc = b.addSystemCommand(&.{ + "bash", "-c", + \\msg=$(printf -- '- name: Alice\n---\n- name: Bob\n' | ./zig-out/bin/sql-pipe -I yaml 'SELECT 1' 2>&1 >/dev/null; echo "EXIT:$?") + \\echo "$msg" | grep -q 'multiple documents not supported' && echo "$msg" | grep -q 'EXIT:2' + }); + fixture_test_step.dependOn(&fixture_yaml_multidoc.step); + const unit_tests = b.addTest(.{ .root_module = b.createModule(.{ .root_source_file = b.path("src/csv.zig"), @@ -2664,6 +2761,19 @@ pub fn build(b: *std.Build) void { }), }); xml_unit_tests.root_module.addImport("c", translate_c.createModule()); + xml_unit_tests.root_module.addImport("yaml", translate_yaml.createModule()); + xml_unit_tests.root_module.addIncludePath(b.path("lib/yaml")); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"), .flags = &.{ + "-DYAML_VERSION_MAJOR=0", + "-DYAML_VERSION_MINOR=2", + "-DYAML_VERSION_PATCH=5", + "-DYAML_VERSION_STRING=\"0.2.5\"", + } }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"), .flags = &.{} }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"), .flags = &.{} }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); if (bundle_sqlite) { xml_unit_tests.root_module.addIncludePath(b.path("lib")); xml_unit_tests.root_module.addCSourceFile(.{ @@ -2687,6 +2797,19 @@ pub fn build(b: *std.Build) void { }), }); loader_unit_tests.root_module.addImport("c", translate_c.createModule()); + loader_unit_tests.root_module.addImport("yaml", translate_yaml.createModule()); + loader_unit_tests.root_module.addIncludePath(b.path("lib/yaml")); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"), .flags = &.{ + "-DYAML_VERSION_MAJOR=0", + "-DYAML_VERSION_MINOR=2", + "-DYAML_VERSION_PATCH=5", + "-DYAML_VERSION_STRING=\"0.2.5\"", + } }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"), .flags = &.{} }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"), .flags = &.{} }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); if (bundle_sqlite) { loader_unit_tests.root_module.addIncludePath(b.path("lib")); loader_unit_tests.root_module.addCSourceFile(.{ diff --git a/docs/sql-pipe.1.scd b/docs/sql-pipe.1.scd index 8c32f3b..7274ba0 100644 --- a/docs/sql-pipe.1.scd +++ b/docs/sql-pipe.1.scd @@ -1,7 +1,7 @@ sql-pipe(1) NAME - sql-pipe - Execute SQL queries on CSV/JSON/XML data from stdin or files + sql-pipe - Execute SQL queries on CSV/JSON/NDJSON/YAML/XML data from stdin or files SYNOPSIS *sql-pipe* [OPTIONS] [...] @@ -16,10 +16,10 @@ SYNOPSIS *sql-pipe* --completions DESCRIPTION - sql-pipe reads CSV, JSON, NDJSON, or XML data from standard input and/or - file arguments, loads it into an in-memory SQLite database, executes the - provided SQL query, and prints the results as CSV (or JSON/XML when - requested) to standard output. + sql-pipe reads CSV, JSON, NDJSON, YAML, or XML data from standard input + and/or file arguments, loads it into an in-memory SQLite database, + executes the provided SQL query, and prints the results as CSV (or + JSON/XML when requested) to standard output. When reading from stdin, data is loaded into a table named *t*. When files are passed as positional arguments, each file becomes a table named after @@ -27,9 +27,9 @@ DESCRIPTION Stdin and file arguments can be combined — stdin is always table *t*. Input format for files is auto-detected from the file extension (*.csv*, - *.tsv*, *.json*, *.ndjson*, *.xml*). Unrecognized or missing extensions - default to CSV. Use *-I* to override auto-detection (e.g., when a - TSV file has a *.txt* extension). + *.tsv*, *.json*, *.ndjson*, *.yaml*, *.yml*, *.xml*). Unrecognized or + missing extensions default to CSV. Use *-I* to override auto-detection + (e.g., when a TSV file has a *.txt* extension). This tool is useful for quick data transformations, filtering, grouping, aggregations, and multi-file joins without manual SQL database setup. @@ -61,10 +61,11 @@ OPTIONS *-I, --input-format* Set the input format explicitly: *csv* (default), *tsv*, *json*, - *ndjson*, or *xml*. When set, overrides file extension auto-detection - for all file arguments. Stdin always uses this value (no filename - to inspect). Useful when a file has an ambiguous extension (*.txt*, - *.dat*) or no extension at all. + *ndjson*, *yaml*, or *xml*. When set, overrides file extension + auto-detection for all file arguments. Stdin always uses this value + (no filename to inspect). Useful when a file has an ambiguous + extension (*.txt*, *.dat*) or no extension at all. Both *.yaml* and + *.yml* extensions auto-detect to *yaml*. *-O, --output-format* Set the output format: *csv* (default), *tsv*, *json*, *ndjson*, @@ -124,14 +125,14 @@ OPTIONS *OK: rows, columns ( , ...)* and exits 0. On parse error, prints the error message and exits 2. Compatible with *--delimiter*, *--tsv*, *--no-type-inference*, and - *-I* / *--input-format* (csv, tsv, json, ndjson, xml). JSON, NDJSON, - and XML columns are reported as TEXT. Mutually exclusive with a query - argument. + *-I* / *--input-format* (csv, tsv, json, ndjson, yaml, xml). JSON, + NDJSON, YAML, and XML columns are reported as TEXT. Mutually + exclusive with a query argument. *--columns* Read the input header, print each column name on its own line to standard output, and exit with code 0. Supported for CSV, TSV, - JSON, NDJSON, and XML input. When combined with *-v* / *--verbose*, + JSON, NDJSON, YAML, and XML input. When combined with *-v* / *--verbose*, also shows the inferred type (INTEGER, REAL, DATE, DATETIME, or TEXT) for each column (CSV/TSV only; other formats always show TEXT), using the first 100 data rows for inference. Respects *--delimiter* and *--tsv*. @@ -154,8 +155,8 @@ OPTIONS non-null count, min, max, mean) as a formatted table. Mean is blank for non-numeric columns. Compatible with *--delimiter*, *--tsv*, *--no-type-inference*, and *-I* / *--input-format* (csv, tsv, json, - ndjson, xml). Mutually exclusive with a query argument, *--columns*, - *--validate*, *--sample*, and *--output*. + ndjson, yaml, xml). Mutually exclusive with a query argument, + *--columns*, *--validate*, *--sample*, and *--output*. Alias: *--profile*. *--schema* @@ -163,7 +164,7 @@ OPTIONS output. Each input file generates one DDL block; stdin uses table name *t*. No query is required. Compatible with *--delimiter*, *--tsv*, *--no-type-inference*, and *-I* / *--input-format* (csv, - tsv, json, ndjson, xml). Mutually exclusive with a query argument, + tsv, json, ndjson, yaml, xml). Mutually exclusive with a query argument, *--columns*, *--validate*, *--sample*, *--stats*, *--explain*, and *--output*. diff --git a/lib/yaml/yaml_translate.h b/lib/yaml/yaml_translate.h new file mode 100644 index 0000000..dca0d68 --- /dev/null +++ b/lib/yaml/yaml_translate.h @@ -0,0 +1,22 @@ +/* + * Wrapper header for Zig translateC that avoids the MSVC secure CRT trap. + * + * yaml.h includes which on MSVC pulls in declaring + * wcscat_s/wcscpy_s. Zig 0.16's translateC generates bindings for all + * declarations, but libyaml never calls these, causing "unused local + * constant" errors. + * + * Pre-define _INC_STRING to skip (we provide size_t via ). + * The C compilation step uses the real yaml.h with string.h intact. + */ + +#ifndef YAML_TRANSLATE_H +#define YAML_TRANSLATE_H + +#include + +#define _INC_STRING 1 + +#include "yaml.h" + +#endif /* YAML_TRANSLATE_H */ \ No newline at end of file diff --git a/src/args.zig b/src/args.zig index 12005cb..3cea52a 100644 --- a/src/args.zig +++ b/src/args.zig @@ -254,7 +254,7 @@ pub fn printUsage(writer: *std.Io.Writer) !void { \\Options: \\ -d, --delimiter Input field delimiter for CSV: 1–8 chars (default: ,) \\ --tsv Alias for --delimiter '\t' - \\ -I, --input-format Input format: csv (default), tsv, json, ndjson, xml + \\ -I, --input-format Input format: csv (default), tsv, json, ndjson, xml, yaml \\ Overrides file extension auto-detection; stdin always uses this value \\ -O, --output-format Output format: csv (default), tsv, json, ndjson, xml, markdown (alias: md), html, sql \\ --json Alias for --output-format json diff --git a/src/format.zig b/src/format.zig index fa3b839..72f2614 100644 --- a/src/format.zig +++ b/src/format.zig @@ -22,6 +22,7 @@ pub const InputFormat = enum { json, ndjson, xml, + yaml, /// Parse a format name string. /// Returns error.InvalidInputFormat when the value is unrecognised. @@ -35,6 +36,7 @@ pub const InputFormat = enum { const ext = std.fs.path.extension(filename); if (ext.len == 0) return null; const ext_no_dot = ext[1..]; // skip the leading '.' + if (std.mem.eql(u8, ext_no_dot, "yml")) return .yaml; return std.meta.stringToEnum(InputFormat, ext_no_dot); } }; diff --git a/src/main.zig b/src/main.zig index 1b1d5c0..7bba676 100644 --- a/src/main.zig +++ b/src/main.zig @@ -9,6 +9,7 @@ const build_options = @import("build_options"); const args_mod = @import("args.zig"); const sqlite_mod = @import("sqlite.zig"); const loader = @import("loader.zig"); +const yaml_mod = @import("yaml.zig"); const columns_mode = @import("modes/columns.zig"); const validate_mode = @import("modes/validate.zig"); @@ -122,6 +123,7 @@ fn loadInput( .json => json.loadJsonArray(allocator, reader, db, table_name, parsed.max_rows, parsed.json_path, stderr_writer), .ndjson => json.loadNdjsonInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), .xml => xml.loadXmlInput(allocator, reader, db, table_name, parsed.xml_root_input, parsed.xml_row_input, parsed.max_rows, stderr_writer), + .yaml => yaml_mod.loadYamlInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), }; } @@ -273,7 +275,7 @@ pub fn main(init: std.process.Init.Minimal) void { error.IncompatibleFlags => fatal("--header cannot be combined with non-CSV/TSV/HTML output format", stderr_writer, .usage, .{}), error.SilentVerboseConflict => fatal("--silent cannot be combined with --verbose", stderr_writer, .usage, .{}), error.InvalidMaxRows => fatal("--max-rows must be a positive integer", stderr_writer, .usage, .{}), - error.InvalidInputFormat => fatal("unknown input format; supported: csv, tsv, json, ndjson, xml", stderr_writer, .usage, .{}), + error.InvalidInputFormat => fatal("unknown input format; supported: csv, tsv, json, ndjson, xml, yaml", stderr_writer, .usage, .{}), error.InvalidOutputFormat => fatal("unknown output format; supported: csv, tsv, json, ndjson, xml, markdown (md), html, sql", stderr_writer, .usage, .{}), error.ColumnsWithQuery => fatal("--columns cannot be combined with a query argument", stderr_writer, .usage, .{}), error.ValidateWithQuery => fatal("--validate cannot be combined with a query argument", stderr_writer, .usage, .{}), diff --git a/src/modes/columns.zig b/src/modes/columns.zig index efe98fc..55776d4 100644 --- a/src/modes/columns.zig +++ b/src/modes/columns.zig @@ -1,7 +1,10 @@ const std = @import("std"); +const c = @import("c"); const csv_mod = @import("../csv.zig"); const json_mod = @import("../json.zig"); const xml_mod = @import("../xml.zig"); +const yaml_mod = @import("../yaml.zig"); +const sqlite_mod = @import("../sqlite.zig"); const loader = @import("../loader.zig"); const args_mod = @import("../args.zig"); @@ -169,6 +172,31 @@ pub fn runColumns( break; } }, + .yaml => { + var yaml_buf: [4096]u8 = undefined; + const opened = source.openInput(io, input_source, stderr_writer); + defer opened.deinit(io); + var yaml_reader = std.Io.File.reader(opened.file, io, &yaml_buf); + const yaml_db = sqlite_mod.openDb(false, stderr_writer); + defer _ = c.sqlite3_close(yaml_db); + _ = yaml_mod.loadYamlInput(allocator, &yaml_reader.interface, yaml_db, "t", null, stderr_writer); + const cols = sqlite_mod.getTableColumns(allocator, yaml_db, "t", stderr_writer); + defer { + for (cols) |col| allocator.free(col); + allocator.free(cols); + } + for (cols) |col| { + if (args.verbose) { + stdout_writer.print("{s} TEXT\n", .{col}) catch |err| { + std.log.err("failed to write output: {}", .{err}); + }; + } else { + stdout_writer.print("{s}\n", .{col}) catch |err| { + std.log.err("failed to write output: {}", .{err}); + }; + } + } + }, .xml => { var read_buf: [4096]u8 = undefined; const opened = source.openInput(io, input_source, stderr_writer); diff --git a/src/modes/sample.zig b/src/modes/sample.zig index 0915ab4..c69a905 100644 --- a/src/modes/sample.zig +++ b/src/modes/sample.zig @@ -28,7 +28,7 @@ pub fn runSample( .stdin; switch (args.input_format) { - .json, .ndjson, .xml => fatal( + .json, .ndjson, .xml, .yaml => fatal( "--sample is only supported with CSV and TSV input", stderr_writer, .usage, diff --git a/src/modes/schema.zig b/src/modes/schema.zig index 2f226f8..6ac656e 100644 --- a/src/modes/schema.zig +++ b/src/modes/schema.zig @@ -2,6 +2,7 @@ const std = @import("std"); const c = @import("c"); const json_mod = @import("../json.zig"); const xml_mod = @import("../xml.zig"); +const yaml_mod = @import("../yaml.zig"); const sqlite_mod = @import("../sqlite.zig"); const loader = @import("../loader.zig"); const format = @import("../format.zig"); @@ -86,6 +87,7 @@ fn loadTable( .json => json_mod.loadJsonArray(allocator, reader, db, table_name, parsed.max_rows, parsed.json_path, stderr_writer), .ndjson => json_mod.loadNdjsonInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), .xml => xml_mod.loadXmlInput(allocator, reader, db, table_name, parsed.xml_root_input, parsed.xml_row_input, parsed.max_rows, stderr_writer), + .yaml => yaml_mod.loadYamlInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), }; if (rows == 0) fatal("empty input", stderr_writer, .csv_error, .{}); } diff --git a/src/modes/stats.zig b/src/modes/stats.zig index 4ed23a2..e713a50 100644 --- a/src/modes/stats.zig +++ b/src/modes/stats.zig @@ -2,6 +2,7 @@ const std = @import("std"); const c = @import("c"); const json_mod = @import("../json.zig"); const xml_mod = @import("../xml.zig"); +const yaml_mod = @import("../yaml.zig"); const sqlite_mod = @import("../sqlite.zig"); const loader = @import("../loader.zig"); const format = @import("../format.zig"); @@ -87,6 +88,7 @@ fn loadTable( .json => json_mod.loadJsonArray(allocator, reader, db, table_name, parsed.max_rows, parsed.json_path, stderr_writer), .ndjson => json_mod.loadNdjsonInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), .xml => xml_mod.loadXmlInput(allocator, reader, db, table_name, parsed.xml_root_input, parsed.xml_row_input, parsed.max_rows, stderr_writer), + .yaml => yaml_mod.loadYamlInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), }; if (rows == 0) fatal("empty input", stderr_writer, .csv_error, .{}); } diff --git a/src/modes/validate.zig b/src/modes/validate.zig index ed2e569..04382b1 100644 --- a/src/modes/validate.zig +++ b/src/modes/validate.zig @@ -1,7 +1,9 @@ const std = @import("std"); +const c = @import("c"); const csv_mod = @import("../csv.zig"); const json_mod = @import("../json.zig"); const xml_mod = @import("../xml.zig"); +const yaml_mod = @import("../yaml.zig"); const sqlite_mod = @import("../sqlite.zig"); const loader = @import("../loader.zig"); const args_mod = @import("../args.zig"); @@ -277,6 +279,40 @@ pub fn runValidate( std.process.exit(@intFromEnum(ExitCode.usage)); }; }, + .yaml => { + var yaml_buf: [4096]u8 = undefined; + const opened = source.openInput(io, input_source, stderr_writer); + defer opened.deinit(io); + var yaml_reader = std.Io.File.reader(opened.file, io, &yaml_buf); + const yaml_db = sqlite_mod.openDb(false, stderr_writer); + defer _ = c.sqlite3_close(yaml_db); + const count = yaml_mod.loadYamlInput(allocator, &yaml_reader.interface, yaml_db, "t", null, stderr_writer); + const cols = sqlite_mod.getTableColumns(allocator, yaml_db, "t", stderr_writer); + defer { + for (cols) |col| allocator.free(col); + allocator.free(cols); + } + var count_buf: [32]u8 = undefined; + const count_str = fmtThousands(&count_buf, count); + stdout_writer.print("OK: {s} rows, {d} columns (", .{ count_str, cols.len }) catch |err| { + std.log.err("failed to write output: {}", .{err}); + std.process.exit(@intFromEnum(ExitCode.usage)); + }; + for (cols, 0..) |col, i| { + if (i > 0) stdout_writer.writeAll(", ") catch |err| { + std.log.err("failed to write output: {}", .{err}); + std.process.exit(@intFromEnum(ExitCode.usage)); + }; + stdout_writer.print("{s} TEXT", .{col}) catch |err| { + std.log.err("failed to write output: {}", .{err}); + std.process.exit(@intFromEnum(ExitCode.usage)); + }; + } + stdout_writer.writeAll(")\n") catch |err| { + std.log.err("failed to write output: {}", .{err}); + std.process.exit(@intFromEnum(ExitCode.usage)); + }; + }, .xml => { var read_buf: [4096]u8 = undefined; const opened = source.openInput(io, input_source, stderr_writer); diff --git a/src/yaml.zig b/src/yaml.zig new file mode 100644 index 0000000..8874e52 --- /dev/null +++ b/src/yaml.zig @@ -0,0 +1,220 @@ +//! YAML input loading — reads YAML sequence-of-mappings as rows. +//! +//! loadYamlInput +//! Read all of `reader` as a YAML document, create table `table_name` in `db`, +//! and insert every mapping as a row. Expects a top-level YAML sequence of +//! mappings (list of objects). All columns are TEXT (no type inference). +//! Comments and flow-style mappings are handled by libyaml transparently. +//! Multi-document streams, anchors/aliases, non-string keys, and nested +//! sequences/mappings are rejected with a fatal error. + +const std = @import("std"); +const c = @import("c"); +const yaml = @import("yaml"); + +const sqlite_helpers = @import("sqlite.zig"); + +const createAllTextTable = sqlite_helpers.createAllTextTable; +const prepareInsertStmt = sqlite_helpers.prepareInsertStmt; +const beginTransaction = sqlite_helpers.beginTransaction; +const commitTransaction = sqlite_helpers.commitTransaction; +const fatal = sqlite_helpers.fatal; +const ExitCode = sqlite_helpers.ExitCode; +const sqlite_static = sqlite_helpers.sqlite_static; + +/// loadYamlInput(allocator, reader, db, table_name, max_rows, stderr_writer) → usize +/// +/// Pre: reader is positioned at the start of a YAML document (single stream) +/// db is an open, empty SQLite database +/// Post: table is created with TEXT columns from the first mapping's keys; +/// all mappings are inserted as rows +/// result = number of rows inserted +/// aborts the process on any parse, I/O, or SQL error +pub fn loadYamlInput( + allocator: std.mem.Allocator, + reader: *std.Io.Reader, + db: *c.sqlite3, + table_name: []const u8, + max_rows: ?usize, + stderr_writer: *std.Io.Writer, +) usize { + // Read all input into a buffer + const buf = sqlite_helpers.readAllInput(allocator, reader, stderr_writer, "YAML input"); + defer allocator.free(buf); + + if (buf.len == 0) return 0; // Empty input - return 0 rows gracefully + + // Initialize libyaml parser + var parser: yaml.yaml_parser_t = undefined; + if (yaml.yaml_parser_initialize(&parser) != 1) + fatal("out of memory initializing YAML parser", stderr_writer, .csv_error, .{}); + defer yaml.yaml_parser_delete(&parser); + yaml.yaml_parser_set_input_string(&parser, @as([*c]const u8, @ptrCast(buf.ptr)), buf.len); + + // State variables + var cols: std.ArrayList([]const u8) = .empty; // column names (first mapping keys, owned) + var first_row_vals: std.ArrayList([]const u8) = .empty; // first row's values (owned) + var current_keys: std.ArrayList([]const u8) = .empty; // keys for current mapping (owned) + var current_vals: std.ArrayList([]const u8) = .empty; // values for current mapping (owned) + var current_key: ?[]const u8 = null; // temp key storage (owned when non-null) + var rows_inserted: usize = 0; + var got_first_mapping = false; + var in_transaction = false; + var stmt: ?*c.sqlite3_stmt = null; + var mapping_depth: usize = 0; + var outer_sequence_ended = false; // for multi-doc detection + var in_outer_sequence = false; + + defer { + for (cols.items) |c_name| allocator.free(c_name); + cols.deinit(allocator); + for (first_row_vals.items) |v| allocator.free(v); + first_row_vals.deinit(allocator); + for (current_keys.items) |k| allocator.free(k); + current_keys.deinit(allocator); + for (current_vals.items) |v| allocator.free(v); + current_vals.deinit(allocator); + if (current_key) |k| allocator.free(k); + if (stmt) |s| _ = c.sqlite3_finalize(s); + } + + while (true) { + var event: yaml.yaml_event_t = undefined; + if (yaml.yaml_parser_parse(&parser, &event) != 1) { + const mark = parser.problem_mark; + const problem = if (parser.problem) |p| std.mem.span(p) else "(unknown)"; + fatal("YAML parse error at line {d}, col {d}: {s}", stderr_writer, .csv_error, .{ + mark.line + 1, mark.column + 1, problem, + }); + } + defer yaml.yaml_event_delete(&event); + + switch (@as(c_int, @intCast(event.type))) { + yaml.YAML_STREAM_START_EVENT => {}, + yaml.YAML_STREAM_END_EVENT => break, + + yaml.YAML_DOCUMENT_START_EVENT => { + if (outer_sequence_ended) + fatal("YAML input: multiple documents not supported (use a single document with one top-level sequence)", stderr_writer, .csv_error, .{}); + }, + yaml.YAML_DOCUMENT_END_EVENT => {}, + + yaml.YAML_SEQUENCE_START_EVENT => { + if (mapping_depth > 0) { + // Nested sequence inside a mapping (as key or value) + if (current_key == null) + fatal("YAML input: mapping keys must be scalar strings", stderr_writer, .csv_error, .{}) + else + fatal("YAML input: mapping values must be scalar strings (nested sequences not supported)", stderr_writer, .csv_error, .{}); + } + in_outer_sequence = true; + }, + yaml.YAML_SEQUENCE_END_EVENT => { + outer_sequence_ended = true; + }, + + yaml.YAML_MAPPING_START_EVENT => { + mapping_depth += 1; + if (mapping_depth == 1 and !in_outer_sequence) + fatal("YAML input must be a top-level sequence of mappings", stderr_writer, .csv_error, .{}); + if (mapping_depth > 1) + fatal("YAML input: nested mappings are not supported", stderr_writer, .csv_error, .{}); + current_key = null; + }, + + yaml.YAML_MAPPING_END_EVENT => { + mapping_depth -= 1; + + if (!got_first_mapping) { + // First mapping: keys define the schema + if (current_keys.items.len == 0) + fatal("first YAML mapping has no keys", stderr_writer, .csv_error, .{}); + + // Take ownership of current_keys as cols + cols = current_keys; + current_keys = .empty; + first_row_vals = current_vals; + current_vals = .empty; + + createAllTextTable(allocator, db, table_name, cols.items, stderr_writer); + beginTransaction(db, stderr_writer); + in_transaction = true; + stmt = prepareInsertStmt(allocator, db, table_name, cols.items.len, stderr_writer); + + // Insert first row + _ = c.sqlite3_reset(stmt.?); + _ = c.sqlite3_clear_bindings(stmt.?); + for (first_row_vals.items, 0..) |v, j| { + if (c.sqlite3_bind_text(stmt.?, @intCast(j + 1), v.ptr, @intCast(v.len), sqlite_static) != c.SQLITE_OK) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + } + if (c.sqlite3_step(stmt.?) != c.SQLITE_DONE) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + rows_inserted = 1; + sqlite_helpers.checkMaxRows(rows_inserted, max_rows, stderr_writer); + got_first_mapping = true; + } else { + // Subsequent mappings: bind values by key lookup + _ = c.sqlite3_reset(stmt.?); + _ = c.sqlite3_clear_bindings(stmt.?); + + for (cols.items, 0..) |col_name, j| { + const param_idx: c_int = @intCast(j + 1); + var found = false; + for (current_keys.items, current_vals.items) |k, v| { + if (std.mem.eql(u8, k, col_name)) { + if (c.sqlite3_bind_text(stmt.?, param_idx, v.ptr, @intCast(v.len), sqlite_static) != c.SQLITE_OK) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + found = true; + break; + } + } + if (!found) { + if (c.sqlite3_bind_null(stmt.?, param_idx) != c.SQLITE_OK) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + } + } + if (c.sqlite3_step(stmt.?) != c.SQLITE_DONE) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + rows_inserted += 1; + sqlite_helpers.checkMaxRows(rows_inserted, max_rows, stderr_writer); + } + + // Free current mapping keys and values (except when taken above) + for (current_keys.items) |k| allocator.free(k); + for (current_vals.items) |v| allocator.free(v); + current_keys.items.len = 0; + current_vals.items.len = 0; + }, + + yaml.YAML_SCALAR_EVENT => { + const value_slice: []const u8 = @as([*]const u8, @ptrCast(event.data.scalar.value))[0..event.data.scalar.length]; + if (current_key == null) { + // This is a key + current_key = allocator.dupe(u8, value_slice) catch + fatal("out of memory", stderr_writer, .csv_error, .{}); + } else { + // This is a value — pair it with the stored key + const key = current_key.?; + current_key = null; + const val_dup = allocator.dupe(u8, value_slice) catch + fatal("out of memory", stderr_writer, .csv_error, .{}); + current_keys.append(allocator, key) catch + fatal("out of memory", stderr_writer, .csv_error, .{}); + current_vals.append(allocator, val_dup) catch + fatal("out of memory", stderr_writer, .csv_error, .{}); + } + }, + + yaml.YAML_ALIAS_EVENT => fatal("YAML input: anchors and aliases are not supported", stderr_writer, .csv_error, .{}), + yaml.YAML_NO_EVENT => unreachable, + else => {}, + } + } + + // No rows loaded at all — return 0 (graceful) + if (cols.items.len == 0) return 0; + + if (in_transaction) commitTransaction(db, stderr_writer); + return rows_inserted; +} diff --git a/tests/fixtures/users.yaml b/tests/fixtures/users.yaml new file mode 100644 index 0000000..ee9890a --- /dev/null +++ b/tests/fixtures/users.yaml @@ -0,0 +1,17 @@ +# Sample products fixture mirroring tests/fixtures/products.json +- name: Widget + price: 25.00 + category: hardware + stock: 150 +- name: Gadget + price: 40.25 + category: electronics + stock: 75 +- name: Doohickey + price: 50.00 + category: hardware + stock: 30 +- name: Thingamajig + price: 60.00 + category: electronics + stock: 0 diff --git a/tests/fixtures/users.yml b/tests/fixtures/users.yml new file mode 100644 index 0000000..ee9890a --- /dev/null +++ b/tests/fixtures/users.yml @@ -0,0 +1,17 @@ +# Sample products fixture mirroring tests/fixtures/products.json +- name: Widget + price: 25.00 + category: hardware + stock: 150 +- name: Gadget + price: 40.25 + category: electronics + stock: 75 +- name: Doohickey + price: 50.00 + category: hardware + stock: 30 +- name: Thingamajig + price: 60.00 + category: electronics + stock: 0