From 864c035b94c0798a9fc13007b6b73db2dfc6ed2c Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 14:48:36 +0200 Subject: [PATCH 01/10] feat: add YAML input support (-I yaml) with bundled libyaml 0.2.5 --- .github/workflows/ci.yml | 16 +++ .github/workflows/release.yml | 15 +++ build.zig | 123 +++++++++++++++++++ src/args.zig | 2 +- src/format.zig | 2 + src/main.zig | 4 +- src/modes/columns.zig | 28 +++++ src/modes/sample.zig | 2 +- src/modes/schema.zig | 2 + src/modes/stats.zig | 2 + src/modes/validate.zig | 36 ++++++ src/yaml.zig | 220 ++++++++++++++++++++++++++++++++++ tests/fixtures/users.yaml | 17 +++ tests/fixtures/users.yml | 17 +++ 14 files changed, 483 insertions(+), 3 deletions(-) create mode 100644 src/yaml.zig create mode 100644 tests/fixtures/users.yaml create mode 100644 tests/fixtures/users.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3277419..cfa89e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,22 @@ jobs: print('SQLite amalgamation ready') " + - name: Download libyaml amalgamation + shell: bash + run: | + mkdir -p lib/yaml + curl -fsSL "https://github.com/yaml/libyaml/archive/refs/tags/0.2.5.tar.gz" -o /tmp/yaml-0.2.5.tar.gz + tar -xzf /tmp/yaml-0.2.5.tar.gz -C /tmp + cp /tmp/libyaml-0.2.5/include/yaml.h lib/yaml/yaml.h + cp /tmp/libyaml-0.2.5/src/yaml_private.h lib/yaml/yaml_private.h + cp /tmp/libyaml-0.2.5/src/api.c lib/yaml/api.c + cp /tmp/libyaml-0.2.5/src/parser.c lib/yaml/parser.c + cp /tmp/libyaml-0.2.5/src/scanner.c lib/yaml/scanner.c + cp /tmp/libyaml-0.2.5/src/reader.c lib/yaml/reader.c + cp /tmp/libyaml-0.2.5/src/writer.c lib/yaml/writer.c + cp /tmp/libyaml-0.2.5/src/loader.c lib/yaml/loader.c + echo "libyaml 0.2.5 ready" + - name: Build run: zig build -Dbundle-sqlite=true -Doptimize=ReleaseSafe diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e81f68..34f9b6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,6 +86,21 @@ jobs: unzip -j sqlite.zip "*/sqlite3.c" "*/sqlite3.h" -d lib/ echo "SQLite $(grep -m1 SQLITE_VERSION lib/sqlite3.h | awk '{print $3}')" + - name: Download libyaml amalgamation + run: | + mkdir -p lib/yaml + curl -fsSL "https://github.com/yaml/libyaml/archive/refs/tags/0.2.5.tar.gz" -o /tmp/yaml-0.2.5.tar.gz + tar -xzf /tmp/yaml-0.2.5.tar.gz -C /tmp + cp /tmp/libyaml-0.2.5/include/yaml.h lib/yaml/yaml.h + cp /tmp/libyaml-0.2.5/src/yaml_private.h lib/yaml/yaml_private.h + cp /tmp/libyaml-0.2.5/src/api.c lib/yaml/api.c + cp /tmp/libyaml-0.2.5/src/parser.c lib/yaml/parser.c + cp /tmp/libyaml-0.2.5/src/scanner.c lib/yaml/scanner.c + cp /tmp/libyaml-0.2.5/src/reader.c lib/yaml/reader.c + cp /tmp/libyaml-0.2.5/src/writer.c lib/yaml/writer.c + cp /tmp/libyaml-0.2.5/src/loader.c lib/yaml/loader.c + echo "libyaml 0.2.5 ready" + - name: Build run: | VERSION="${GITHUB_REF_NAME#v}" diff --git a/build.zig b/build.zig index 61f9aa9..43e71e2 100644 --- a/build.zig +++ b/build.zig @@ -44,6 +44,16 @@ pub fn build(b: *std.Build) void { }); exe.root_module.addImport("c", translate_c.createModule()); + // Translate yaml.h to Zig declarations, exposed as @import("yaml"). + // libyaml is always bundled (no --bundle-yaml option) — it's tiny and + // not always available as a system library. + const translate_yaml = b.addTranslateC(.{ + .root_source_file = b.path("lib/yaml/yaml.h"), + .target = target, + .optimize = optimize, + }); + exe.root_module.addImport("yaml", translate_yaml.createModule()); + if (bundle_sqlite) { exe.root_module.addIncludePath(b.path("lib")); exe.root_module.addCSourceFile(.{ @@ -54,6 +64,22 @@ pub fn build(b: *std.Build) void { exe.root_module.linkSystemLibrary("sqlite3", .{}); } + // Bundle libyaml — only used for loading, so compile api.c, parser.c, + // scanner.c, reader.c, writer.c, loader.c. Skip dumper.c, emitter.c + // (we don't emit YAML). + exe.root_module.addIncludePath(b.path("lib/yaml")); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"), .flags = &.{ + "-DYAML_VERSION_MAJOR=0", + "-DYAML_VERSION_MINOR=2", + "-DYAML_VERSION_PATCH=5", + "-DYAML_VERSION_STRING=\"0.2.5\"", + } }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"), .flags = &.{} }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"), .flags = &.{} }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); + b.installArtifact(exe); // Generate man page from scdoc source if scdoc (and gzip) are available (optional dependencies) @@ -2643,6 +2669,77 @@ pub fn build(b: *std.Build) void { }); fixture_test_step.dependOn(&fixture_sample_file.step); + // Fixture test 168a: YAML file argument — auto-detected from .yaml extension + // Note: YAML preserves original number formatting (e.g., 25.00 stays 25.00) + const fixture_yaml_file = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yaml 'SELECT name, price FROM users ORDER BY CAST(price AS REAL) DESC') + \\expected=$(printf 'Thingamajig,60.00\nDoohickey,50.00\nGadget,40.25\nWidget,25.00') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_file.step); + + // Fixture test 168b: YAML file argument — .yml extension alias + const fixture_yaml_yml_alias = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yml 'SELECT name FROM users WHERE category = "hardware" ORDER BY name') + \\expected=$(printf 'Doohickey\nWidget') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_yml_alias.step); + + // Fixture test 168c: YAML file via stdin with -I yaml + const fixture_yaml_stdin = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(cat tests/fixtures/users.yaml | ./zig-out/bin/sql-pipe -I yaml 'SELECT name FROM t WHERE CAST(stock AS INTEGER) > 0 ORDER BY name') + \\expected=$(printf 'Doohickey\nGadget\nWidget') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_stdin.step); + + // Fixture test 168d: YAML file — filter and aggregate + const fixture_yaml_aggregate = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(./zig-out/bin/sql-pipe tests/fixtures/users.yaml 'SELECT category, COUNT(*) FROM users GROUP BY category ORDER BY category') + \\expected=$(printf 'electronics,2\nhardware,2') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_aggregate.step); + + // Fixture test 168e: YAML comments are transparent + const fixture_yaml_comments = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(printf -- '# This is a comment\n# Another comment\n- name: Alice\n age: 30\n# Mid comment\n- name: Bob\n age: 25\n' \ + \\ | ./zig-out/bin/sql-pipe -I yaml 'SELECT name FROM t ORDER BY name') + \\expected=$(printf 'Alice\nBob') + \\[ "$result" = "$expected" ] + }); + fixture_test_step.dependOn(&fixture_yaml_comments.step); + + // Fixture test 168f: Malformed YAML — error exit 2 with line number + const fixture_yaml_malformed = b.addSystemCommand(&.{ + "bash", "-c", + \\msg=$(printf -- '- name: Alice\n\tbad: indent\n' | ./zig-out/bin/sql-pipe -I yaml 'SELECT 1' 2>&1 >/dev/null; echo "EXIT:$?") + \\echo "$msg" | grep -q 'YAML parse error at line' && echo "$msg" | grep -q 'EXIT:2' + }); + fixture_test_step.dependOn(&fixture_yaml_malformed.step); + + // Fixture test 168g: Empty YAML input — graceful (returns 0, no crash) + const fixture_yaml_empty = b.addSystemCommand(&.{ + "bash", "-c", + \\result=$(./zig-out/bin/sql-pipe -I yaml 'SELECT 1 AS one' < /dev/null 2>/dev/null) + \\[ "$result" = "1" ] + }); + fixture_test_step.dependOn(&fixture_yaml_empty.step); + + // Fixture test 168h: Multi-document YAML stream — error exit 2 + const fixture_yaml_multidoc = b.addSystemCommand(&.{ + "bash", "-c", + \\msg=$(printf -- '- name: Alice\n---\n- name: Bob\n' | ./zig-out/bin/sql-pipe -I yaml 'SELECT 1' 2>&1 >/dev/null; echo "EXIT:$?") + \\echo "$msg" | grep -q 'multiple documents not supported' && echo "$msg" | grep -q 'EXIT:2' + }); + fixture_test_step.dependOn(&fixture_yaml_multidoc.step); + const unit_tests = b.addTest(.{ .root_module = b.createModule(.{ .root_source_file = b.path("src/csv.zig"), @@ -2664,6 +2761,19 @@ pub fn build(b: *std.Build) void { }), }); xml_unit_tests.root_module.addImport("c", translate_c.createModule()); + xml_unit_tests.root_module.addImport("yaml", translate_yaml.createModule()); + xml_unit_tests.root_module.addIncludePath(b.path("lib/yaml")); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"), .flags = &.{ + "-DYAML_VERSION_MAJOR=0", + "-DYAML_VERSION_MINOR=2", + "-DYAML_VERSION_PATCH=5", + "-DYAML_VERSION_STRING=\"0.2.5\"", + } }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"), .flags = &.{} }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"), .flags = &.{} }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); + xml_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); if (bundle_sqlite) { xml_unit_tests.root_module.addIncludePath(b.path("lib")); xml_unit_tests.root_module.addCSourceFile(.{ @@ -2687,6 +2797,19 @@ pub fn build(b: *std.Build) void { }), }); loader_unit_tests.root_module.addImport("c", translate_c.createModule()); + loader_unit_tests.root_module.addImport("yaml", translate_yaml.createModule()); + loader_unit_tests.root_module.addIncludePath(b.path("lib/yaml")); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/api.c"), .flags = &.{ + "-DYAML_VERSION_MAJOR=0", + "-DYAML_VERSION_MINOR=2", + "-DYAML_VERSION_PATCH=5", + "-DYAML_VERSION_STRING=\"0.2.5\"", + } }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/parser.c"), .flags = &.{} }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/scanner.c"), .flags = &.{} }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"), .flags = &.{} }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); + loader_unit_tests.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); if (bundle_sqlite) { loader_unit_tests.root_module.addIncludePath(b.path("lib")); loader_unit_tests.root_module.addCSourceFile(.{ diff --git a/src/args.zig b/src/args.zig index 12005cb..3cea52a 100644 --- a/src/args.zig +++ b/src/args.zig @@ -254,7 +254,7 @@ pub fn printUsage(writer: *std.Io.Writer) !void { \\Options: \\ -d, --delimiter Input field delimiter for CSV: 1–8 chars (default: ,) \\ --tsv Alias for --delimiter '\t' - \\ -I, --input-format Input format: csv (default), tsv, json, ndjson, xml + \\ -I, --input-format Input format: csv (default), tsv, json, ndjson, xml, yaml \\ Overrides file extension auto-detection; stdin always uses this value \\ -O, --output-format Output format: csv (default), tsv, json, ndjson, xml, markdown (alias: md), html, sql \\ --json Alias for --output-format json diff --git a/src/format.zig b/src/format.zig index fa3b839..72f2614 100644 --- a/src/format.zig +++ b/src/format.zig @@ -22,6 +22,7 @@ pub const InputFormat = enum { json, ndjson, xml, + yaml, /// Parse a format name string. /// Returns error.InvalidInputFormat when the value is unrecognised. @@ -35,6 +36,7 @@ pub const InputFormat = enum { const ext = std.fs.path.extension(filename); if (ext.len == 0) return null; const ext_no_dot = ext[1..]; // skip the leading '.' + if (std.mem.eql(u8, ext_no_dot, "yml")) return .yaml; return std.meta.stringToEnum(InputFormat, ext_no_dot); } }; diff --git a/src/main.zig b/src/main.zig index 1b1d5c0..7bba676 100644 --- a/src/main.zig +++ b/src/main.zig @@ -9,6 +9,7 @@ const build_options = @import("build_options"); const args_mod = @import("args.zig"); const sqlite_mod = @import("sqlite.zig"); const loader = @import("loader.zig"); +const yaml_mod = @import("yaml.zig"); const columns_mode = @import("modes/columns.zig"); const validate_mode = @import("modes/validate.zig"); @@ -122,6 +123,7 @@ fn loadInput( .json => json.loadJsonArray(allocator, reader, db, table_name, parsed.max_rows, parsed.json_path, stderr_writer), .ndjson => json.loadNdjsonInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), .xml => xml.loadXmlInput(allocator, reader, db, table_name, parsed.xml_root_input, parsed.xml_row_input, parsed.max_rows, stderr_writer), + .yaml => yaml_mod.loadYamlInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), }; } @@ -273,7 +275,7 @@ pub fn main(init: std.process.Init.Minimal) void { error.IncompatibleFlags => fatal("--header cannot be combined with non-CSV/TSV/HTML output format", stderr_writer, .usage, .{}), error.SilentVerboseConflict => fatal("--silent cannot be combined with --verbose", stderr_writer, .usage, .{}), error.InvalidMaxRows => fatal("--max-rows must be a positive integer", stderr_writer, .usage, .{}), - error.InvalidInputFormat => fatal("unknown input format; supported: csv, tsv, json, ndjson, xml", stderr_writer, .usage, .{}), + error.InvalidInputFormat => fatal("unknown input format; supported: csv, tsv, json, ndjson, xml, yaml", stderr_writer, .usage, .{}), error.InvalidOutputFormat => fatal("unknown output format; supported: csv, tsv, json, ndjson, xml, markdown (md), html, sql", stderr_writer, .usage, .{}), error.ColumnsWithQuery => fatal("--columns cannot be combined with a query argument", stderr_writer, .usage, .{}), error.ValidateWithQuery => fatal("--validate cannot be combined with a query argument", stderr_writer, .usage, .{}), diff --git a/src/modes/columns.zig b/src/modes/columns.zig index efe98fc..55776d4 100644 --- a/src/modes/columns.zig +++ b/src/modes/columns.zig @@ -1,7 +1,10 @@ const std = @import("std"); +const c = @import("c"); const csv_mod = @import("../csv.zig"); const json_mod = @import("../json.zig"); const xml_mod = @import("../xml.zig"); +const yaml_mod = @import("../yaml.zig"); +const sqlite_mod = @import("../sqlite.zig"); const loader = @import("../loader.zig"); const args_mod = @import("../args.zig"); @@ -169,6 +172,31 @@ pub fn runColumns( break; } }, + .yaml => { + var yaml_buf: [4096]u8 = undefined; + const opened = source.openInput(io, input_source, stderr_writer); + defer opened.deinit(io); + var yaml_reader = std.Io.File.reader(opened.file, io, &yaml_buf); + const yaml_db = sqlite_mod.openDb(false, stderr_writer); + defer _ = c.sqlite3_close(yaml_db); + _ = yaml_mod.loadYamlInput(allocator, &yaml_reader.interface, yaml_db, "t", null, stderr_writer); + const cols = sqlite_mod.getTableColumns(allocator, yaml_db, "t", stderr_writer); + defer { + for (cols) |col| allocator.free(col); + allocator.free(cols); + } + for (cols) |col| { + if (args.verbose) { + stdout_writer.print("{s} TEXT\n", .{col}) catch |err| { + std.log.err("failed to write output: {}", .{err}); + }; + } else { + stdout_writer.print("{s}\n", .{col}) catch |err| { + std.log.err("failed to write output: {}", .{err}); + }; + } + } + }, .xml => { var read_buf: [4096]u8 = undefined; const opened = source.openInput(io, input_source, stderr_writer); diff --git a/src/modes/sample.zig b/src/modes/sample.zig index 0915ab4..c69a905 100644 --- a/src/modes/sample.zig +++ b/src/modes/sample.zig @@ -28,7 +28,7 @@ pub fn runSample( .stdin; switch (args.input_format) { - .json, .ndjson, .xml => fatal( + .json, .ndjson, .xml, .yaml => fatal( "--sample is only supported with CSV and TSV input", stderr_writer, .usage, diff --git a/src/modes/schema.zig b/src/modes/schema.zig index 2f226f8..6ac656e 100644 --- a/src/modes/schema.zig +++ b/src/modes/schema.zig @@ -2,6 +2,7 @@ const std = @import("std"); const c = @import("c"); const json_mod = @import("../json.zig"); const xml_mod = @import("../xml.zig"); +const yaml_mod = @import("../yaml.zig"); const sqlite_mod = @import("../sqlite.zig"); const loader = @import("../loader.zig"); const format = @import("../format.zig"); @@ -86,6 +87,7 @@ fn loadTable( .json => json_mod.loadJsonArray(allocator, reader, db, table_name, parsed.max_rows, parsed.json_path, stderr_writer), .ndjson => json_mod.loadNdjsonInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), .xml => xml_mod.loadXmlInput(allocator, reader, db, table_name, parsed.xml_root_input, parsed.xml_row_input, parsed.max_rows, stderr_writer), + .yaml => yaml_mod.loadYamlInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), }; if (rows == 0) fatal("empty input", stderr_writer, .csv_error, .{}); } diff --git a/src/modes/stats.zig b/src/modes/stats.zig index 4ed23a2..e713a50 100644 --- a/src/modes/stats.zig +++ b/src/modes/stats.zig @@ -2,6 +2,7 @@ const std = @import("std"); const c = @import("c"); const json_mod = @import("../json.zig"); const xml_mod = @import("../xml.zig"); +const yaml_mod = @import("../yaml.zig"); const sqlite_mod = @import("../sqlite.zig"); const loader = @import("../loader.zig"); const format = @import("../format.zig"); @@ -87,6 +88,7 @@ fn loadTable( .json => json_mod.loadJsonArray(allocator, reader, db, table_name, parsed.max_rows, parsed.json_path, stderr_writer), .ndjson => json_mod.loadNdjsonInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), .xml => xml_mod.loadXmlInput(allocator, reader, db, table_name, parsed.xml_root_input, parsed.xml_row_input, parsed.max_rows, stderr_writer), + .yaml => yaml_mod.loadYamlInput(allocator, reader, db, table_name, parsed.max_rows, stderr_writer), }; if (rows == 0) fatal("empty input", stderr_writer, .csv_error, .{}); } diff --git a/src/modes/validate.zig b/src/modes/validate.zig index ed2e569..04382b1 100644 --- a/src/modes/validate.zig +++ b/src/modes/validate.zig @@ -1,7 +1,9 @@ const std = @import("std"); +const c = @import("c"); const csv_mod = @import("../csv.zig"); const json_mod = @import("../json.zig"); const xml_mod = @import("../xml.zig"); +const yaml_mod = @import("../yaml.zig"); const sqlite_mod = @import("../sqlite.zig"); const loader = @import("../loader.zig"); const args_mod = @import("../args.zig"); @@ -277,6 +279,40 @@ pub fn runValidate( std.process.exit(@intFromEnum(ExitCode.usage)); }; }, + .yaml => { + var yaml_buf: [4096]u8 = undefined; + const opened = source.openInput(io, input_source, stderr_writer); + defer opened.deinit(io); + var yaml_reader = std.Io.File.reader(opened.file, io, &yaml_buf); + const yaml_db = sqlite_mod.openDb(false, stderr_writer); + defer _ = c.sqlite3_close(yaml_db); + const count = yaml_mod.loadYamlInput(allocator, &yaml_reader.interface, yaml_db, "t", null, stderr_writer); + const cols = sqlite_mod.getTableColumns(allocator, yaml_db, "t", stderr_writer); + defer { + for (cols) |col| allocator.free(col); + allocator.free(cols); + } + var count_buf: [32]u8 = undefined; + const count_str = fmtThousands(&count_buf, count); + stdout_writer.print("OK: {s} rows, {d} columns (", .{ count_str, cols.len }) catch |err| { + std.log.err("failed to write output: {}", .{err}); + std.process.exit(@intFromEnum(ExitCode.usage)); + }; + for (cols, 0..) |col, i| { + if (i > 0) stdout_writer.writeAll(", ") catch |err| { + std.log.err("failed to write output: {}", .{err}); + std.process.exit(@intFromEnum(ExitCode.usage)); + }; + stdout_writer.print("{s} TEXT", .{col}) catch |err| { + std.log.err("failed to write output: {}", .{err}); + std.process.exit(@intFromEnum(ExitCode.usage)); + }; + } + stdout_writer.writeAll(")\n") catch |err| { + std.log.err("failed to write output: {}", .{err}); + std.process.exit(@intFromEnum(ExitCode.usage)); + }; + }, .xml => { var read_buf: [4096]u8 = undefined; const opened = source.openInput(io, input_source, stderr_writer); diff --git a/src/yaml.zig b/src/yaml.zig new file mode 100644 index 0000000..8874e52 --- /dev/null +++ b/src/yaml.zig @@ -0,0 +1,220 @@ +//! YAML input loading — reads YAML sequence-of-mappings as rows. +//! +//! loadYamlInput +//! Read all of `reader` as a YAML document, create table `table_name` in `db`, +//! and insert every mapping as a row. Expects a top-level YAML sequence of +//! mappings (list of objects). All columns are TEXT (no type inference). +//! Comments and flow-style mappings are handled by libyaml transparently. +//! Multi-document streams, anchors/aliases, non-string keys, and nested +//! sequences/mappings are rejected with a fatal error. + +const std = @import("std"); +const c = @import("c"); +const yaml = @import("yaml"); + +const sqlite_helpers = @import("sqlite.zig"); + +const createAllTextTable = sqlite_helpers.createAllTextTable; +const prepareInsertStmt = sqlite_helpers.prepareInsertStmt; +const beginTransaction = sqlite_helpers.beginTransaction; +const commitTransaction = sqlite_helpers.commitTransaction; +const fatal = sqlite_helpers.fatal; +const ExitCode = sqlite_helpers.ExitCode; +const sqlite_static = sqlite_helpers.sqlite_static; + +/// loadYamlInput(allocator, reader, db, table_name, max_rows, stderr_writer) → usize +/// +/// Pre: reader is positioned at the start of a YAML document (single stream) +/// db is an open, empty SQLite database +/// Post: table is created with TEXT columns from the first mapping's keys; +/// all mappings are inserted as rows +/// result = number of rows inserted +/// aborts the process on any parse, I/O, or SQL error +pub fn loadYamlInput( + allocator: std.mem.Allocator, + reader: *std.Io.Reader, + db: *c.sqlite3, + table_name: []const u8, + max_rows: ?usize, + stderr_writer: *std.Io.Writer, +) usize { + // Read all input into a buffer + const buf = sqlite_helpers.readAllInput(allocator, reader, stderr_writer, "YAML input"); + defer allocator.free(buf); + + if (buf.len == 0) return 0; // Empty input - return 0 rows gracefully + + // Initialize libyaml parser + var parser: yaml.yaml_parser_t = undefined; + if (yaml.yaml_parser_initialize(&parser) != 1) + fatal("out of memory initializing YAML parser", stderr_writer, .csv_error, .{}); + defer yaml.yaml_parser_delete(&parser); + yaml.yaml_parser_set_input_string(&parser, @as([*c]const u8, @ptrCast(buf.ptr)), buf.len); + + // State variables + var cols: std.ArrayList([]const u8) = .empty; // column names (first mapping keys, owned) + var first_row_vals: std.ArrayList([]const u8) = .empty; // first row's values (owned) + var current_keys: std.ArrayList([]const u8) = .empty; // keys for current mapping (owned) + var current_vals: std.ArrayList([]const u8) = .empty; // values for current mapping (owned) + var current_key: ?[]const u8 = null; // temp key storage (owned when non-null) + var rows_inserted: usize = 0; + var got_first_mapping = false; + var in_transaction = false; + var stmt: ?*c.sqlite3_stmt = null; + var mapping_depth: usize = 0; + var outer_sequence_ended = false; // for multi-doc detection + var in_outer_sequence = false; + + defer { + for (cols.items) |c_name| allocator.free(c_name); + cols.deinit(allocator); + for (first_row_vals.items) |v| allocator.free(v); + first_row_vals.deinit(allocator); + for (current_keys.items) |k| allocator.free(k); + current_keys.deinit(allocator); + for (current_vals.items) |v| allocator.free(v); + current_vals.deinit(allocator); + if (current_key) |k| allocator.free(k); + if (stmt) |s| _ = c.sqlite3_finalize(s); + } + + while (true) { + var event: yaml.yaml_event_t = undefined; + if (yaml.yaml_parser_parse(&parser, &event) != 1) { + const mark = parser.problem_mark; + const problem = if (parser.problem) |p| std.mem.span(p) else "(unknown)"; + fatal("YAML parse error at line {d}, col {d}: {s}", stderr_writer, .csv_error, .{ + mark.line + 1, mark.column + 1, problem, + }); + } + defer yaml.yaml_event_delete(&event); + + switch (@as(c_int, @intCast(event.type))) { + yaml.YAML_STREAM_START_EVENT => {}, + yaml.YAML_STREAM_END_EVENT => break, + + yaml.YAML_DOCUMENT_START_EVENT => { + if (outer_sequence_ended) + fatal("YAML input: multiple documents not supported (use a single document with one top-level sequence)", stderr_writer, .csv_error, .{}); + }, + yaml.YAML_DOCUMENT_END_EVENT => {}, + + yaml.YAML_SEQUENCE_START_EVENT => { + if (mapping_depth > 0) { + // Nested sequence inside a mapping (as key or value) + if (current_key == null) + fatal("YAML input: mapping keys must be scalar strings", stderr_writer, .csv_error, .{}) + else + fatal("YAML input: mapping values must be scalar strings (nested sequences not supported)", stderr_writer, .csv_error, .{}); + } + in_outer_sequence = true; + }, + yaml.YAML_SEQUENCE_END_EVENT => { + outer_sequence_ended = true; + }, + + yaml.YAML_MAPPING_START_EVENT => { + mapping_depth += 1; + if (mapping_depth == 1 and !in_outer_sequence) + fatal("YAML input must be a top-level sequence of mappings", stderr_writer, .csv_error, .{}); + if (mapping_depth > 1) + fatal("YAML input: nested mappings are not supported", stderr_writer, .csv_error, .{}); + current_key = null; + }, + + yaml.YAML_MAPPING_END_EVENT => { + mapping_depth -= 1; + + if (!got_first_mapping) { + // First mapping: keys define the schema + if (current_keys.items.len == 0) + fatal("first YAML mapping has no keys", stderr_writer, .csv_error, .{}); + + // Take ownership of current_keys as cols + cols = current_keys; + current_keys = .empty; + first_row_vals = current_vals; + current_vals = .empty; + + createAllTextTable(allocator, db, table_name, cols.items, stderr_writer); + beginTransaction(db, stderr_writer); + in_transaction = true; + stmt = prepareInsertStmt(allocator, db, table_name, cols.items.len, stderr_writer); + + // Insert first row + _ = c.sqlite3_reset(stmt.?); + _ = c.sqlite3_clear_bindings(stmt.?); + for (first_row_vals.items, 0..) |v, j| { + if (c.sqlite3_bind_text(stmt.?, @intCast(j + 1), v.ptr, @intCast(v.len), sqlite_static) != c.SQLITE_OK) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + } + if (c.sqlite3_step(stmt.?) != c.SQLITE_DONE) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + rows_inserted = 1; + sqlite_helpers.checkMaxRows(rows_inserted, max_rows, stderr_writer); + got_first_mapping = true; + } else { + // Subsequent mappings: bind values by key lookup + _ = c.sqlite3_reset(stmt.?); + _ = c.sqlite3_clear_bindings(stmt.?); + + for (cols.items, 0..) |col_name, j| { + const param_idx: c_int = @intCast(j + 1); + var found = false; + for (current_keys.items, current_vals.items) |k, v| { + if (std.mem.eql(u8, k, col_name)) { + if (c.sqlite3_bind_text(stmt.?, param_idx, v.ptr, @intCast(v.len), sqlite_static) != c.SQLITE_OK) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + found = true; + break; + } + } + if (!found) { + if (c.sqlite3_bind_null(stmt.?, param_idx) != c.SQLITE_OK) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + } + } + if (c.sqlite3_step(stmt.?) != c.SQLITE_DONE) + fatal("{s}", stderr_writer, .sql_error, .{std.mem.span(c.sqlite3_errmsg(db))}); + rows_inserted += 1; + sqlite_helpers.checkMaxRows(rows_inserted, max_rows, stderr_writer); + } + + // Free current mapping keys and values (except when taken above) + for (current_keys.items) |k| allocator.free(k); + for (current_vals.items) |v| allocator.free(v); + current_keys.items.len = 0; + current_vals.items.len = 0; + }, + + yaml.YAML_SCALAR_EVENT => { + const value_slice: []const u8 = @as([*]const u8, @ptrCast(event.data.scalar.value))[0..event.data.scalar.length]; + if (current_key == null) { + // This is a key + current_key = allocator.dupe(u8, value_slice) catch + fatal("out of memory", stderr_writer, .csv_error, .{}); + } else { + // This is a value — pair it with the stored key + const key = current_key.?; + current_key = null; + const val_dup = allocator.dupe(u8, value_slice) catch + fatal("out of memory", stderr_writer, .csv_error, .{}); + current_keys.append(allocator, key) catch + fatal("out of memory", stderr_writer, .csv_error, .{}); + current_vals.append(allocator, val_dup) catch + fatal("out of memory", stderr_writer, .csv_error, .{}); + } + }, + + yaml.YAML_ALIAS_EVENT => fatal("YAML input: anchors and aliases are not supported", stderr_writer, .csv_error, .{}), + yaml.YAML_NO_EVENT => unreachable, + else => {}, + } + } + + // No rows loaded at all — return 0 (graceful) + if (cols.items.len == 0) return 0; + + if (in_transaction) commitTransaction(db, stderr_writer); + return rows_inserted; +} diff --git a/tests/fixtures/users.yaml b/tests/fixtures/users.yaml new file mode 100644 index 0000000..ee9890a --- /dev/null +++ b/tests/fixtures/users.yaml @@ -0,0 +1,17 @@ +# Sample products fixture mirroring tests/fixtures/products.json +- name: Widget + price: 25.00 + category: hardware + stock: 150 +- name: Gadget + price: 40.25 + category: electronics + stock: 75 +- name: Doohickey + price: 50.00 + category: hardware + stock: 30 +- name: Thingamajig + price: 60.00 + category: electronics + stock: 0 diff --git a/tests/fixtures/users.yml b/tests/fixtures/users.yml new file mode 100644 index 0000000..ee9890a --- /dev/null +++ b/tests/fixtures/users.yml @@ -0,0 +1,17 @@ +# Sample products fixture mirroring tests/fixtures/products.json +- name: Widget + price: 25.00 + category: hardware + stock: 150 +- name: Gadget + price: 40.25 + category: electronics + stock: 75 +- name: Doohickey + price: 50.00 + category: hardware + stock: 30 +- name: Thingamajig + price: 60.00 + category: electronics + stock: 0 From 493368a6142b09b402988df6e97880e798cc0611 Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 14:55:34 +0200 Subject: [PATCH 02/10] docs: add YAML input documentation --- README.md | 35 ++++++++++++++++++++++++++++------- docs/sql-pipe.1.scd | 39 ++++++++++++++++++++------------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 8e9d282..5cc00c9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Release](https://img.shields.io/github/v/release/vmvarela/sql-pipe)](https://github.com/vmvarela/sql-pipe/releases/latest) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -`sql-pipe` reads CSV, JSON, or NDJSON from stdin or file arguments, loads it into an in-memory SQLite database, runs a SQL query, and prints the results. No server, no schema files, no setup. +`sql-pipe` reads CSV, JSON, NDJSON, YAML, or XML from stdin or file arguments, loads it into an in-memory SQLite database, runs a SQL query, and prints the results. No server, no schema files, no setup. It exists because `awk` is cryptic, spinning up a Python interpreter for a one-liner feels wrong, and `sqlite3 :memory:` takes four commands before you can query anything. If you know SQL and work with CSV in the terminal, this is the tool you've been reaching for. @@ -196,6 +196,24 @@ $ printf '[{"name":"Alice","score":95},{"name":"Bob","score":72}]' \ Alice,95 ``` +For YAML input, pass `-I yaml` (reads a top-level sequence of mappings). Both `.yaml` and `.yml` extensions are auto-detected. All values are loaded as `TEXT`: + +```sh +$ cat users.yaml +- name: Alice + country: US +- name: Bob + country: UK +- name: Carol + country: DE + +$ cat users.yaml | sql-pipe -I yaml 'SELECT name FROM t WHERE country != "US" ORDER BY name' +Bob +Carol +``` + +YAML is auto-detected from the `.yaml` and `.yml` extensions, or use `-I yaml` explicitly. Comments (`#`), flow-style mappings (`[{name: Alice}]`), and standard scalar types (strings, numbers, booleans) work transparently. Multi-document streams and anchors/aliases are rejected with a clear error. + Columns are auto-detected as `INTEGER`, `REAL`, `DATE`, `DATETIME`, or `TEXT` based on the first 100 rows. Date and datetime values are normalized to ISO 8601 on insert, so SQLite date functions (`date()`, `strftime()`, `julianday()`) work immediately. Use `--no-type-inference` to force all columns to `TEXT`: ```sh @@ -258,7 +276,7 @@ $ cat events.xml | sql-pipe -I xml --xml-root events --xml-row event \ ### File arguments -Pass files as positional arguments instead of piping through stdin. Each file becomes a table named after its basename (without extension). The input format is auto-detected from the file extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.xml`): +Pass files as positional arguments instead of piping through stdin. Each file becomes a table named after its basename (without extension). The input format is auto-detected from the file extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.yaml`, `.yml`, `.xml`): ```sh # Single file — no more cat @@ -267,6 +285,9 @@ $ sql-pipe orders.csv 'SELECT * FROM orders WHERE amount > 100' # JSON file — extension tells sql-pipe the format, no -I needed $ sql-pipe data.json 'SELECT * FROM data WHERE score > 80' +# YAML file — .yaml and .yml both auto-detect +$ sql-pipe users.yaml 'SELECT name FROM users WHERE country = "US"' + # Multi-file join — the #1 reason people reach for DuckDB $ sql-pipe orders.csv customers.csv \ 'SELECT c.name, SUM(o.amount) FROM orders o @@ -321,15 +342,15 @@ When `-f` is used, all positional arguments are treated as data files (no positi |------|-------------| | `-d`, `--delimiter ` | Input field delimiter (single character, default `,`) | | `--tsv` | Alias for `--delimiter '\t'` | -| `-I`, `--input-format ` | Input format: `csv` (default), `tsv`, `json`, `ndjson`, `xml`. Overrides file extension auto-detection. | +| `-I`, `--input-format ` | Input format: `csv` (default), `tsv`, `json`, `ndjson`, `yaml`, `xml`. Overrides file extension auto-detection. Both `.yaml` and `.yml` file extensions auto-detect to `yaml`. | | `-O`, `--output-format ` | Output format: `csv` (default), `tsv`, `json`, `ndjson`, `xml`, `markdown` (alias: `md`), `html`, `sql` | | `--sql-table ` | Target table name for `-O sql` INSERT output (default: `t`) | | `--no-type-inference` | Treat all columns as TEXT (skip auto-detection) | | `-H`, `--header` | Print column names as the first output row (CSV/TSV/HTML) | | `--json` | Alias for `--output-format json` (mutually exclusive with `-H`) | | `--max-rows ` | Stop if more than `n` data rows are read (exit 1) | -| `--validate` | Parse the entire input and print a summary (`OK: rows, columns (col TYPE, ...)`) to stdout. Exit 0 on success, exit 2 on parse error. No query required. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, xml). JSON/NDJSON/XML columns are reported as TEXT. | -| `--columns` | Read the input header, print each column name on its own line, and exit 0. Supports CSV, TSV, JSON, NDJSON, and XML input. With `-v`/`--verbose`, also shows the inferred type per column (`name INTEGER`). Respects `--delimiter` and `--tsv`. Mutually exclusive with a query argument. | +| `--validate` | Parse the entire input and print a summary (`OK: rows, columns (col TYPE, ...)`) to stdout. Exit 0 on success, exit 2 on parse error. No query required. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format` (csv, tsv, json, ndjson, yaml, xml). JSON/NDJSON/YAML/XML columns are reported as TEXT. | +| `--columns` | Read the input header, print each column name on its own line, and exit 0. Supports CSV, TSV, JSON, NDJSON, YAML, and XML input. With `-v`/`--verbose`, also shows the inferred type per column (`name INTEGER`). Respects `--delimiter` and `--tsv`. Mutually exclusive with a query argument. | | `--sample []` | Print a schema comment block to stderr and the first `` data rows to stdout as CSV (default: `n=10`). The schema block lists each column name and its inferred type, prefixed with `#`. Implies `--header`. Compatible with `--delimiter` and `--tsv`. Mutually exclusive with `--json` and a query argument. No query required. | | `--stats` | Load input and print per-column statistics (column name, type, non-null count, min, max, mean) as a formatted table. Mean is blank for non-numeric columns. Compatible with `--delimiter`, `--tsv`, `--no-type-inference`, `-I`/`--input-format`. Mutually exclusive with a query, `--columns`, `--validate`, `--sample`, and `--output`. | | `--profile` | Alias for `--stats` | @@ -643,10 +664,10 @@ The database never touches disk and vanishes when the process exits. No state, n ## Limitations -- **File format auto-detection** is based on file extension. Files without a recognized extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.xml`) default to CSV. Use `-I` to override. +- **File format auto-detection** is based on file extension. Files without a recognized extension (`.csv`, `.tsv`, `.json`, `.ndjson`, `.yaml`, `.yml`, `.xml`) default to CSV. Use `-I` to override. ## Related - **[q](https://harelba.github.io/q/)** — Python-based SQL on tabular data. Similar concept, but requires Python runtime. Better if you're already in a Python environment or need Python-specific integrations. -- **[trdsql](https://github.com/noborus/trdsql)** — Go alternative with broader format support (LTSV, TBLN) and more output options. Better if you need formats beyond CSV/JSON/NDJSON/XML or want more output formatting choices. +- **[trdsql](https://github.com/noborus/trdsql)** — Go alternative with broader format support (LTSV, TBLN) and more output options. Better if you need formats beyond CSV/JSON/NDJSON/YAML/XML or want more output formatting choices. - **[sqlite-utils](https://sqlite-utils.datasette.io/)** — better if you need persistent databases, schema management, or Python scripting. sql-pipe is designed for one-shot queries on ephemeral in-memory data. diff --git a/docs/sql-pipe.1.scd b/docs/sql-pipe.1.scd index 8c32f3b..7274ba0 100644 --- a/docs/sql-pipe.1.scd +++ b/docs/sql-pipe.1.scd @@ -1,7 +1,7 @@ sql-pipe(1) NAME - sql-pipe - Execute SQL queries on CSV/JSON/XML data from stdin or files + sql-pipe - Execute SQL queries on CSV/JSON/NDJSON/YAML/XML data from stdin or files SYNOPSIS *sql-pipe* [OPTIONS] [...] @@ -16,10 +16,10 @@ SYNOPSIS *sql-pipe* --completions DESCRIPTION - sql-pipe reads CSV, JSON, NDJSON, or XML data from standard input and/or - file arguments, loads it into an in-memory SQLite database, executes the - provided SQL query, and prints the results as CSV (or JSON/XML when - requested) to standard output. + sql-pipe reads CSV, JSON, NDJSON, YAML, or XML data from standard input + and/or file arguments, loads it into an in-memory SQLite database, + executes the provided SQL query, and prints the results as CSV (or + JSON/XML when requested) to standard output. When reading from stdin, data is loaded into a table named *t*. When files are passed as positional arguments, each file becomes a table named after @@ -27,9 +27,9 @@ DESCRIPTION Stdin and file arguments can be combined — stdin is always table *t*. Input format for files is auto-detected from the file extension (*.csv*, - *.tsv*, *.json*, *.ndjson*, *.xml*). Unrecognized or missing extensions - default to CSV. Use *-I* to override auto-detection (e.g., when a - TSV file has a *.txt* extension). + *.tsv*, *.json*, *.ndjson*, *.yaml*, *.yml*, *.xml*). Unrecognized or + missing extensions default to CSV. Use *-I* to override auto-detection + (e.g., when a TSV file has a *.txt* extension). This tool is useful for quick data transformations, filtering, grouping, aggregations, and multi-file joins without manual SQL database setup. @@ -61,10 +61,11 @@ OPTIONS *-I, --input-format* Set the input format explicitly: *csv* (default), *tsv*, *json*, - *ndjson*, or *xml*. When set, overrides file extension auto-detection - for all file arguments. Stdin always uses this value (no filename - to inspect). Useful when a file has an ambiguous extension (*.txt*, - *.dat*) or no extension at all. + *ndjson*, *yaml*, or *xml*. When set, overrides file extension + auto-detection for all file arguments. Stdin always uses this value + (no filename to inspect). Useful when a file has an ambiguous + extension (*.txt*, *.dat*) or no extension at all. Both *.yaml* and + *.yml* extensions auto-detect to *yaml*. *-O, --output-format* Set the output format: *csv* (default), *tsv*, *json*, *ndjson*, @@ -124,14 +125,14 @@ OPTIONS *OK: rows, columns ( , ...)* and exits 0. On parse error, prints the error message and exits 2. Compatible with *--delimiter*, *--tsv*, *--no-type-inference*, and - *-I* / *--input-format* (csv, tsv, json, ndjson, xml). JSON, NDJSON, - and XML columns are reported as TEXT. Mutually exclusive with a query - argument. + *-I* / *--input-format* (csv, tsv, json, ndjson, yaml, xml). JSON, + NDJSON, YAML, and XML columns are reported as TEXT. Mutually + exclusive with a query argument. *--columns* Read the input header, print each column name on its own line to standard output, and exit with code 0. Supported for CSV, TSV, - JSON, NDJSON, and XML input. When combined with *-v* / *--verbose*, + JSON, NDJSON, YAML, and XML input. When combined with *-v* / *--verbose*, also shows the inferred type (INTEGER, REAL, DATE, DATETIME, or TEXT) for each column (CSV/TSV only; other formats always show TEXT), using the first 100 data rows for inference. Respects *--delimiter* and *--tsv*. @@ -154,8 +155,8 @@ OPTIONS non-null count, min, max, mean) as a formatted table. Mean is blank for non-numeric columns. Compatible with *--delimiter*, *--tsv*, *--no-type-inference*, and *-I* / *--input-format* (csv, tsv, json, - ndjson, xml). Mutually exclusive with a query argument, *--columns*, - *--validate*, *--sample*, and *--output*. + ndjson, yaml, xml). Mutually exclusive with a query argument, + *--columns*, *--validate*, *--sample*, and *--output*. Alias: *--profile*. *--schema* @@ -163,7 +164,7 @@ OPTIONS output. Each input file generates one DDL block; stdin uses table name *t*. No query is required. Compatible with *--delimiter*, *--tsv*, *--no-type-inference*, and *-I* / *--input-format* (csv, - tsv, json, ndjson, xml). Mutually exclusive with a query argument, + tsv, json, ndjson, yaml, xml). Mutually exclusive with a query argument, *--columns*, *--validate*, *--sample*, *--stats*, *--explain*, and *--output*. From 3292adc5c934e8041b9d627415979d85ea8636b5 Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 15:39:20 +0200 Subject: [PATCH 03/10] fix: win --- build.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.zig b/build.zig index 43e71e2..2577623 100644 --- a/build.zig +++ b/build.zig @@ -79,6 +79,9 @@ pub fn build(b: *std.Build) void { exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"), .flags = &.{} }); exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); + // zig_compat.c is a Windows MSVC shim (see file header). The CI step + // copies 10 libyaml files from the tarball; zig_compat.c is local-only. + exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/zig_compat.c"), .flags = &.{} }); b.installArtifact(exe); From 7c35a3dc14caceeaa1cb25ac04e64703a6c58a06 Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 15:49:35 +0200 Subject: [PATCH 04/10] fix(ci): materialize lib/yaml/zig_compat.c in libyaml step build.zig:84 always adds lib/yaml/zig_compat.c as a C source, but the CI step 'Download libyaml amalgamation' only copied 8 files from the upstream tarball and never wrote zig_compat.c. The file exists locally (gitignored, amalgamation pattern) but not on CI runners, so zig build failed on all 3 OS with 'file_hash FileNotFound'. Add a heredoc inside the same step that produces byte-identical zig_compat.c at the project root. Also correct the stale '10 files' comment in build.zig. --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ build.zig | 3 ++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfa89e3..43c1046 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,46 @@ jobs: cp /tmp/libyaml-0.2.5/src/reader.c lib/yaml/reader.c cp /tmp/libyaml-0.2.5/src/writer.c lib/yaml/writer.c cp /tmp/libyaml-0.2.5/src/loader.c lib/yaml/loader.c + cat > lib/yaml/zig_compat.c <<'ZIG_COMPAT_EOF' + /* + * Zig 0.16 + MSVC + libyaml compatibility shim. + * + * On Windows MSVC, the secure CRT functions wcscat_s/wcscpy_s are declared + * in , which gets pulled in transitively via lib/yaml/yaml.h's + * include. libyaml never calls them, so Zig 0.16's C-to-Zig + * wrapper marks them as unused local consts and fails to compile. The same + * functions do not exist in glibc/BSD libc, so Linux and macOS are + * unaffected. + * + * This file force-references wcscat_s/wcscpy_s in a constructor function + * that runs at program startup, so the Zig wrapper sees them as used. + * On non-MSVC the silencer is a no-op kept alive by __attribute__((used)). + * ponytail: zig-0.16 windows MSVC workaround + */ + #include "yaml_private.h" + + #if defined(_MSC_VER) + # pragma comment(linker, "/include:_zig_yaml_compat_init") + # pragma init_seg(".CRT$XCU") + static void _zig_yaml_compat_silencer(void) { + /* volatile forces the compiler to actually compute the function + * addresses; (void) cast alone can be optimized away. */ + volatile void *p1 = (void *)wcscat_s; + volatile void *p2 = (void *)wcscpy_s; + (void)p1; (void)p2; + } + __declspec(noinline) static void _zig_yaml_compat_init(void) { + _zig_yaml_compat_silencer(); + } + #elif defined(__GNUC__) || defined(__clang__) + __attribute__((used, constructor)) + static void _zig_yaml_compat_silencer(void) { + /* No-op on non-MSVC platforms; the attributes keep this in the + * binary and let it run harmlessly at startup. */ + (void)0; + } + #endif + ZIG_COMPAT_EOF echo "libyaml 0.2.5 ready" - name: Build diff --git a/build.zig b/build.zig index 2577623..67d9c5b 100644 --- a/build.zig +++ b/build.zig @@ -80,7 +80,8 @@ pub fn build(b: *std.Build) void { exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); // zig_compat.c is a Windows MSVC shim (see file header). The CI step - // copies 10 libyaml files from the tarball; zig_compat.c is local-only. + // copies 8 libyaml files from the tarball and materializes zig_compat.c + // via heredoc; the file is gitignored locally. exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/zig_compat.c"), .flags = &.{} }); b.installArtifact(exe); From 81fd2405ffd89b80ec793e1b9464af8a77efd6df Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 16:19:00 +0200 Subject: [PATCH 05/10] =?UTF-8?q?fix:=20windows=20CI=20failure=20=E2=80=94?= =?UTF-8?q?=20wrap=20yaml.h=20in=20translate=20header=20to=20hide=20MSVC?= =?UTF-8?q?=20secure=20CRT=20from=20Zig=20translator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Windows MSVC, yaml.h's transitively pulls in which declares wcscat_s/wcscpy_s. Zig's translateC generates bindings for these, but libyaml never calls them, causing 'unused local constant' errors. The existing zig_compat.c runtime shim was insufficient because Zig checks const usage at compile time during translation. Introduce a wrapper header that defines these functions as no-op macros on MSVC before including yaml.h, preventing the translator from ever seeing them. --- build.zig | 2 +- lib/yaml/yaml_translate.h | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 lib/yaml/yaml_translate.h diff --git a/build.zig b/build.zig index 67d9c5b..259d66e 100644 --- a/build.zig +++ b/build.zig @@ -48,7 +48,7 @@ pub fn build(b: *std.Build) void { // libyaml is always bundled (no --bundle-yaml option) — it's tiny and // not always available as a system library. const translate_yaml = b.addTranslateC(.{ - .root_source_file = b.path("lib/yaml/yaml.h"), + .root_source_file = b.path("lib/yaml/yaml_translate.h"), .target = target, .optimize = optimize, }); diff --git a/lib/yaml/yaml_translate.h b/lib/yaml/yaml_translate.h new file mode 100644 index 0000000..94395bd --- /dev/null +++ b/lib/yaml/yaml_translate.h @@ -0,0 +1,11 @@ +#ifndef YAML_TRANSLATE_H +#define YAML_TRANSLATE_H + +#if defined(_MSC_VER) +# define wcscat_s(dest, destsz, src) ((void)0) +# define wcscpy_s(dest, destsz, src) ((void)0) +#endif + +#include "yaml.h" + +#endif From 57710c77cf1763ff40fc08bd5a8fd18ced6cb961 Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 16:27:30 +0200 Subject: [PATCH 06/10] =?UTF-8?q?fix:=20windows=20CI=20failure=20=E2=80=94?= =?UTF-8?q?=20hide=20MSVC=20secure=20CRT=20from=20Zig=20translator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yaml.h's transitively pulls in on MSVC, declaring wcscat_s/wcscpy_s. Zig 0.16's translateC generates bindings for these but libyaml never calls them, causing 'unused local constant' errors. Define them as empty macros via defineCMacroRaw before translation so the translator never sees them. The existing zig_compat.c runtime shim remains as defense-in-depth. --- build.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.zig b/build.zig index 259d66e..bcc1520 100644 --- a/build.zig +++ b/build.zig @@ -52,6 +52,9 @@ pub fn build(b: *std.Build) void { .target = target, .optimize = optimize, }); + // Hide MSVC secure CRT functions from translator to avoid "unused local constant" errors + translate_yaml.defineCMacroRaw("wcscat_s="); + translate_yaml.defineCMacroRaw("wcscpy_s="); exe.root_module.addImport("yaml", translate_yaml.createModule()); if (bundle_sqlite) { From e92508f53624093915dfc2a0f6bfa6111176745f Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 16:32:29 +0200 Subject: [PATCH 07/10] =?UTF-8?q?fix:=20windows=20CI=20=E2=80=94=20rename?= =?UTF-8?q?=20MSVC=20secure=20CRT=20funcs=20via=20macro=20to=20avoid=20tra?= =?UTF-8?q?nslator=20syntax=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -D wcscat_s= broke translator (invalid syntax in function declaration). Rename to _wcscat_s instead so generated Zig constants start with '_' (suppresses unused warning) and translator sees valid C syntax. --- build.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build.zig b/build.zig index bcc1520..c239b45 100644 --- a/build.zig +++ b/build.zig @@ -52,9 +52,10 @@ pub fn build(b: *std.Build) void { .target = target, .optimize = optimize, }); - // Hide MSVC secure CRT functions from translator to avoid "unused local constant" errors - translate_yaml.defineCMacroRaw("wcscat_s="); - translate_yaml.defineCMacroRaw("wcscpy_s="); + // Hide MSVC secure CRT functions from translator to avoid "unused local constant" errors. + // Rename with '_' prefix so generated Zig constants start with '_' (suppresses unused warning). + translate_yaml.defineCMacroRaw("wcscat_s=_wcscat_s"); + translate_yaml.defineCMacroRaw("wcscpy_s=_wcscpy_s"); exe.root_module.addImport("yaml", translate_yaml.createModule()); if (bundle_sqlite) { From 79f6affc25a859d95fffc67df82d26336d21e4ab Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 16:49:43 +0200 Subject: [PATCH 08/10] =?UTF-8?q?fix:=20windows=20CI=20=E2=80=94=20pre-def?= =?UTF-8?q?ine=20=5FINC=5FSTRING=20to=20avoid=20wcscat=5Fs/wcscpy=5Fs=20in?= =?UTF-8?q?=20translateC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yaml.h includes which on MSVC pulls in declaring wcscat_s/wcscpy_s. Zig 0.16's translateC generates bindings for all declarations from system headers, but libyaml never calls these secure CRT functions, causing 'unused local constant' errors. Instead of a minimal header (breaks opaque types), rename macros (breaks translator syntax), or static inline definitions (still generates unused bindings), we pre-define _INC_STRING in the wrapper header to skip string.h entirely. size_t is provided via instead. --- build.zig | 4 ---- lib/yaml/yaml_translate.h | 21 ++++++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/build.zig b/build.zig index c239b45..259d66e 100644 --- a/build.zig +++ b/build.zig @@ -52,10 +52,6 @@ pub fn build(b: *std.Build) void { .target = target, .optimize = optimize, }); - // Hide MSVC secure CRT functions from translator to avoid "unused local constant" errors. - // Rename with '_' prefix so generated Zig constants start with '_' (suppresses unused warning). - translate_yaml.defineCMacroRaw("wcscat_s=_wcscat_s"); - translate_yaml.defineCMacroRaw("wcscpy_s=_wcscpy_s"); exe.root_module.addImport("yaml", translate_yaml.createModule()); if (bundle_sqlite) { diff --git a/lib/yaml/yaml_translate.h b/lib/yaml/yaml_translate.h index 94395bd..dca0d68 100644 --- a/lib/yaml/yaml_translate.h +++ b/lib/yaml/yaml_translate.h @@ -1,11 +1,22 @@ +/* + * Wrapper header for Zig translateC that avoids the MSVC secure CRT trap. + * + * yaml.h includes which on MSVC pulls in declaring + * wcscat_s/wcscpy_s. Zig 0.16's translateC generates bindings for all + * declarations, but libyaml never calls these, causing "unused local + * constant" errors. + * + * Pre-define _INC_STRING to skip (we provide size_t via ). + * The C compilation step uses the real yaml.h with string.h intact. + */ + #ifndef YAML_TRANSLATE_H #define YAML_TRANSLATE_H -#if defined(_MSC_VER) -# define wcscat_s(dest, destsz, src) ((void)0) -# define wcscpy_s(dest, destsz, src) ((void)0) -#endif +#include + +#define _INC_STRING 1 #include "yaml.h" -#endif +#endif /* YAML_TRANSLATE_H */ \ No newline at end of file From 973bf2605b2cedd6a3ca73efff4451c977a1285c Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 16:56:32 +0200 Subject: [PATCH 09/10] =?UTF-8?q?chore:=20remove=20zig=5Fcompat.c=20refere?= =?UTF-8?q?nces=20=E2=80=94=20no=20longer=20needed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The _INC_STRING guard trick in yaml_translate.h eliminates the need for the MSVC secure CRT runtime shim. The file was gitignored and materialized by CI via heredoc; CI step will need update separately. --- build.zig | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.zig b/build.zig index 259d66e..78737e8 100644 --- a/build.zig +++ b/build.zig @@ -79,10 +79,6 @@ pub fn build(b: *std.Build) void { exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/reader.c"), .flags = &.{} }); exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/writer.c"), .flags = &.{} }); exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/loader.c"), .flags = &.{} }); - // zig_compat.c is a Windows MSVC shim (see file header). The CI step - // copies 8 libyaml files from the tarball and materializes zig_compat.c - // via heredoc; the file is gitignored locally. - exe.root_module.addCSourceFile(.{ .file = b.path("lib/yaml/zig_compat.c"), .flags = &.{} }); b.installArtifact(exe); From 71c7963c0181f32dd8f1935b58d3e49e54db0781 Mon Sep 17 00:00:00 2001 From: "Victor M. Varela" Date: Fri, 10 Jul 2026 17:01:50 +0200 Subject: [PATCH 10/10] =?UTF-8?q?ci:=20remove=20zig=5Fcompat.c=20shim=20?= =?UTF-8?q?=E2=80=94=20=5FINC=5FSTRING=20guard=20in=20yaml=5Ftranslate.h?= =?UTF-8?q?=20handles=20MSVC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43c1046..cfa89e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,46 +68,6 @@ jobs: cp /tmp/libyaml-0.2.5/src/reader.c lib/yaml/reader.c cp /tmp/libyaml-0.2.5/src/writer.c lib/yaml/writer.c cp /tmp/libyaml-0.2.5/src/loader.c lib/yaml/loader.c - cat > lib/yaml/zig_compat.c <<'ZIG_COMPAT_EOF' - /* - * Zig 0.16 + MSVC + libyaml compatibility shim. - * - * On Windows MSVC, the secure CRT functions wcscat_s/wcscpy_s are declared - * in , which gets pulled in transitively via lib/yaml/yaml.h's - * include. libyaml never calls them, so Zig 0.16's C-to-Zig - * wrapper marks them as unused local consts and fails to compile. The same - * functions do not exist in glibc/BSD libc, so Linux and macOS are - * unaffected. - * - * This file force-references wcscat_s/wcscpy_s in a constructor function - * that runs at program startup, so the Zig wrapper sees them as used. - * On non-MSVC the silencer is a no-op kept alive by __attribute__((used)). - * ponytail: zig-0.16 windows MSVC workaround - */ - #include "yaml_private.h" - - #if defined(_MSC_VER) - # pragma comment(linker, "/include:_zig_yaml_compat_init") - # pragma init_seg(".CRT$XCU") - static void _zig_yaml_compat_silencer(void) { - /* volatile forces the compiler to actually compute the function - * addresses; (void) cast alone can be optimized away. */ - volatile void *p1 = (void *)wcscat_s; - volatile void *p2 = (void *)wcscpy_s; - (void)p1; (void)p2; - } - __declspec(noinline) static void _zig_yaml_compat_init(void) { - _zig_yaml_compat_silencer(); - } - #elif defined(__GNUC__) || defined(__clang__) - __attribute__((used, constructor)) - static void _zig_yaml_compat_silencer(void) { - /* No-op on non-MSVC platforms; the attributes keep this in the - * binary and let it run harmlessly at startup. */ - (void)0; - } - #endif - ZIG_COMPAT_EOF echo "libyaml 0.2.5 ready" - name: Build