diff --git a/cmd/hardhat/targets.go b/cmd/hardhat/targets.go index 723205a..c2d4288 100644 --- a/cmd/hardhat/targets.go +++ b/cmd/hardhat/targets.go @@ -7,6 +7,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "fmt" "io" "maps" @@ -18,6 +19,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "cyphar.com/go-pathrs" "github.com/opencontainers/go-digest" @@ -31,6 +33,7 @@ import ( "go.amutable.dev/quarry/internal/keystore" "go.amutable.dev/quarry/internal/linux" "go.amutable.dev/quarry/internal/pathrsext" + "go.amutable.dev/quarry/internal/transferlayout" "go.amutable.dev/quarry/internal/tufext" ) @@ -101,6 +104,106 @@ func hashToTargets(ctx context.Context, builder *tufext.TargetsBuilder, logicalP return nil } +// maxInlineSize bounds the contents the --quarry-* options embed into +// targets.json. Inline data is meant for tiny metadata files, not payloads. +const maxInlineSize = 64 << 10 + +// addInlineBytes adds data as a target file with its contents inlined. +func addInlineBytes(builder *tufext.TargetsBuilder, logicalPath string, data []byte) (*tufmetadata.TargetFiles, error) { + if int64(len(data)) > maxInlineSize { + return nil, fmt.Errorf("target %s is too large (%d bytes) to be inlined (limit %d bytes)", logicalPath, len(data), maxInlineSize) + } + target, err := builder.AddTargetFile(logicalPath, int64(len(data)), hashAlgorithm.FromBytes(data)) + if err != nil { + return nil, fmt.Errorf("add file %s to targets data: %w", logicalPath, err) + } + tufext.TargetFilesExt(target).WithInlineData(data) + return target, nil +} + +// applyCustomFrom merges the per-target "custom" objects from the given JSON +// file ({"": {}}) into the targets. Every path +// must name a target that was already added, so that a typo cannot silently +// drop metadata. No path may be under .quarry/ because the +// --quarry-* options own that metadata. Existing custom fields are kept unless +// overridden by key. +func applyCustomFrom(builder *tufext.TargetsBuilder, path string) error { + data, err := os.ReadFile(path) //nolint:forbidigo // user-controlled host path + if err != nil { + return fmt.Errorf("--custom-from=%q file is invalid: %w", path, err) + } + customs, err := jsonutils.Parse[map[string]map[string]json.RawMessage](data) + if err != nil { + return fmt.Errorf("--custom-from=%q file is invalid json: %w", path, err) + } + targets := builder.TargetsType().Targets + for targetPath, custom := range customs { + if strings.HasPrefix(targetPath, transferlayout.Prefix) { + return fmt.Errorf("--custom-from=%q: target %q is quarry metadata, which --custom-from must not change", path, targetPath) + } + target, ok := targets[targetPath] + if !ok { + return fmt.Errorf("--custom-from=%q: target %q does not exist", path, targetPath) + } + err := updateCustom(target, func(fields map[string]json.RawMessage) error { + maps.Copy(fields, custom) + return nil + }) + if err != nil { + return fmt.Errorf("--custom-from=%q: target %q: %w", path, targetPath, err) + } + } + return nil +} + +// updateCustom lets update change the fields of a target's custom object and +// stores the result. A missing or null custom object is an empty one. +func updateCustom(target *tufmetadata.TargetFiles, update func(fields map[string]json.RawMessage) error) error { + fields := map[string]json.RawMessage{} + if target.Custom != nil { + if err := json.Unmarshal(*target.Custom, &fields); err != nil { + return fmt.Errorf("existing custom data is not an object: %w", err) + } + } + if fields == nil { // "custom": null + fields = map[string]json.RawMessage{} + } + if err := update(fields); err != nil { + return err + } + encoded, err := jsonutils.MarshalNoEscapeHTML(fields) + if err != nil { + return fmt.Errorf("encode custom data: %w", err) + } + target.Custom = (*json.RawMessage)(&encoded) + return nil +} + +// addIncludedOverrideURLs points the targets of an --include-from file at +// dataRootURL. A target with inline data needs no URL because its bytes are +// in the metadata, and a target that already has an override URL keeps it. +func addIncludedOverrideURLs(targets map[string]*tufmetadata.TargetFiles, dataRootURL *url.URL) error { + for path, target := range targets { + ext := tufext.TargetFilesExt(target) + inline, err := ext.InlineData() + if err != nil { + return fmt.Errorf("target %s: %w", path, err) + } + if inline != nil { + continue + } + existing, err := ext.OverrideURL() + if err != nil { + return fmt.Errorf("target %s: %w", path, err) + } + if existing != nil { + continue + } + ext.WithOverrideURL(dataRootURL.JoinPath(path)) + } + return nil +} + const dataRootURLCtxKey ctxKey = "--data-root-url" func ctxDataRoolURL(ctx context.Context) *url.URL { @@ -252,10 +355,10 @@ func addPrehashedToTargets(ctx context.Context, builder *tufext.TargetsBuilder, return nil } -var targetsCommand = withKeystoreFlag(&cli.Command{ +var targetsCommand = withKeystoreFlagUnless(&cli.Command{ Name: "targets", Usage: "generate a TUF targets.json (or delegated target) file", - Flags: []cli.Flag{ + Flags: append([]cli.Flag{ &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, @@ -271,13 +374,18 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ Name: "include-from", Usage: "include targets defined in the given targets.json files (later files take precedence, with arguments taking highest precedence)", }, + &cli.StringSliceFlag{ + Name: "custom-from", + Usage: "merge per-target \"custom\" objects from the given JSON file ({\"\": {...}}), where every path must name a target", + TakesFile: true, + }, &cli.BoolFlag{ Name: "ext-override-url", - Usage: "when using --data-root-url --pre-hashed, add an override URL to each target file to make clients fetch from --data-root-url", + Usage: "with --data-root-url, add an override URL to make clients fetch from --data-root-url, for each --pre-hashed file and for each --include-from target without inline data or an override URL", }, &cli.StringFlag{ Name: "data-root-url", - Usage: "base url for target files (with --pre-hashed this allows you to generate a target.json entirely from a SHA256SUMS file)", + Usage: "base url for target files (with --pre-hashed this allows you to generate a target.json entirely from a SHA256SUMS file, and with --ext-override-url the --include-from targets point there)", }, &cli.BoolFlag{ Name: "pre-hashed", @@ -289,7 +397,7 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ Name: "expire-after", Usage: "configure the expiry of the targets file (duration relative to --ref-time)", }, - }, + }, quarryTargetsFlags()...), Arguments: []cli.Argument{ // TODO: This doesn't work if you have pre-hashed values. // TODO: This won't work if you have enough files to hit the @@ -318,14 +426,20 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ Usage: "use the given keys to sign the targets file", }, }, + { + &cli.BoolFlag{ + Name: "unsigned", + Usage: "write the targets file without signatures and, unless --expire-after is set, with the zero time as its expiry, for a consumer that signs it itself (needs no --keystore)", + }, + }, }, Required: true, }, }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { if urlStr := cmd.String("data-root-url"); urlStr != "" { - if !cmd.Bool("pre-hashed") { - return nil, fmt.Errorf("--data-root-url doesn't make sense without --pre-hashed") + if !cmd.Bool("pre-hashed") && !cmd.Bool("ext-override-url") { + return nil, fmt.Errorf("--data-root-url doesn't make sense without --pre-hashed or --ext-override-url") } rootURL, err := url.Parse(urlStr) if err != nil { @@ -335,7 +449,7 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ } if cmd.Bool("ext-override-url") { if !cmd.IsSet("data-root-url") { - return nil, fmt.Errorf("--ext-override-url doesn't make sense without --data-root-url (and --pre-hashed)") + return nil, fmt.Errorf("--ext-override-url doesn't make sense without --data-root-url") } ctx = context.WithValue(ctx, extOverrideURLCtxKey, true) } @@ -344,6 +458,13 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ Action: func(ctx context.Context, cmd *cli.Command) error { store := ctxKeystore(ctx) + // This happens before creating --output so that bad options leave it + // alone. + quarry, err := quarrySpecFromFlags(cmd) + if err != nil { + return err + } + var output io.Writer if outPath := cmd.String("output"); outPath != "-" { outFile, err := os.Create(outPath) //nolint:forbidigo // user-controlled host path @@ -358,6 +479,8 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ var keyIDs []string switch { + case cmd.Bool("unsigned"): + // No keys, so the targets file gets no signatures. case cmd.IsSet("from-root"): rootPath := cmd.String("from-root") @@ -374,7 +497,7 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ case cmd.IsSet("keyid"): keyIDs = cmd.StringSlice("keyid") default: - return fmt.Errorf("one of --from-root or --keyid must be set") + return fmt.Errorf("one of --from-root, --keyid or --unsigned must be set") } keys := make([]*keystore.GenericKey, 0, len(keyIDs)) @@ -414,6 +537,11 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ if targets.Signed.Delegations != nil { return fmt.Errorf("--include-from=%s file is unsupported: hardhat does not yet support delegations", includePath) } + if ctxExtOverrideURL(ctx) { + if err := addIncludedOverrideURLs(targets.Signed.Targets, ctxDataRoolURL(ctx)); err != nil { + return fmt.Errorf("--include-from=%q: %w", includePath, err) + } + } // Copy all of the target file specifications into the builder, // including any possible extension fields! maps.Copy(builder.TargetsType().Targets, targets.Signed.Targets) @@ -438,10 +566,38 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ } } + if err := addQuarryTargets(builder, quarry); err != nil { + return err + } + + for _, customPath := range cmd.StringSlice("custom-from") { + if err := applyCustomFrom(builder, customPath); err != nil { + return err + } + } + // This comes after --custom-from because that replaces a target's + // whole "quarry" object. The .quarry/ targets are skipped + // because they are quarry metadata and belong to no version. + if version := cmd.String("custom-version"); version != "" { + for path, target := range builder.TargetsType().Targets { + if strings.HasPrefix(path, transferlayout.Prefix) { + continue + } + if err := setQuarryCustomVersion(target, version); err != nil { + return fmt.Errorf("--custom-version: %s: %w", path, err) + } + } + } + signed, err := builder.SignWith(ctx, store, keys...) if err != nil { return fmt.Errorf("failed to sign targets data: %w", err) } + // Nothing verifies an unsigned file, so nothing checks its expiry + // either. The zero time says so, unless --expire-after asks for one. + if cmd.Bool("unsigned") && !cmd.IsSet("expire-after") { + signed.Signed.Expires = time.Time{} + } payload, err := cjson.EncodeCanonical(signed) if err != nil { return fmt.Errorf("failed to encode targets data: %w", err) @@ -457,4 +613,4 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ } return nil }, -}) +}, "unsigned") diff --git a/cmd/hardhat/targets_custom_test.go b/cmd/hardhat/targets_custom_test.go new file mode 100644 index 0000000..b65174b --- /dev/null +++ b/cmd/hardhat/targets_custom_test.go @@ -0,0 +1,72 @@ +// Copyright (C) 2026 Amutable GmbH + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.amutable.dev/quarry/internal/tufext" +) + +func customFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "custom.json") //nolint:forbidigo // test code + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) //nolint:forbidigo // test code + return path +} + +func TestApplyCustomFrom(t *testing.T) { + builder := tufext.NewTargetsBuilder() + _, err := builder.AddTargetFile("a.raw", 1, hashAlgorithm.FromBytes([]byte("a"))) + require.NoError(t, err) + existing, err := builder.AddTargetFile("b.raw", 1, hashAlgorithm.FromBytes([]byte("b"))) + require.NoError(t, err) + old := json.RawMessage(`{"quarry":{"old":true},"keep":1}`) + existing.Custom = &old + + null := json.RawMessage(`null`) + _, err = builder.AddTargetFile("c.raw", 1, hashAlgorithm.FromBytes([]byte("c"))) + require.NoError(t, err) + builder.TargetsType().Targets["c.raw"].Custom = &null + + path := customFile(t, `{ + "a.raw": {"quarry": {"version": "1", "update": true}}, + "b.raw": {"quarry": {"version": "1"}}, + "c.raw": {"quarry": {"version": "1"}} + }`) + require.NoError(t, applyCustomFrom(builder, path)) + + targets := builder.TargetsType().Targets + var a map[string]any + require.NoError(t, json.Unmarshal(*targets["a.raw"].Custom, &a)) + assert.Equal(t, map[string]any{"quarry": map[string]any{"version": "1", "update": true}}, a) + + // Keys are merged at the top level. "quarry" is replaced and "keep" survives. + var b map[string]any + require.NoError(t, json.Unmarshal(*targets["b.raw"].Custom, &b)) + assert.Equal(t, map[string]any{"quarry": map[string]any{"version": "1"}, "keep": float64(1)}, b) + + // An existing "custom": null is replaced. + var c map[string]any + require.NoError(t, json.Unmarshal(*targets["c.raw"].Custom, &c)) + assert.Equal(t, map[string]any{"quarry": map[string]any{"version": "1"}}, c) +} + +func TestApplyCustomFromRejects(t *testing.T) { + builder := tufext.NewTargetsBuilder() + _, err := builder.AddTargetFile(".quarry/sysupdate.d/x.transfer", 1, hashAlgorithm.FromBytes([]byte("a"))) + require.NoError(t, err) + for content, want := range map[string]string{ + `{"missing.raw": {"quarry": {}}}`: `target "missing.raw" does not exist`, + `["not", "an", "object"]`: "invalid json", + `{".quarry/sysupdate.d/x.transfer": {"quarry": {}}}`: "is quarry metadata", + } { + require.ErrorContains(t, applyCustomFrom(builder, customFile(t, content)), want, content) + } +} diff --git a/cmd/hardhat/targets_include_test.go b/cmd/hardhat/targets_include_test.go new file mode 100644 index 0000000..dbfa36e --- /dev/null +++ b/cmd/hardhat/targets_include_test.go @@ -0,0 +1,87 @@ +// Copyright (C) 2026 Amutable GmbH + +package main + +import ( + "encoding/json" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.amutable.dev/quarry/internal/tufext" +) + +func TestAddIncludedOverrideURLs(t *testing.T) { + builder := tufext.NewTargetsBuilder() + _, err := addInlineBytes(builder, ".quarry/sysupdate.d/a.transfer", []byte("abc")) + require.NoError(t, err) + kept, err := builder.AddTargetFile("kept.raw", 1, hashAlgorithm.FromBytes([]byte("k"))) + require.NoError(t, err) + tufext.TargetFilesExt(kept).WithOverrideURL(&url.URL{Scheme: "https", Host: "elsewhere.example", Path: "/kept.raw"}) + _, err = builder.AddTargetFile("dir/plain.raw", 1, hashAlgorithm.FromBytes([]byte("p"))) + require.NoError(t, err) + + root, err := url.Parse("https://data.example/repo/") + require.NoError(t, err) + targets := builder.TargetsType().Targets + require.NoError(t, addIncludedOverrideURLs(targets, root)) + + urlOf := func(path string) string { + u, err := tufext.TargetFilesExt(targets[path]).OverrideURL() + require.NoError(t, err) + if u == nil { + return "" + } + return u.String() + } + // Inline data needs no URL, and an existing URL is kept. + assert.Empty(t, urlOf(".quarry/sysupdate.d/a.transfer")) + assert.Equal(t, "https://elsewhere.example/kept.raw", urlOf("kept.raw")) + assert.Equal(t, "https://data.example/repo/dir/plain.raw", urlOf("dir/plain.raw")) +} + +// A signed targets file is included into an unsigned one, which needs no +// keystore, and its targets point at --data-root-url. +func TestTargetsUnsignedIncludeWithOverrideURLs(t *testing.T) { + dir := t.TempDir() + include := filepath.Join(dir, "include.json") //nolint:forbidigo // test code + abcHash := hashAlgorithm.FromBytes([]byte("abc")).Encoded() + doc := `{"signatures": [{"keyid": "x", "sig": "00"}], "signed": { + "_type": "targets", "spec_version": "1.0.31", "version": 1, "expires": "2030-01-01T00:00:00Z", + "targets": { + "a.raw": {"length": 1, "hashes": {"sha256": "` + strings.Repeat("a", 64) + `"}}, + ".quarry/sysupdate.d/a.transfer": {"length": 3, "hashes": {"sha256": "` + abcHash + `"}, "x-quarry-inline-data": "YWJj"} + }}}` + require.NoError(t, os.WriteFile(include, []byte(doc), 0o644)) //nolint:forbidigo // test code + out := filepath.Join(dir, "targets.json") //nolint:forbidigo // test code + + require.NoError(t, targetsCommand.Run(t.Context(), []string{ + "targets", + "--unsigned", + "--include-from", include, + "--data-root-url", "https://data.example/repo", + "--ext-override-url", + "--output", out, + })) + + data, err := os.ReadFile(out) //nolint:forbidigo // test code + require.NoError(t, err) + var got struct { + Signatures []json.RawMessage `json:"signatures"` + Signed map[string]json.RawMessage `json:"signed"` + } + require.NoError(t, json.Unmarshal(data, &got)) + assert.NotNil(t, got.Signatures) + assert.Empty(t, got.Signatures) + // The zero time states that the expiry is not checked. + assert.JSONEq(t, `"0001-01-01T00:00:00Z"`, string(got.Signed["expires"])) + var targets map[string]map[string]any + require.NoError(t, json.Unmarshal(got.Signed["targets"], &targets)) + assert.Equal(t, "https://data.example/repo/a.raw", targets["a.raw"]["x-quarry-override-url"]) + assert.NotContains(t, targets[".quarry/sysupdate.d/a.transfer"], "x-quarry-override-url") +} diff --git a/cmd/hardhat/targets_quarry.go b/cmd/hardhat/targets_quarry.go new file mode 100644 index 0000000..2f5c3fe --- /dev/null +++ b/cmd/hardhat/targets_quarry.go @@ -0,0 +1,243 @@ +// Copyright (C) 2026 Amutable GmbH + +package main + +import ( + "encoding/json" + "fmt" + "io/fs" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "unicode" + + tufmetadata "github.com/theupdateframework/go-tuf/v2/metadata" + "github.com/urfave/cli/v3" + + "go.amutable.dev/quarry/internal/jsonutils" + "go.amutable.dev/quarry/internal/transferlayout" + "go.amutable.dev/quarry/internal/tufext" +) + +// quarryTargetsFlags returns the --quarry-* options of "hardhat targets" (and +// --custom-version). They add the transfer definition directories of +// internal/transferlayout that quarry-sysupdate reads. hardhat lays them out +// from plain definition files, so producers need not know the layout. +func quarryTargetsFlags() []cli.Flag { + return []cli.Flag{ + &cli.StringSliceFlag{ + Name: "quarry-transfer-dir", + Usage: "ship sysupdate definitions for one component, given as =. is the component's sysupdate definitions directory. It is \"sysupdate.d\" for the default component or \"sysupdate..d\". is a directory of definitions (*.transfer, *.feature, .component and drop-in directories) or \"" + confextSourcePrefix + "\" for a generated transfer installing the confext DDI _.confext.raw. Repeat the option for the same to merge sources. The files are published under " + transferlayout.Prefix + "/. For a named component, .component is generated (without Enabled=, so it is an enabled-by-default component) if no source ships it", + }, + &cli.StringFlag{ + Name: "custom-version", + Usage: "set custom.quarry.version on every target outside " + transferlayout.Prefix, + }, + } +} + +// quarryTransferDir is one of --quarry-transfer-dir with its +// sources. +type quarryTransferDir struct { + dir string + sources []string +} + +// quarrySpecFromFlags parses and checks the --quarry-* options. +func quarrySpecFromFlags(cmd *cli.Command) ([]*quarryTransferDir, error) { + var dirs []*quarryTransferDir + byDir := map[string]*quarryTransferDir{} + for _, arg := range cmd.StringSlice("quarry-transfer-dir") { + dir, source, ok := strings.Cut(arg, "=") + if !ok || source == "" { + return nil, fmt.Errorf("--quarry-transfer-dir=%q must be =", arg) + } + if _, err := transferlayout.ParseComponentDir(dir); err != nil { + return nil, fmt.Errorf("--quarry-transfer-dir=%q: %w", arg, err) + } + entry, ok := byDir[dir] + if !ok { + entry = &quarryTransferDir{dir: dir} + byDir[dir] = entry + dirs = append(dirs, entry) + } + if slices.Contains(entry.sources, source) { + return nil, fmt.Errorf("--quarry-transfer-dir=%q given twice", arg) + } + entry.sources = append(entry.sources, source) + } + return dirs, nil +} + +// confextSourcePrefix marks a --quarry-transfer-dir source that names a +// confext to generate the definitions for rather than a directory. +const confextSourcePrefix = "confext:" + +// hasSpaceOrControl returns whether s would break a definition line. +func hasSpaceOrControl(s string) bool { + return strings.ContainsFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) +} + +// confextTransfer returns the transfer definition installing the confext DDI +// "_.confext.raw" into the /var/lib/confexts/.raw.v/ +// directory, from which systemd picks the newest instance. The source pattern +// matches the image at any depth (**/), so a repository may serve it from a +// subdirectory as well as flat. The source path is a placeholder that +// quarry-sysupdate replaces with its proxy URL when staging. Verify=no is set +// because TUF already authenticated the bytes. +func confextTransfer(name string) []byte { + return []byte(`[Transfer] +Verify=no + +[Source] +Type=url-file +Path=__quarry_will_replace_this__ +MatchPattern=**/` + name + `_@v.confext.raw + +[Target] +Type=regular-file +Path=/var/lib/confexts/` + name + `.raw.v +PathRelativeTo=root +MatchPattern=` + name + `_@v.raw +Mode=0644 +InstancesMax=2 +`) +} + +// readDefinitionSource adds the definition files of one source directory to +// files, keyed by their path relative to it, or generates them for a confext +// source. A file name that another source already added is an error. +func readDefinitionSource(files map[string][]byte, source string) error { + add := func(rel string, data []byte) error { + if _, dup := files[rel]; dup { + return fmt.Errorf("%s: definition file %s is also in another source", source, rel) + } + files[rel] = data + return nil + } + if name, ok := strings.CutPrefix(source, confextSourcePrefix); ok { + // systemd reads the confext name up to the first "_" of the file name. + if !transferlayout.ValidPathComponent(name) || strings.Contains(name, "_") || hasSpaceOrControl(name) { + return fmt.Errorf("invalid confext name %q in %s", name, source) + } + return add("50-"+name+".transfer", confextTransfer(name)) + } + found := false + err := filepath.WalkDir(source, func(path string, entry fs.DirEntry, err error) error { //nolint:forbidigo // user-controlled host path + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + rel, err := filepath.Rel(source, path) //nolint:forbidigo // user-controlled host path + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + info, err := entry.Info() + if err != nil { + return err + } + if info.Size() > maxInlineSize { + return fmt.Errorf("definition file %s is too large (%d bytes) to be inlined (limit %d bytes)", path, info.Size(), maxInlineSize) + } + data, err := os.ReadFile(path) //nolint:forbidigo // user-controlled host path + if err != nil { + return err + } + found = true + return add(rel, data) + }) + if err != nil { + return err + } + if !found { + return fmt.Errorf("%s holds no definition files", source) + } + return nil +} + +// layoutTransferDir turns the sources of one transfer directory into its +// files. For a named component, these include the generated component file +// if no source ships one. +func layoutTransferDir(entry *quarryTransferDir) (map[string][]byte, error) { + files := map[string][]byte{} + for _, source := range entry.sources { + if err := readDefinitionSource(files, source); err != nil { + return nil, err + } + } + // quarrySpecFromFlags already checked the directory name. + component, _ := transferlayout.ParseComponentDir(entry.dir) + if component == "" { + return files, nil + } + componentFile := component + ".component" + if _, shipped := files[componentFile]; !shipped { + files[componentFile] = transferlayout.ComponentFile(component) + } + return files, nil +} + +// addQuarryTargets adds what the --quarry-* options ask for to the targets. +func addQuarryTargets(builder *tufext.TargetsBuilder, dirs []*quarryTransferDir) error { + for _, entry := range dirs { + files, err := layoutTransferDir(entry) + if err != nil { + return err + } + for _, rel := range slices.Sorted(maps.Keys(files)) { + path, err := transferlayout.TargetPath(entry.dir, rel) + if err != nil { + return fmt.Errorf("%s: %s: %w", entry.dir, rel, err) + } + if err := rejectExisting(builder, path); err != nil { + return err + } + if _, err := addInlineBytes(builder, path, files[rel]); err != nil { + return err + } + } + } + return nil +} + +// rejectExisting returns an error if the targets already hold path. The +// builder would otherwise silently replace the entry (a generated entry +// colliding with a hashed or merged one). +func rejectExisting(builder *tufext.TargetsBuilder, path string) error { + if _, dup := builder.TargetsType().Targets[path]; dup { + return fmt.Errorf("target %s is given twice", path) + } + return nil +} + +// setQuarryCustomVersion sets custom.quarry.version on a target, keeping its +// other custom fields. +func setQuarryCustomVersion(target *tufmetadata.TargetFiles, version string) error { + return updateCustom(target, func(custom map[string]json.RawMessage) error { + quarry := map[string]json.RawMessage{} + if raw, ok := custom[tufext.QuarryCustomField]; ok { + if err := json.Unmarshal(raw, &quarry); err != nil { + return fmt.Errorf("existing custom.quarry data is not an object: %w", err) + } + } + if quarry == nil { // "quarry": null + quarry = map[string]json.RawMessage{} + } + encodedVersion, err := json.Marshal(version) + if err != nil { + return err + } + quarry["version"] = encodedVersion + encodedQuarry, err := jsonutils.MarshalNoEscapeHTML(quarry) + if err != nil { + return err + } + custom[tufext.QuarryCustomField] = encodedQuarry + return nil + }) +} diff --git a/cmd/hardhat/targets_quarry_test.go b/cmd/hardhat/targets_quarry_test.go new file mode 100644 index 0000000..d9843df --- /dev/null +++ b/cmd/hardhat/targets_quarry_test.go @@ -0,0 +1,196 @@ +// Copyright (C) 2026 Amutable GmbH + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v3" + + "go.amutable.dev/quarry/internal/tufext" +) + +func writeDefinitions(t *testing.T, dir string, files map[string]string) string { + t.Helper() + for rel, content := range files { + path := filepath.Join(dir, rel) //nolint:forbidigo // test code + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) //nolint:forbidigo // test code + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) //nolint:forbidigo // test code + } + return dir +} + +func inlineString(t *testing.T, builder *tufext.TargetsBuilder, path string) string { + t.Helper() + target := builder.TargetsType().Targets[path] + require.NotNil(t, target, "target %s", path) + data, err := tufext.TargetFilesExt(target).InlineData() + require.NoError(t, err) + return string(data) +} + +func TestAddQuarryTargets(t *testing.T) { + base := writeDefinitions(t, t.TempDir(), map[string]string{ + "10-usr.transfer": "[Transfer]\nProtectVersion=%A\n\n[Source]\nType=url-file\n", + "10-usr.transfer.d/x.conf": "[Transfer]\nVerify=no\n", + "docker.feature": "[Feature]\nSuggestOnMachineTag=amutable.ext.docker\n", + }) + k8s := writeDefinitions(t, t.TempDir(), map[string]string{ + "k8s.transfer": "[Transfer]\n\n[Source]\nType=url-file\n", + }) + shipped := writeDefinitions(t, t.TempDir(), map[string]string{ + "own.transfer": "[Transfer]\n", + "own.component": "[Component]\nDescription=Own\n", + }) + spec := []*quarryTransferDir{ + {dir: "sysupdate.d", sources: []string{base}}, + {dir: "sysupdate.k8s.d", sources: []string{k8s}}, + {dir: "sysupdate.own.d", sources: []string{shipped}}, + } + builder := tufext.NewTargetsBuilder() + require.NoError(t, addQuarryTargets(builder, spec)) + + // The files are published as they are. + const baseDir = ".quarry/sysupdate.d/" + assert.Equal(t, "[Transfer]\nProtectVersion=%A\n\n[Source]\nType=url-file\n", inlineString(t, builder, baseDir+"10-usr.transfer")) + assert.Equal(t, "[Transfer]\nVerify=no\n", inlineString(t, builder, baseDir+"10-usr.transfer.d/x.conf")) + assert.Equal(t, "[Feature]\nSuggestOnMachineTag=amutable.ext.docker\n", inlineString(t, builder, baseDir+"docker.feature")) + + const k8sDir = ".quarry/sysupdate.k8s.d/" + assert.Equal(t, "[Component]\nDescription=k8s\n", + inlineString(t, builder, k8sDir+"k8s.component"), "a missing component file is generated") + assert.Equal(t, "[Component]\nDescription=Own\n", inlineString(t, builder, ".quarry/sysupdate.own.d/own.component")) + + targets := builder.TargetsType().Targets + assert.Len(t, targets, 7) + for path, target := range targets { + assert.Nil(t, target.Custom, "%s: quarry metadata belongs to no version", path) + } +} + +func TestAddQuarryTargetsRejects(t *testing.T) { + dir := writeDefinitions(t, t.TempDir(), map[string]string{}) + spec := []*quarryTransferDir{{dir: "sysupdate.d", sources: []string{dir}}} + require.ErrorContains(t, addQuarryTargets(tufext.NewTargetsBuilder(), spec), "holds no definition files") + + // A path already in the targets is an error rather than a silent + // replacement. + dir = writeDefinitions(t, t.TempDir(), map[string]string{"x.transfer": "[Transfer]\n"}) + builder := tufext.NewTargetsBuilder() + _, err := addInlineBytes(builder, ".quarry/sysupdate.d/x.transfer", nil) + require.NoError(t, err) + spec = []*quarryTransferDir{{dir: "sysupdate.d", sources: []string{dir}}} + require.ErrorContains(t, addQuarryTargets(builder, spec), "given twice") +} + +// A confext source generates the definitions. The transfer comes from +// hardhat's template, and the component file is generated as for any component +// directory without one. +func TestAddQuarryTargets_Confext(t *testing.T) { + spec := []*quarryTransferDir{ + {dir: "sysupdate.edge-gw.d", sources: []string{"confext:edge-gw"}}, + } + builder := tufext.NewTargetsBuilder() + require.NoError(t, addQuarryTargets(builder, spec)) + + const dir = ".quarry/sysupdate.edge-gw.d/" + transfer := inlineString(t, builder, dir+"50-edge-gw.transfer") + assert.True(t, strings.HasPrefix(transfer, "[Transfer]\nVerify=no\n"), transfer) + for _, want := range []string{ + "MatchPattern=**/edge-gw_@v.confext.raw\n", + "Path=/var/lib/confexts/edge-gw.raw.v\n", + "MatchPattern=edge-gw_@v.raw\n", + } { + assert.Contains(t, transfer, want) + } + assert.Equal(t, "[Component]\nDescription=edge-gw\n", inlineString(t, builder, dir+"edge-gw.component")) + + // Several sources merge into one directory. A file in two of them is an error. + extra := writeDefinitions(t, t.TempDir(), map[string]string{"60-extra.transfer": "[Transfer]\n"}) + spec = []*quarryTransferDir{ + {dir: "sysupdate.gw.d", sources: []string{"confext:gw", "confext:gw-certs", extra}}, + } + builder = tufext.NewTargetsBuilder() + require.NoError(t, addQuarryTargets(builder, spec)) + assert.Contains(t, inlineString(t, builder, ".quarry/sysupdate.gw.d/50-gw.transfer"), "MatchPattern=**/gw_@v.confext.raw\n") + assert.Contains(t, inlineString(t, builder, ".quarry/sysupdate.gw.d/50-gw-certs.transfer"), "MatchPattern=**/gw-certs_@v.confext.raw\n") + assert.Equal(t, "[Transfer]\n", inlineString(t, builder, ".quarry/sysupdate.gw.d/60-extra.transfer")) + spec = []*quarryTransferDir{ + {dir: "sysupdate.gw.d", sources: []string{"confext:gw", extra, extra}}, + } + require.ErrorContains(t, addQuarryTargets(tufext.NewTargetsBuilder(), spec), "also in another source") + + for _, name := range []string{"", "a_b", "a/b", "..", "a b", "a\nb"} { + spec := []*quarryTransferDir{ + {dir: "sysupdate.x.d", sources: []string{"confext:" + name}}, + } + require.Error(t, addQuarryTargets(tufext.NewTargetsBuilder(), spec), "confext name %q", name) + } +} + +func TestSetQuarryCustomVersion(t *testing.T) { + builder := tufext.NewTargetsBuilder() + target, err := addInlineBytes(builder, "x", []byte("x")) + require.NoError(t, err) + // Existing custom fields, inside and outside quarry, survive. + existing := json.RawMessage(`{"quarry": {"update": true}, "other": 1}`) + target.Custom = &existing + require.NoError(t, setQuarryCustomVersion(target, "2")) + assert.JSONEq(t, `{"quarry": {"update": true, "version": "2"}, "other": 1}`, string(*target.Custom)) + + null := json.RawMessage(`{"quarry": null}`) + target.Custom = &null + require.NoError(t, setQuarryCustomVersion(target, "3")) + assert.JSONEq(t, `{"quarry": {"version": "3"}}`, string(*target.Custom)) +} + +// quarrySpecFor runs quarrySpecFromFlags on a dummy command with the args. +func quarrySpecFor(t *testing.T, args ...string) ([]*quarryTransferDir, error) { + t.Helper() + var ( + spec []*quarryTransferDir + specErr error + ) + cmd := &cli.Command{ + Name: "targets", + Flags: quarryTargetsFlags(), + Action: func(_ context.Context, cmd *cli.Command) error { + spec, specErr = quarrySpecFromFlags(cmd) + return nil + }, + } + require.NoError(t, cmd.Run(t.Context(), append([]string{"targets"}, args...))) + return spec, specErr +} + +func TestQuarrySpecFromFlags(t *testing.T) { + spec, err := quarrySpecFor(t, + "--quarry-transfer-dir=sysupdate.gw.d=confext:gw", + "--quarry-transfer-dir=sysupdate.d=/defs", + "--quarry-transfer-dir=sysupdate.gw.d=confext:gw-certs", + "--quarry-transfer-dir=sysupdate.k8s.d=/k8s", + ) + require.NoError(t, err) + assert.Equal(t, []*quarryTransferDir{ + {dir: "sysupdate.gw.d", sources: []string{"confext:gw", "confext:gw-certs"}}, + {dir: "sysupdate.d", sources: []string{"/defs"}}, + {dir: "sysupdate.k8s.d", sources: []string{"/k8s"}}, + }, spec) + + const dir = "--quarry-transfer-dir=sysupdate.x.d=/defs" + for name, args := range map[string][]string{ + "invalid component dir": {"--quarry-transfer-dir=other.d=/defs"}, + "no source": {"--quarry-transfer-dir=sysupdate.d="}, + "repeated source": {dir, dir}, + } { + _, err := quarrySpecFor(t, args...) + require.Error(t, err, name) + } +} diff --git a/cmd/hardhat/utils_ux.go b/cmd/hardhat/utils_ux.go index dd49827..e40d9ad 100644 --- a/cmd/hardhat/utils_ux.go +++ b/cmd/hardhat/utils_ux.go @@ -26,16 +26,31 @@ type ctxKey string const keystoreCtxKey ctxKey = "--keystore" func withKeystoreFlag(cmd *cli.Command) *cli.Command { + return withKeystoreFlagUnless(cmd, "") +} + +// withKeystoreFlagUnless is withKeystoreFlag for a command that needs no +// keystore when its boolean flag unless is set. Without that flag, the +// keystore is required as usual. +func withKeystoreFlagUnless(cmd *cli.Command, unless string) *cli.Command { cmd.Flags = append(cmd.Flags, &cli.StringFlag{ Name: "keystore", Usage: "path to the key store of signing keys", - Required: true, + Required: unless == "", TakesFile: true, Value: "/var/lib/quarry/keystore", Sources: cli.EnvVars("QUARRY_KEYSTORE"), }) cmd.Before = cliext.WrapBeforeFuncs(cmd.Before, func(ctx context.Context, cmd *cli.Command) (context.Context, error) { + if unless != "" { + if cmd.Bool(unless) { + return ctx, nil + } + if !cmd.IsSet("keystore") { + return nil, fmt.Errorf("--keystore is required without --%s", unless) + } + } keystorePath := cmd.String("keystore") if err := os.MkdirAll(keystorePath, 0o700); err != nil { //nolint:forbidigo // user-controlled host path return nil, fmt.Errorf("invalid keystore: could not create directory: %w", err) diff --git a/internal/transferlayout/layout.go b/internal/transferlayout/layout.go new file mode 100644 index 0000000..d4463e1 --- /dev/null +++ b/internal/transferlayout/layout.go @@ -0,0 +1,78 @@ +// Copyright (C) 2026 Amutable GmbH + +// Package transferlayout describes the repository layout of what quarry +// ships as special TUF targets for systemd-sysupdate. This covers the +// transfer definition directories under ".quarry/". +// +// hardhat writes this layout ("hardhat targets --quarry-*") and +// quarry-sysupdate (internal/xsysupdate) reads it, so both sides share the +// names and rules here. +package transferlayout + +import ( + "fmt" + "strings" +) + +const ( + // Prefix is the target path prefix of everything quarry-specific in a + // repository. Such a target is never a file that a client installs + // directly. + Prefix = ".quarry/" + + // defaultDir is the definitions directory of the default (component-less) + // sysupdate component. The directory of a named component is + // "sysupdate..d". + defaultDir = "sysupdate.d" +) + +// ValidPathComponent returns whether name is usable as one path element. +func ValidPathComponent(name string) bool { + return name != "" && name != "." && name != ".." && !strings.Contains(name, "/") +} + +// ComponentDir returns the definitions directory name of a component. The +// component "" names the default component. +func ComponentDir(component string) string { + if component == "" { + return defaultDir + } + return "sysupdate." + component + ".d" +} + +// ParseComponentDir returns the component a definitions directory name is +// for. It returns "" for the default component's directory "sysupdate.d". +func ParseComponentDir(dir string) (string, error) { + if dir == defaultDir { + return "", nil + } + if !strings.HasPrefix(dir, "sysupdate.") || !strings.HasSuffix(dir, ".d") { + return "", fmt.Errorf("invalid definitions directory name %q", dir) + } + component := strings.TrimSuffix(strings.TrimPrefix(dir, "sysupdate."), ".d") + if !ValidPathComponent(component) || strings.Contains(component, ".") { + return "", fmt.Errorf("invalid component name %q in directory %q", component, dir) + } + return component, nil +} + +// TargetPath returns the target path of one file of a transfer definition +// directory, which is "/". +func TargetPath(dir, file string) (string, error) { + if _, err := ParseComponentDir(dir); err != nil { + return "", err + } + for part := range strings.SplitSeq(file, "/") { + if !ValidPathComponent(part) { + return "", fmt.Errorf("invalid transfer file path %q", file) + } + } + return Prefix + dir + "/" + file, nil +} + +// ComponentFile returns the generated ".component" definition for +// a component directory that ships none. It sets no Enabled=, so it is an +// enabled-by-default component as in sysupdate. +func ComponentFile(component string) []byte { + return []byte("[Component]\nDescription=" + component + "\n") +} diff --git a/internal/transferlayout/layout_test.go b/internal/transferlayout/layout_test.go new file mode 100644 index 0000000..656fb59 --- /dev/null +++ b/internal/transferlayout/layout_test.go @@ -0,0 +1,48 @@ +// Copyright (C) 2026 Amutable GmbH + +package transferlayout + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTargetPath(t *testing.T) { + path, err := TargetPath("sysupdate.d", "12-usr.transfer") + require.NoError(t, err) + assert.Equal(t, ".quarry/sysupdate.d/12-usr.transfer", path) + + path, err = TargetPath("sysupdate.k8s.d", "k8s.transfer.d/10-x.conf") + require.NoError(t, err) + assert.Equal(t, ".quarry/sysupdate.k8s.d/k8s.transfer.d/10-x.conf", path) + + for _, tc := range []struct{ dir, file string }{ + {"other.d", "x.transfer"}, + {"sysupdate.a.b.d", "x.transfer"}, + {"sysupdate.d", ""}, + {"sysupdate.d", "a/../b"}, + } { + _, err := TargetPath(tc.dir, tc.file) + require.Error(t, err, "%v", tc) + } +} + +func TestComponentDir(t *testing.T) { + assert.Equal(t, "sysupdate.d", ComponentDir("")) + assert.Equal(t, "sysupdate.k8s.d", ComponentDir("k8s")) + for _, dir := range []string{"sysupdate.a.b.d", "sysupdate..d"} { + _, err := ParseComponentDir(dir) + require.Error(t, err, dir) + } + for _, dir := range []string{"sysupdate.d", "sysupdate.k8s.d"} { + component, err := ParseComponentDir(dir) + require.NoError(t, err) + assert.Equal(t, dir, ComponentDir(component)) + } +} + +func TestComponentFile(t *testing.T) { + assert.Equal(t, "[Component]\nDescription=k8s\n", string(ComponentFile("k8s"))) +} diff --git a/internal/tufext/quarrycustom.go b/internal/tufext/quarrycustom.go new file mode 100644 index 0000000..a8365fe --- /dev/null +++ b/internal/tufext/quarrycustom.go @@ -0,0 +1,8 @@ +// Copyright (C) 2026 Amutable GmbH + +package tufext + +// QuarryCustomField is the sub-field of the TUF "custom" target field that +// holds the Quarry-specific target metadata, as written by the build ("hardhat +// targets" --custom-from, --custom-version). +const QuarryCustomField = "quarry"