implement fetch and flush

cmd/fetch.go

@@ -11,3 +11,82 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package cmd
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+
+	"git.rob.mx/nidito/joao/internal/command"
+	"git.rob.mx/nidito/joao/internal/registry"
+	"git.rob.mx/nidito/joao/pkg/config"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	registry.Register(fetchCommand)
+}
+
+var fetchCommand = (&command.Command{
+	Path:        []string{"fetch"},
+	Summary:     "fetches configuration values from 1Password",
+	Description: `Fetches secrets for local ﹅CONFIG﹅ files from 1Password.`,
+	Arguments: command.Arguments{
+		{
+			Name:        "config",
+			Description: "The configuration file(s) to fetch",
+			Required:    false,
+			Variadic:    true,
+			Values: &command.ValueSource{
+				Files: &[]string{"joao.yaml"},
+			},
+		},
+	},
+	Options: command.Options{
+		"dry-run": {
+			Description: "Don't persist to the filesystem",
+			Type:        "bool",
+		},
+	},
+	Action: func(cmd *command.Command) error {
+		paths := cmd.Arguments[0].ToValue().([]string)
+		for _, path := range paths {
+			remote, err := config.Load(path, true)
+			if err != nil {
+				return err
+			}
+			local, err := config.Load(path, false)
+			if err != nil {
+				return err
+			}
+
+			if err = local.Merge(remote); err != nil {
+				return err
+			}
+
+			b, err := local.AsYAML(false)
+			if err != nil {
+				return err
+			}
+
+			if dryRun := cmd.Options["dry-run"].ToValue().(bool); dryRun {
+				logrus.Warnf("dry-run: would write to %s", path)
+				_, _ = cmd.Cobra.OutOrStdout().Write(b)
+			} else {
+				var mode fs.FileMode = 0644
+				if info, err := os.Stat(path); err == nil {
+					mode = info.Mode().Perm()
+				}
+
+				if err := os.WriteFile(path, b, mode); err != nil {
+					return fmt.Errorf("could not save changes to %s: %w", path, err)
+				}
+			}
+
+			logrus.Infof("Updated %s", path)
+		}
+
+		logrus.Info("Done")
+		return nil
+	},
+}).SetBindings()

cmd/flush.go

@@ -11,3 +11,61 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package cmd
+
+import (
+	"fmt"
+
+	"git.rob.mx/nidito/joao/internal/command"
+	opclient "git.rob.mx/nidito/joao/internal/op-client"
+	"git.rob.mx/nidito/joao/internal/registry"
+	"git.rob.mx/nidito/joao/pkg/config"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	registry.Register(flushCommand)
+}
+
+var flushCommand = (&command.Command{
+	Path:        []string{"flush"},
+	Summary:     "flush configuration values to 1Password",
+	Description: `Creates or updates existing items for every ﹅CONFIG﹅ file provided. Does not delete 1Password items.`,
+	Arguments: command.Arguments{
+		{
+			Name:        "config",
+			Description: "The configuration file(s) to flush",
+			Required:    false,
+			Variadic:    true,
+			Values: &command.ValueSource{
+				Files: &[]string{"yaml"},
+			},
+		},
+	},
+	Options: command.Options{
+		"dry-run": {
+			Description: "Don't persist to 1Password",
+			Type:        "bool",
+		},
+	},
+	Action: func(cmd *command.Command) error {
+		paths := cmd.Arguments[0].ToValue().([]string)
+
+		if dryRun := cmd.Options["dry-run"].ToValue().(bool); dryRun {
+			opclient.Use(&opclient.CLI{DryRun: true})
+		}
+
+		for _, path := range paths {
+			cfg, err := config.Load(path, false)
+			if err != nil {
+				return err
+			}
+
+			if err := opclient.Update(cfg.Vault, cfg.Name, cfg.ToOP()); err != nil {
+				return fmt.Errorf("could not flush to 1password: %w", err)
+			}
+		}
+
+		logrus.Info("Done")
+		return nil
+	},
+}).SetBindings()

internal/op-client/cli.go

@@ -25,7 +25,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type CLI struct{}
+type CLI struct {
+	DryRun bool // Won't write to 1Password
+}
 
 func invoke(vault string, args ...string) (bytes.Buffer, error) {
 	if vault != "" {
@@ -74,9 +76,9 @@ func (b *CLI) Get(vault, name string) (*op.Item, error) {
 	return item, nil
 }
 
-func (b *CLI) create(item *op.Item) error {
+func (b *CLI) Create(item *op.Item) error {
 	logrus.Infof("Creating new item: %s/%s", item.Vault.ID, item.Title)
-	cmd := exec.Command("op", "--vault", shellescape.Quote(item.Vault.ID), "item", "create")
+	cmd := exec.Command("op", "--vault", shellescape.Quote(item.Vault.ID), "item", "create") // nolint: gosec
 
 	itemJSON, err := json.Marshal(item)
 	if err != nil {
@@ -89,6 +91,10 @@ func (b *CLI) create(item *op.Item) error {
 	cmd.Stdout = &stdout
 	var stderr bytes.Buffer
 	cmd.Stderr = &stderr
+	if b.DryRun {
+		logrus.Warnf("dry-run: Would have invoked %v", cmd.Args)
+		return nil
+	}
 	if err := cmd.Run(); err != nil {
 		return fmt.Errorf("could not create item: %w", err)
 	}
@@ -100,41 +106,8 @@ func (b *CLI) create(item *op.Item) error {
 	return nil
 }
 
-type hashResult int
+func (b *CLI) Update(item *op.Item, remote *op.Item) error {
-
+	args := []string{"item", "edit", item.Title, "--"}
-const (
-	HashItemError hashResult = iota
-	HashItemMissing
-	HashMatch
-	HashMismatch
-)
-
-func keyForField(field *op.ItemField) string {
-	name := strings.ReplaceAll(field.Label, ".", "\\.")
-	if field.Section != nil {
-		name = field.Section.ID + "." + name
-	}
-	return name
-}
-
-func (b *CLI) Update(vault, name string, item *op.Item) error {
-	remote, err := b.Get(vault, name)
-	if err != nil {
-		if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) {
-			return b.create(item)
-		}
-
-		return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err)
-	}
-
-	if remote.GetValue("password") == item.GetValue("password") {
-		logrus.Warn("item is already up to date")
-		return nil
-	}
-
-	logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title)
-
-	args := []string{"item", "edit", name, "--"}
 	localKeys := map[string]int{}
 
 	for _, field := range item.Fields {
@@ -158,7 +131,11 @@ func (b *CLI) Update(vault, name string, item *op.Item) error {
 		}
 	}
 
-	stdout, err := invoke(vault, args...)
+	if b.DryRun {
+		logrus.Warnf("dry-run: Would have invoked op %v", args)
+		return nil
+	}
+	stdout, err := invoke(item.Vault.ID, args...)
 	if err != nil {
 		logrus.Errorf("op stderr: %s", stdout.String())
 		return err

internal/op-client/op.go

@@ -13,14 +13,19 @@
 package opclient
 
 import (
+	"fmt"
+	"strings"
+
 	op "github.com/1Password/connect-sdk-go/onepassword"
+	"github.com/sirupsen/logrus"
 )
 
 var client opClient
 
 type opClient interface {
 	Get(vault, name string) (*op.Item, error)
-	Update(vault, name string, item *op.Item) error
+	Update(item *op.Item, remote *op.Item) error
+	Create(item *op.Item) error
 	List(vault, prefix string) ([]string, error)
 }
 
@@ -37,9 +42,32 @@ func Get(vault, name string) (*op.Item, error) {
 }
 
 func Update(vault, name string, item *op.Item) error {
-	return client.Update(vault, name, item)
+	remote, err := client.Get(vault, name)
+	if err != nil {
+		if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) {
+			return client.Create(item)
+		}
+
+		return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err)
+	}
+
+	if remote.GetValue("password") == item.GetValue("password") {
+		logrus.Warn("item is already up to date")
+		return nil
+	}
+
+	logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title)
+	return client.Update(item, remote)
 }
 
 func List(vault, prefix string) ([]string, error) {
 	return client.List(vault, prefix)
 }
+
+func keyForField(field *op.ItemField) string {
+	name := strings.ReplaceAll(field.Label, ".", "\\.")
+	if field.Section != nil {
+		name = field.Section.ID + "." + name
+	}
+	return name
+}

pkg/config/config.go

@@ -14,13 +14,14 @@ package config
 
 import (
 	"bytes"
-	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"sort"
 	"strings"
 
 	op "github.com/1Password/connect-sdk-go/onepassword"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/crypto/blake2b"
 	"gopkg.in/yaml.v3"
 )
 
@@ -66,17 +67,39 @@ func (cfg *Config) ToMap() map[string]any {
 	return ret
 }
 
+func checksum(fields []*op.ItemField) string {
+	newHash, err := blake2b.New256(nil)
+	if err != nil {
+		panic(err)
+	}
+	// newHash := md5.New()
+	df := []string{}
+	for _, field := range fields {
+		if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") {
+			continue
+		}
+		label := field.Label
+		if field.Section != nil && field.Section.ID != "" {
+			label = field.Section.ID + "." + label
+		}
+		df = append(df, label+field.Value)
+	}
+	sort.Strings(df)
+	newHash.Write([]byte(strings.Join(df, "")))
+	checksum := newHash.Sum(nil)
+
+	return fmt.Sprintf("%x", checksum)
+}
+
 // ToOp turns a config into an 1Password Item.
 func (cfg *Config) ToOP() *op.Item {
 	sections := []*op.ItemSection{annotationsSection}
 	fields := append([]*op.ItemField{}, defaultItemFields...)
 
-	newHash := md5.New()
 	datafields := cfg.Tree.ToOP()
-	for _, field := range datafields {
+	cs := checksum(datafields)
-		newHash.Write([]byte(field.ID + field.Value))
+
-	}
+	fields[0].Value = cs
-	fields[0].Value = fmt.Sprintf("%x", newHash.Sum(nil))
 	fields = append(fields, datafields...)
 
 	for i := 0; i < len(cfg.Tree.Content); i += 2 {
@@ -107,7 +130,7 @@ func (cfg *Config) MarshalYAML() (any, error) {
 	return cfg.Tree.MarshalYAML()
 }
 
-// AsYAML returns the config encoded as YAML
+// AsYAML returns the config encoded as YAML.
 func (cfg *Config) AsYAML(redacted bool) ([]byte, error) {
 	redactOutput = redacted
 	var out bytes.Buffer
@@ -221,21 +244,23 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
 		}
 
 		if child := entry.ChildNamed(key); child != nil {
-			logrus.Infof("found child named %s, with len %v", key, len(child.Content))
 			entry = child
 			continue
 		}
 
-		logrus.Infof("no child named %s found in %s", key, entry.Name())
 		kind := yaml.MappingNode
 		if isNumeric(key) {
 			kind = yaml.SequenceNode
 		}
 		sub := NewEntry(key, kind)
-		sub.Path = append(entry.Path, key)
+		sub.Path = append(entry.Path, key) // nolint: gocritic
 		entry.Content = append(entry.Content, sub)
 		entry = sub
 	}
 
 	return nil
 }
+
+func (cfg *Config) Merge(other *Config) error {
+	return cfg.Tree.Merge(other.Tree)
+}

pkg/config/entry.go

@@ -14,7 +14,6 @@ package config
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 
 	op "github.com/1Password/connect-sdk-go/onepassword"
@@ -31,10 +30,8 @@ func isNumeric(s string) bool {
 	return true
 }
 
-type secretValue string
-
 // Entry is a configuration entry.
-// Basically a copy of a yaml.Node with extra methods
+// Basically a copy of a yaml.Node with extra methods.
 type Entry struct {
 	Value       string
 	Kind        yaml.Kind
@@ -59,7 +56,7 @@ func NewEntry(name string, kind yaml.Kind) *Entry {
 	}
 }
 
-func copyFromNode(e *Entry, n *yaml.Node) *Entry {
+func (e *Entry) copyFromNode(n *yaml.Node) {
 	if e.Content == nil {
 		e.Content = []*Entry{}
 	}
@@ -74,7 +71,6 @@ func copyFromNode(e *Entry, n *yaml.Node) *Entry {
 	e.Line = n.Line
 	e.Column = n.Column
 	e.Type = n.ShortTag()
-	return e
 }
 
 func (e *Entry) String() string {
@@ -91,7 +87,7 @@ func (e *Entry) ChildNamed(name string) *Entry {
 }
 
 func (e *Entry) SetPath(parent []string, current string) {
-	e.Path = append(parent, current)
+	e.Path = append(parent, current) // nolint: gocritic
 	switch e.Kind {
 	case yaml.MappingNode, yaml.DocumentNode:
 		for idx := 0; idx < len(e.Content); idx += 2 {
@@ -101,19 +97,19 @@ func (e *Entry) SetPath(parent []string, current string) {
 		}
 	case yaml.SequenceNode:
 		for idx, child := range e.Content {
-			child.Path = append(e.Path, fmt.Sprintf("%d", idx))
+			child.Path = append(e.Path, fmt.Sprintf("%d", idx)) // nolint: gocritic
 		}
 	}
 }
 
 func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
-	copyFromNode(e, node)
+	e.copyFromNode(node)
 
 	switch node.Kind {
 	case yaml.SequenceNode, yaml.ScalarNode:
 		for _, n := range node.Content {
 			sub := &Entry{}
-			copyFromNode(sub, n)
+			sub.copyFromNode(n)
 			if err := n.Decode(&sub); err != nil {
 				return err
 			}
@@ -147,6 +143,10 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
+func (e *Entry) IsScalar() bool {
+	return e.Kind != yaml.DocumentNode && e.Kind != yaml.MappingNode && e.Kind != yaml.SequenceNode
+}
+
 func (e *Entry) IsSecret() bool {
 	return e.Tag == YAMLTypeSecret
 }
@@ -216,8 +216,7 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
 	annotations := map[string]string{}
 	data := map[string]string{}
 
-	for i := 0; i < len(fields); i++ {
+	for _, field := range fields {
-		field := fields[i]
 		label := field.Label
 		if field.Section != nil {
 			if field.Section.Label == "~annotations" {
@@ -234,35 +233,12 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
 	}
 
 	for label, valueStr := range data {
-		var value any
-		var err error
 		var style yaml.Style
 		var tag string
 
-		switch annotations[label] {
+		if annotations[label] == "secret" {
-		case "bool":
-			value, err = strconv.ParseBool(valueStr)
-			if err != nil {
-				return err
-			}
-		case "int":
-			value, err = strconv.ParseInt(valueStr, 10, 64)
-			if err != nil {
-				return err
-			}
-		case "float":
-			var err error
-			value, err = strconv.ParseFloat(valueStr, 64)
-			if err != nil {
-				return err
-			}
-		case "secret":
-			value = secretValue(value.(string))
 			style = yaml.TaggedStyle
 			tag = YAMLTypeSecret
-		default:
-			// either no annotation or an unknown value
-			value = valueStr
 		}
 
 		path := strings.Split(label, ".")
@@ -270,7 +246,7 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
 
 		for idx, key := range path {
 			if idx == len(path)-1 {
-				container.Content = append(container.Content, &Entry{
+				container.Content = append(container.Content, NewEntry(key, yaml.ScalarNode), &Entry{
 					Path:  path,
 					Kind:  yaml.ScalarNode,
 					Value: valueStr,
@@ -289,8 +265,12 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
 					kind = yaml.SequenceNode
 				}
 				child := NewEntry(key, kind)
-				child.Path = append(container.Path, key)
+				child.Path = append(container.Path, key) // nolint: gocritic
+				if isNumeric(key) {
 					container.Content = append(container.Content, child)
+				} else {
+					container.Content = append(container.Content, NewEntry(child.Name(), child.Kind), child)
+				}
 				container = child
 			}
 		}
@@ -305,6 +285,7 @@ func (e *Entry) ToOP() []*op.ItemField {
 
 	if e.Kind == yaml.ScalarNode {
 		name := e.Path[len(e.Path)-1]
+		fullPath := strings.Join(e.Path, ".")
 		if len(e.Path) > 1 {
 			section = &op.ItemSection{ID: e.Path[0]}
 			name = strings.Join(e.Path[1:], ".")
@@ -313,20 +294,20 @@ func (e *Entry) ToOP() []*op.ItemField {
 		fieldType := "STRING"
 		if e.IsSecret() {
 			fieldType = "CONCEALED"
-		} else {
+		}
+
 		if annotationType := e.TypeStr(); annotationType != "" {
 			ret = append(ret, &op.ItemField{
-					ID:      "~annotations." + strings.Join(e.Path, "."),
+				ID:      "~annotations." + fullPath,
 				Section: annotationsSection,
-					Label:   name,
+				Label:   fullPath,
 				Type:    "STRING",
 				Value:   annotationType,
 			})
 		}
-		}
 
 		ret = append(ret, &op.ItemField{
-			ID:      strings.Join(e.Path, "."),
+			ID:      fullPath,
 			Section: section,
 			Label:   name,
 			Type:    fieldType,
@@ -385,3 +366,42 @@ func (e *Entry) AsMap() any {
 	}
 	return ret
 }
+
+func (e *Entry) Merge(other *Entry) error {
+	if e.IsScalar() && other.IsScalar() {
+		e.Value = other.Value
+		e.Tag = other.Tag
+		e.Kind = other.Kind
+		e.Type = other.Type
+		return nil
+	}
+
+	if e.Kind == yaml.MappingNode || e.Kind == yaml.DocumentNode {
+		for i := 0; i < len(other.Content); i += 2 {
+			remote := other.Content[i+1]
+			local := e.ChildNamed(remote.Name())
+			if local != nil {
+				if err := local.Merge(remote); err != nil {
+					return err
+				}
+			} else {
+				e.Content = append(e.Content, NewEntry(remote.Name(), remote.Kind), remote)
+			}
+		}
+		return nil
+	}
+
+	for _, remote := range other.Content {
+		local := other.ChildNamed(remote.Name())
+		if local != nil {
+			if err := local.Merge(remote); err != nil {
+				return err
+			}
+		} else {
+			logrus.Debugf("adding new collection value at %s", remote.Path)
+			local.Content = append(local.Content, remote)
+		}
+	}
+
+	return nil
+}

pkg/config/readers.go

@@ -70,6 +70,9 @@ func FromOP(item *op.Item) (*Config, error) {
 		Tree:  NewEntry("root", yaml.MappingNode),
 	}
 
+	if cs := checksum(item.Fields); cs != item.GetValue("password") {
+		logrus.Warnf("1Password item changed and checksum was not updated. Expected %s, found %s", cs, item.GetValue("password"))
+	}
 	err := cfg.Tree.FromOP(item.Fields)
 	return cfg, err
 }

pkg/config/util.go

@@ -28,12 +28,12 @@ import (
 type opDetails struct {
 	Vault        string `yaml:"vault"`
 	Name         string `yaml:"name"`
-	NameTemplate string `yaml:"nameTemplate"`
+	NameTemplate string `yaml:"nameTemplate"` // nolint: tagliatelle
 	Repo         string
 }
 
 type singleModeConfig struct {
-	Config *opDetails `yaml:"_config,omitempty"`
+	Config *opDetails `yaml:"_config,omitempty"` // nolint: tagliatelle
 }
 
 func argIsYAMLFile(path string) bool {