From ba63cc24195e857936809fa4f2c284a537de206f Mon Sep 17 00:00:00 2001 From: Roberto Hidalgo Date: Sun, 18 Dec 2022 12:16:46 -0600 Subject: [PATCH] implement fetch and flush --- cmd/fetch.go | 79 +++++++++++++++++++++++++ cmd/flush.go | 58 ++++++++++++++++++ internal/op-client/cli.go | 55 +++++------------ internal/op-client/op.go | 32 +++++++++- pkg/config/config.go | 45 ++++++++++---- pkg/config/entry.go | 120 ++++++++++++++++++++++---------------- pkg/config/readers.go | 3 + pkg/config/util.go | 4 +- 8 files changed, 293 insertions(+), 103 deletions(-) diff --git a/cmd/fetch.go b/cmd/fetch.go index 1825cd6..b3821f7 100644 --- a/cmd/fetch.go +++ b/cmd/fetch.go @@ -11,3 +11,82 @@ // See the License for the specific language governing permissions and // limitations under the License. package cmd + +import ( + "fmt" + "io/fs" + "os" + + "git.rob.mx/nidito/joao/internal/command" + "git.rob.mx/nidito/joao/internal/registry" + "git.rob.mx/nidito/joao/pkg/config" + "github.com/sirupsen/logrus" +) + +func init() { + registry.Register(fetchCommand) +} + +var fetchCommand = (&command.Command{ + Path: []string{"fetch"}, + Summary: "fetches configuration values from 1Password", + Description: `Fetches secrets for local ﹅CONFIG﹅ files from 1Password.`, + Arguments: command.Arguments{ + { + Name: "config", + Description: "The configuration file(s) to fetch", + Required: false, + Variadic: true, + Values: &command.ValueSource{ + Files: &[]string{"joao.yaml"}, + }, + }, + }, + Options: command.Options{ + "dry-run": { + Description: "Don't persist to the filesystem", + Type: "bool", + }, + }, + Action: func(cmd *command.Command) error { + paths := cmd.Arguments[0].ToValue().([]string) + for _, path := range paths { + remote, err := config.Load(path, true) + if err != nil { + return err + } + local, err := config.Load(path, false) + if err != nil { + return err + } + + if err = local.Merge(remote); err != nil { + return err + } + + b, err := local.AsYAML(false) + if err != nil { + return err + } + + if dryRun := cmd.Options["dry-run"].ToValue().(bool); dryRun { + logrus.Warnf("dry-run: would write to %s", path) + _, _ = cmd.Cobra.OutOrStdout().Write(b) + } else { + var mode fs.FileMode = 0644 + if info, err := os.Stat(path); err == nil { + mode = info.Mode().Perm() + } + + if err := os.WriteFile(path, b, mode); err != nil { + return fmt.Errorf("could not save changes to %s: %w", path, err) + } + } + + logrus.Infof("Updated %s", path) + } + + logrus.Info("Done") + return nil + }, +}).SetBindings() diff --git a/cmd/flush.go b/cmd/flush.go index 1825cd6..1946f9a 100644 --- a/cmd/flush.go +++ b/cmd/flush.go @@ -11,3 +11,61 @@ // See the License for the specific language governing permissions and // limitations under the License. package cmd + +import ( + "fmt" + + "git.rob.mx/nidito/joao/internal/command" + opclient "git.rob.mx/nidito/joao/internal/op-client" + "git.rob.mx/nidito/joao/internal/registry" + "git.rob.mx/nidito/joao/pkg/config" + "github.com/sirupsen/logrus" +) + +func init() { + registry.Register(flushCommand) +} + +var flushCommand = (&command.Command{ + Path: []string{"flush"}, + Summary: "flush configuration values to 1Password", + Description: `Creates or updates existing items for every ﹅CONFIG﹅ file provided. Does not delete 1Password items.`, + Arguments: command.Arguments{ + { + Name: "config", + Description: "The configuration file(s) to flush", + Required: false, + Variadic: true, + Values: &command.ValueSource{ + Files: &[]string{"yaml"}, + }, + }, + }, + Options: command.Options{ + "dry-run": { + Description: "Don't persist to 1Password", + Type: "bool", + }, + }, + Action: func(cmd *command.Command) error { + paths := cmd.Arguments[0].ToValue().([]string) + + if dryRun := cmd.Options["dry-run"].ToValue().(bool); dryRun { + opclient.Use(&opclient.CLI{DryRun: true}) + } + + for _, path := range paths { + cfg, err := config.Load(path, false) + if err != nil { + return err + } + + if err := opclient.Update(cfg.Vault, cfg.Name, cfg.ToOP()); err != nil { + return fmt.Errorf("could not flush to 1password: %w", err) + } + } + + logrus.Info("Done") + return nil + }, +}).SetBindings() diff --git a/internal/op-client/cli.go b/internal/op-client/cli.go index 3d5c471..7f23d7b 100644 --- a/internal/op-client/cli.go +++ b/internal/op-client/cli.go @@ -25,7 +25,9 @@ import ( "github.com/sirupsen/logrus" ) -type CLI struct{} +type CLI struct { + DryRun bool // Won't write to 1Password +} func invoke(vault string, args ...string) (bytes.Buffer, error) { if vault != "" { @@ -74,9 +76,9 @@ func (b *CLI) Get(vault, name string) (*op.Item, error) { return item, nil } -func (b *CLI) create(item *op.Item) error { +func (b *CLI) Create(item *op.Item) error { logrus.Infof("Creating new item: %s/%s", item.Vault.ID, item.Title) - cmd := exec.Command("op", "--vault", shellescape.Quote(item.Vault.ID), "item", "create") + cmd := exec.Command("op", "--vault", shellescape.Quote(item.Vault.ID), "item", "create") // nolint: gosec itemJSON, err := json.Marshal(item) if err != nil { @@ -89,6 +91,10 @@ func (b *CLI) create(item *op.Item) error { cmd.Stdout = &stdout var stderr bytes.Buffer cmd.Stderr = &stderr + if b.DryRun { + logrus.Warnf("dry-run: Would have invoked %v", cmd.Args) + return nil + } if err := cmd.Run(); err != nil { return fmt.Errorf("could not create item: %w", err) } @@ -100,41 +106,8 @@ func (b *CLI) create(item *op.Item) error { return nil } -type hashResult int - -const ( - HashItemError hashResult = iota - HashItemMissing - HashMatch - HashMismatch -) - -func keyForField(field *op.ItemField) string { - name := strings.ReplaceAll(field.Label, ".", "\\.") - if field.Section != nil { - name = field.Section.ID + "." + name - } - return name -} - -func (b *CLI) Update(vault, name string, item *op.Item) error { - remote, err := b.Get(vault, name) - if err != nil { - if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) { - return b.create(item) - } - - return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err) - } - - if remote.GetValue("password") == item.GetValue("password") { - logrus.Warn("item is already up to date") - return nil - } - - logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title) - - args := []string{"item", "edit", name, "--"} +func (b *CLI) Update(item *op.Item, remote *op.Item) error { + args := []string{"item", "edit", item.Title, "--"} localKeys := map[string]int{} for _, field := range item.Fields { @@ -158,7 +131,11 @@ func (b *CLI) Update(vault, name string, item *op.Item) error { } } - stdout, err := invoke(vault, args...) + if b.DryRun { + logrus.Warnf("dry-run: Would have invoked op %v", args) + return nil + } + stdout, err := invoke(item.Vault.ID, args...) if err != nil { logrus.Errorf("op stderr: %s", stdout.String()) return err diff --git a/internal/op-client/op.go b/internal/op-client/op.go index 8c5f84c..2da6b87 100644 --- a/internal/op-client/op.go +++ b/internal/op-client/op.go @@ -13,14 +13,19 @@ package opclient import ( + "fmt" + "strings" + op "github.com/1Password/connect-sdk-go/onepassword" + "github.com/sirupsen/logrus" ) var client opClient type opClient interface { Get(vault, name string) (*op.Item, error) - Update(vault, name string, item *op.Item) error + Update(item *op.Item, remote *op.Item) error + Create(item *op.Item) error List(vault, prefix string) ([]string, error) } @@ -37,9 +42,32 @@ func Get(vault, name string) (*op.Item, error) { } func Update(vault, name string, item *op.Item) error { - return client.Update(vault, name, item) + remote, err := client.Get(vault, name) + if err != nil { + if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) { + return client.Create(item) + } + + return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err) + } + + if remote.GetValue("password") == item.GetValue("password") { + logrus.Warn("item is already up to date") + return nil + } + + logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title) + return client.Update(item, remote) } func List(vault, prefix string) ([]string, error) { return client.List(vault, prefix) } + +func keyForField(field *op.ItemField) string { + name := strings.ReplaceAll(field.Label, ".", "\\.") + if field.Section != nil { + name = field.Section.ID + "." + name + } + return name +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 83ecc63..04de7b2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -14,13 +14,14 @@ package config import ( "bytes" - "crypto/md5" "encoding/json" "fmt" + "sort" "strings" op "github.com/1Password/connect-sdk-go/onepassword" "github.com/sirupsen/logrus" + "golang.org/x/crypto/blake2b" "gopkg.in/yaml.v3" ) @@ -66,17 +67,39 @@ func (cfg *Config) ToMap() map[string]any { return ret } +func checksum(fields []*op.ItemField) string { + newHash, err := blake2b.New256(nil) + if err != nil { + panic(err) + } + // newHash := md5.New() + df := []string{} + for _, field := range fields { + if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") { + continue + } + label := field.Label + if field.Section != nil && field.Section.ID != "" { + label = field.Section.ID + "." + label + } + df = append(df, label+field.Value) + } + sort.Strings(df) + newHash.Write([]byte(strings.Join(df, ""))) + checksum := newHash.Sum(nil) + + return fmt.Sprintf("%x", checksum) +} + // ToOp turns a config into an 1Password Item. func (cfg *Config) ToOP() *op.Item { sections := []*op.ItemSection{annotationsSection} fields := append([]*op.ItemField{}, defaultItemFields...) - newHash := md5.New() datafields := cfg.Tree.ToOP() - for _, field := range datafields { - newHash.Write([]byte(field.ID + field.Value)) - } - fields[0].Value = fmt.Sprintf("%x", newHash.Sum(nil)) + cs := checksum(datafields) + + fields[0].Value = cs fields = append(fields, datafields...) for i := 0; i < len(cfg.Tree.Content); i += 2 { @@ -107,7 +130,7 @@ func (cfg *Config) MarshalYAML() (any, error) { return cfg.Tree.MarshalYAML() } -// AsYAML returns the config encoded as YAML +// AsYAML returns the config encoded as YAML. func (cfg *Config) AsYAML(redacted bool) ([]byte, error) { redactOutput = redacted var out bytes.Buffer @@ -221,21 +244,23 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er } if child := entry.ChildNamed(key); child != nil { - logrus.Infof("found child named %s, with len %v", key, len(child.Content)) entry = child continue } - logrus.Infof("no child named %s found in %s", key, entry.Name()) kind := yaml.MappingNode if isNumeric(key) { kind = yaml.SequenceNode } sub := NewEntry(key, kind) - sub.Path = append(entry.Path, key) + sub.Path = append(entry.Path, key) // nolint: gocritic entry.Content = append(entry.Content, sub) entry = sub } return nil } + +func (cfg *Config) Merge(other *Config) error { + return cfg.Tree.Merge(other.Tree) +} diff --git a/pkg/config/entry.go b/pkg/config/entry.go index f565edc..c5b76eb 100644 --- a/pkg/config/entry.go +++ b/pkg/config/entry.go @@ -14,7 +14,6 @@ package config import ( "fmt" - "strconv" "strings" op "github.com/1Password/connect-sdk-go/onepassword" @@ -31,10 +30,8 @@ func isNumeric(s string) bool { return true } -type secretValue string - // Entry is a configuration entry. -// Basically a copy of a yaml.Node with extra methods +// Basically a copy of a yaml.Node with extra methods. type Entry struct { Value string Kind yaml.Kind @@ -59,7 +56,7 @@ func NewEntry(name string, kind yaml.Kind) *Entry { } } -func copyFromNode(e *Entry, n *yaml.Node) *Entry { +func (e *Entry) copyFromNode(n *yaml.Node) { if e.Content == nil { e.Content = []*Entry{} } @@ -74,7 +71,6 @@ func copyFromNode(e *Entry, n *yaml.Node) *Entry { e.Line = n.Line e.Column = n.Column e.Type = n.ShortTag() - return e } func (e *Entry) String() string { @@ -91,7 +87,7 @@ func (e *Entry) ChildNamed(name string) *Entry { } func (e *Entry) SetPath(parent []string, current string) { - e.Path = append(parent, current) + e.Path = append(parent, current) // nolint: gocritic switch e.Kind { case yaml.MappingNode, yaml.DocumentNode: for idx := 0; idx < len(e.Content); idx += 2 { @@ -101,19 +97,19 @@ func (e *Entry) SetPath(parent []string, current string) { } case yaml.SequenceNode: for idx, child := range e.Content { - child.Path = append(e.Path, fmt.Sprintf("%d", idx)) + child.Path = append(e.Path, fmt.Sprintf("%d", idx)) // nolint: gocritic } } } func (e *Entry) UnmarshalYAML(node *yaml.Node) error { - copyFromNode(e, node) + e.copyFromNode(node) switch node.Kind { case yaml.SequenceNode, yaml.ScalarNode: for _, n := range node.Content { sub := &Entry{} - copyFromNode(sub, n) + sub.copyFromNode(n) if err := n.Decode(&sub); err != nil { return err } @@ -147,6 +143,10 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error { return nil } +func (e *Entry) IsScalar() bool { + return e.Kind != yaml.DocumentNode && e.Kind != yaml.MappingNode && e.Kind != yaml.SequenceNode +} + func (e *Entry) IsSecret() bool { return e.Tag == YAMLTypeSecret } @@ -216,8 +216,7 @@ func (e *Entry) FromOP(fields []*op.ItemField) error { annotations := map[string]string{} data := map[string]string{} - for i := 0; i < len(fields); i++ { - field := fields[i] + for _, field := range fields { label := field.Label if field.Section != nil { if field.Section.Label == "~annotations" { @@ -234,35 +233,12 @@ func (e *Entry) FromOP(fields []*op.ItemField) error { } for label, valueStr := range data { - var value any - var err error var style yaml.Style var tag string - switch annotations[label] { - case "bool": - value, err = strconv.ParseBool(valueStr) - if err != nil { - return err - } - case "int": - value, err = strconv.ParseInt(valueStr, 10, 64) - if err != nil { - return err - } - case "float": - var err error - value, err = strconv.ParseFloat(valueStr, 64) - if err != nil { - return err - } - case "secret": - value = secretValue(value.(string)) + if annotations[label] == "secret" { style = yaml.TaggedStyle tag = YAMLTypeSecret - default: - // either no annotation or an unknown value - value = valueStr } path := strings.Split(label, ".") @@ -270,7 +246,7 @@ func (e *Entry) FromOP(fields []*op.ItemField) error { for idx, key := range path { if idx == len(path)-1 { - container.Content = append(container.Content, &Entry{ + container.Content = append(container.Content, NewEntry(key, yaml.ScalarNode), &Entry{ Path: path, Kind: yaml.ScalarNode, Value: valueStr, @@ -289,8 +265,12 @@ func (e *Entry) FromOP(fields []*op.ItemField) error { kind = yaml.SequenceNode } child := NewEntry(key, kind) - child.Path = append(container.Path, key) - container.Content = append(container.Content, child) + child.Path = append(container.Path, key) // nolint: gocritic + if isNumeric(key) { + container.Content = append(container.Content, child) + } else { + container.Content = append(container.Content, NewEntry(child.Name(), child.Kind), child) + } container = child } } @@ -305,6 +285,7 @@ func (e *Entry) ToOP() []*op.ItemField { if e.Kind == yaml.ScalarNode { name := e.Path[len(e.Path)-1] + fullPath := strings.Join(e.Path, ".") if len(e.Path) > 1 { section = &op.ItemSection{ID: e.Path[0]} name = strings.Join(e.Path[1:], ".") @@ -313,20 +294,20 @@ func (e *Entry) ToOP() []*op.ItemField { fieldType := "STRING" if e.IsSecret() { fieldType = "CONCEALED" - } else { - if annotationType := e.TypeStr(); annotationType != "" { - ret = append(ret, &op.ItemField{ - ID: "~annotations." + strings.Join(e.Path, "."), - Section: annotationsSection, - Label: name, - Type: "STRING", - Value: annotationType, - }) - } + } + + if annotationType := e.TypeStr(); annotationType != "" { + ret = append(ret, &op.ItemField{ + ID: "~annotations." + fullPath, + Section: annotationsSection, + Label: fullPath, + Type: "STRING", + Value: annotationType, + }) } ret = append(ret, &op.ItemField{ - ID: strings.Join(e.Path, "."), + ID: fullPath, Section: section, Label: name, Type: fieldType, @@ -385,3 +366,42 @@ func (e *Entry) AsMap() any { } return ret } + +func (e *Entry) Merge(other *Entry) error { + if e.IsScalar() && other.IsScalar() { + e.Value = other.Value + e.Tag = other.Tag + e.Kind = other.Kind + e.Type = other.Type + return nil + } + + if e.Kind == yaml.MappingNode || e.Kind == yaml.DocumentNode { + for i := 0; i < len(other.Content); i += 2 { + remote := other.Content[i+1] + local := e.ChildNamed(remote.Name()) + if local != nil { + if err := local.Merge(remote); err != nil { + return err + } + } else { + e.Content = append(e.Content, NewEntry(remote.Name(), remote.Kind), remote) + } + } + return nil + } + + for _, remote := range other.Content { + local := other.ChildNamed(remote.Name()) + if local != nil { + if err := local.Merge(remote); err != nil { + return err + } + } else { + logrus.Debugf("adding new collection value at %s", remote.Path) + local.Content = append(local.Content, remote) + } + } + + return nil +} diff --git a/pkg/config/readers.go b/pkg/config/readers.go index 4a50fd0..4d33871 100644 --- a/pkg/config/readers.go +++ b/pkg/config/readers.go @@ -70,6 +70,9 @@ func FromOP(item *op.Item) (*Config, error) { Tree: NewEntry("root", yaml.MappingNode), } + if cs := checksum(item.Fields); cs != item.GetValue("password") { + logrus.Warnf("1Password item changed and checksum was not updated. Expected %s, found %s", cs, item.GetValue("password")) + } err := cfg.Tree.FromOP(item.Fields) return cfg, err } diff --git a/pkg/config/util.go b/pkg/config/util.go index a721ded..d9703b7 100644 --- a/pkg/config/util.go +++ b/pkg/config/util.go @@ -28,12 +28,12 @@ import ( type opDetails struct { Vault string `yaml:"vault"` Name string `yaml:"name"` - NameTemplate string `yaml:"nameTemplate"` + NameTemplate string `yaml:"nameTemplate"` // nolint: tagliatelle Repo string } type singleModeConfig struct { - Config *opDetails `yaml:"_config,omitempty"` + Config *opDetails `yaml:"_config,omitempty"` // nolint: tagliatelle } func argIsYAMLFile(path string) bool {